diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 27bae65..81d2c84 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Go uses: actions/setup-go@v6 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a16f013 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,30 @@ +name: release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + + - uses: goreleaser/goreleaser-action@a4f02ab5355a589de60d917a659ce617ed19b3de # v7.3.0 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..e95babf --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +version: 2 + +project_name: gloak + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + +archives: + - format: tar.gz + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + format_overrides: + - goos: windows + format: zip + +checksum: + name_template: checksums.txt + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + - "^style:" + - "^ci:" + - "^build:" + - "^refactor:" + - "^Merge pull request" + - "^Merge branch" diff --git a/README.md b/README.md index be19795..acf9af2 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,14 @@ > > Use it for testing, non-critical data, or secondary backups only. If you store anything important with gloak, keep another verified backup somewhere else. -gloak splits a file into fixed-size chunks, encrypts each chunk with age, uploads them, and verifies them before restore. Metadata such as filenames, sizes, recipients, and hashes lives inside an age-encrypted manifest. Storage only sees backup IDs, public completion markers, and encrypted chunk objects. +gloak splits one file into fixed-size chunks, encrypts each chunk with age, uploads them, and verifies every chunk before restore. Metadata such as filenames, sizes, recipients, and hashes lives inside an age-encrypted manifest. Storage only sees backup IDs, public completion markers, and encrypted chunk objects. ```text -keygen → chunk → encrypt chunks → upload → resume if needed → verify → restore +keygen -> chunk -> encrypt chunks -> upload -> verify -> restore ``` +Full usage guide: [docs/WORKFLOW.md](docs/WORKFLOW.md) + --- ## Quick start @@ -37,12 +39,9 @@ go build -o gloak . # generate keys ./gloak keygen -# encrypt + upload +# upload ./gloak upload ./secret.tar --remote myremote:backup_folder -# resume an interrupted upload -./gloak resume - # verify ./gloak verify --remote myremote:backup_folder -i ~/.gloak/identity.txt @@ -50,334 +49,83 @@ go build -o gloak . ./gloak restore --remote myremote:backup_folder --output-dir ./restored/ ``` ---- - -## Why this exists - -Most backup tools make at least one tradeoff gloak avoids: - -1. Encrypt a single blob. If a network failure hits at byte 47 GB of 50, retrying is painful. -2. Split into chunks without enough integrity metadata. You find corruption during restore, without a manifest to guide repair. -3. Store metadata in plaintext. Filenames, sizes, recipient keys, and chunk hashes are visible to anyone who can read the storage. -4. Pull in rclone, restic rest-server, or S3 libraries when you just need local filesystem or a mounted remote. - -gloak keeps the design narrow: age encryption, fixed-size chunks, chunk and payload hashes, resumable uploads, and an encrypted manifest so the storage backend does not see filenames or hashes. - ---- - -## Install - -```bash -go mod download -``` - -Build the standalone binary: - -```bash -go build -o gloak . -``` - -This produces `gloak`, a compiled binary you can put in containers, cron jobs, or VPS root filesystems without installing Node, Bun, or Python. - -Run tests: - -```bash -go test ./... -``` - ---- - -## Key generation - -Generate a keypair once: - -```bash -./gloak keygen -``` - -Default location is `~/.gloak/`: - -```text -~/.gloak/ - identity.txt # private key, keep this secret - recipient.txt # public key, safe to share -``` - -Use a custom directory: - -```bash -./gloak keygen --output-dir /path/to/keys -./gloak keygen -o /path/to/keys -``` - -After `keygen` runs once, `-i` (identity) and `-r` (recipient) flags become optional. gloak reads from `~/.gloak/` automatically. - -### Key flags - -| Flag | Alias | Description | -|------|-------|-------------| -| `-r ` | `--recipient` | Recipient public key file. Overrides `~/.gloak/recipient.txt`. | -| `-i ` | `--identity` | Identity private key file. Overrides `~/.gloak/identity.txt`. | - ---- - -## Commands - -### `setup`: install dependencies - -If you don't have `rclone` installed on your system, `gloak setup` will download a standalone, verified binary to `~/.gloak/bin/rclone` so `gloak` can use it without touching your system packages. +Need rclone? Run: ```bash ./gloak setup ``` -| Option | Alias | Description | -|--------|-------|-------------| -| `--dir ` | `-D` | Custom install directory (default: `~/.gloak/bin`). | -| `--update` | `-u` | Force re-download even if `rclone` already exists. | - --- -### `upload`: create an encrypted backup - -Creates a UUID folder, splits the input into fixed-size plaintext chunks, encrypts each chunk independently with age, records SHA-256 checksums in an age-encrypted manifest, and uploads chunks first. `manifest.age` and `locator.json` are uploaded after the chunks. +## What it does -Chunks and manifest are encrypted with the same age recipient. Storage sees filenames like `chunk_00000` and `manifest.age` but cannot read filenames, sizes, or hashes without the identity key. - -```bash -# using auto-resolved keys -./gloak upload ./backup.tar --remote myremote:backup_folder - -# with explicit recipient -./gloak upload ./backup.tar \ - -r /path/to/recipient.txt \ - --remote myremote:backup_folder -``` - -Output: - -```text -Backup ID: 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -Backup completed successfully! -Save this UUID for restoration: 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -``` - -| Option | Alias | Default | Description | -|--------|-------|---------|-------------| -| `--recipient ` | `-r` | `~/.gloak/recipient.txt` | Recipient public key or recipient file. | -| `--remote` | `-R` | _required_ | Target Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | - -The locator is uploaded last. If the upload is interrupted, the backup directory exists but has no `locator.json`. gloak keeps local state under `~/.gloak/uploads/.json`, which lets `resume` skip chunks that already uploaded cleanly. - ---- - -### `resume`: continue an interrupted upload - -Reads local upload state from `~/.gloak/uploads`, scans the source file from the beginning, skips chunks whose plaintext size and SHA-256 still match the saved state, uploads missing chunks, then writes `manifest.age` and `locator.json`. - -```bash -./gloak resume 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 -``` - -Resume requires the original source file at the same path with the same size. gloak does not keep plaintext or encrypted payload spool files locally. It stores only the chunk metadata needed to decide which uploaded chunks can be skipped. +- Encrypts each chunk independently with age. +- Stores backup metadata in an encrypted `manifest.age`. +- Uploads `locator.json` last, so incomplete backups do not look complete. +- Verifies encrypted and plaintext chunk hashes before restore. --- -### `verify`: check backup integrity - -Downloads and decrypts `manifest.age`, checks each encrypted chunk size and SHA-256, decrypts each chunk, checks plaintext size and SHA-256, then compares the full plaintext payload hash with the manifest. - -```bash -./gloak verify 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - -i ~/.gloak/identity.txt -``` - -Output on success: - -```text -VERIFIED! All chunks are healthy and checksums match 100%. -``` - -On failure: - -```text -CHUNK CORRUPT: Hash chunk_00042 mismatch! -``` - -| Option | Alias | Description | -|--------|-------|-------------| -| `--identity ` | `-i` | Identity private key to decrypt `manifest.age` (defaults to `~/.gloak/identity.txt`). | -| `--remote ` | `-R` | Source Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | - ---- - -### `restore`: restore a backup - -Checks each encrypted chunk, decrypts it, checks the plaintext chunk, then writes the restored file chunk by chunk. - -```bash -# with auto-resolved identity -./gloak restore 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - --output-dir ./restored/ - -# with explicit identity -./gloak restore 7f91c6c7-7a0b-44aa-ae23-997b60e4e998 \ - --remote myremote:backup_folder \ - -i ~/.gloak/identity.txt \ - --output-dir ./restored/ -``` - -`--output-dir` specifies the directory where gloak restores the backup. gloak writes the restored file using the original filename from the decrypted manifest after validating that filename is safe. +## Why this exists -| Option | Alias | Description | -|--------|-------|-------------| -| `--identity ` | `-i` | Identity private key to decrypt `manifest.age` (defaults to `~/.gloak/identity.txt`). | -| `--remote ` | `-R` | Source Rclone destination (e.g. `myremote:backup_folder` or `/mnt/backups`). | -| `--output-dir ` | `-o` | Output directory for restored data. | +Most backup tools make at least one tradeoff gloak avoids: -Restore rejects: +1. Encrypt a single blob. If a network failure hits at byte 47 GB of 50, retrying is painful. +2. Split into chunks without enough integrity metadata. You find corruption during restore, without a manifest to guide repair. +3. Store metadata in plaintext. Filenames, sizes, recipient keys, and chunk hashes are visible to anyone who can read the storage. +4. Pull in rclone, restic rest-server, or S3 libraries when you just need local filesystem or a mounted remote. -- Backup IDs that fail UUID v4 validation -- Corrupted or tampered `manifest.age` -- Unsafe filenames in the decrypted manifest (path separators, null bytes, dots only, Windows drive letters) -- Chunks with wrong size or SHA-256 +gloak keeps the design narrow: age encryption, fixed-size chunks, chunk and payload hashes, and an encrypted manifest so the storage backend does not see filenames or hashes. --- -### `cleanup`: remove incomplete backups - -Scans remote storage for backup directories that are missing `manifest.age`, usually from interrupted uploads, and offers to delete them. - -```bash -# interactive prompt before deletion -./gloak cleanup --remote myremote:backup_folder +## Commands -# skip prompt (useful for cron jobs) -./gloak cleanup --remote myremote:backup_folder --yes -``` +| Command | Purpose | +|---------|---------| +| `gloak setup` | Download a standalone rclone binary to `~/.gloak/bin`. | +| `gloak keygen` | Generate `identity.txt` and `recipient.txt`. | +| `gloak upload --remote ` | Encrypt, chunk, and upload a backup. | +| `gloak verify --remote ` | Verify manifest, encrypted chunks, plaintext chunks, and payload hash. | +| `gloak restore --remote ` | Restore a verified backup to an output directory. | -| Option | Alias | Description | -|--------|-------|-------------| -| `--remote ` | `-R` | Rclone remote URI (e.g. `myremote:backup_folder`). | -| `--yes` | `-y` | Skip confirmation prompt. | +For command options, examples, storage layout, and troubleshooting, read [docs/WORKFLOW.md](docs/WORKFLOW.md). --- -## Storage layout +## Storage shape -### filesystem-v1 (current, alpha) +An upload (`gloak upload`) writes one folder per backup: ```text /mnt/backups/ {backup_id}/ - locator.json # public completion marker - manifest.age # age-encrypted manifest - chunk_00000 # encrypted chunk (fixed-size) + locator.json + manifest.age + chunk_00000 chunk_00001 chunk_00002 ``` -`locator.json` is a small public file: - -```json -{ - "version": "1.0", - "uuid": "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" -} -``` - -The locator has no sensitive metadata. It contains no filename, recipient, chunk hashes, or payload hash. Tools can use it to detect backup completion without decrypting anything. - -`manifest.age` is the full backup metadata encrypted with age. Once decrypted: - -| Field | Contains | -|-------|----------| -| `original_name` | Basename of backed-up file | -| `original_size` | Original file size | -| `recipient` | Age recipient | -| `payload_sha256` | SHA-256 of the full plaintext payload | -| `chunks[].index` | Chunk index (0-based) | -| `chunks[].name` | Chunk filename (e.g. `chunk_00042`) | -| `chunks[].plain_size` | Plaintext chunk byte size | -| `chunks[].plain_sha256` | SHA-256 of plaintext chunk content | -| `chunks[].encrypted_size` | Encrypted chunk byte size | -| `chunks[].encrypted_sha256` | SHA-256 of encrypted chunk content | +`locator.json` is public and contains only the format version and backup ID. `manifest.age` is encrypted and contains the original filename, size, creation time, the master key, the payload hash, and per-chunk hashes. -## Architecture +A repository (`gloak init` and `gloak backup`) keeps a content-addressed layout instead: ```text -├── main.go # CLI entrypoint -├── setup.go # rclone downloader command -├── keygen.go # keygen command definition -├── upload.go # upload command definition -├── resume.go # resume command definition -├── verify.go # verify command definition -├── restore.go # restore command definition -├── cleanup.go # cleanup command definition -└── internal/ - ├── core/ - │ ├── chunk.go # fixed-size chunking + hashing - │ └── manifest.go # Manifest types + validation - ├── crypto/ - │ ├── keygen.go # age keypair generation - │ └── stream.go # streaming age encrypt/decrypt wrapper - ├── flow/ - │ ├── upload.go # upload workflow logic - │ ├── resume.go # resume workflow logic - │ ├── verify.go # verification workflow logic - │ └── restore.go # restore workflow logic - └── storage/ - └── rclone.go # Rclone backend wrapper -``` - -### Data flow - -```text -upload: - input file → split into plaintext chunks - → SHA-256 plaintext chunk + full payload - → age encrypt each chunk - → SHA-256 encrypted chunk - → upload chunks → upload manifest.age → upload locator.json - -resume: - upload state → re-read source chunks - → skip chunks whose plaintext hash still matches state - → upload missing chunks → upload manifest.age → upload locator.json - -verify: - locator.json → completion marker - → manifest.age → age decrypt → validate manifest - → check encrypted chunk size/SHA-256 - → decrypt each chunk - → check plaintext chunk size/SHA-256 - → check full payload SHA-256 - -restore: - manifest.age → download encrypted chunk → verify → decrypt chunk → write output +/mnt/backups/ + repo.json + keys/{fingerprint}.age + data/{ab}/{sha256} + snapshots/{uuid}.age + index/000001.age ``` -### Key design decisions - -| Decision | Rationale | -|----------|-----------| -| Recipient-mode age encryption | One identity decrypts. No passphrase to remember or leak. | -| Per-chunk encryption | Each chunk is a standalone age payload, so interrupted uploads can resume without deterministic encryption. | -| Fixed-size chunks | Deterministic boundaries regardless of content. Simplifies verification, resumability, and storage layout. | -| Encrypted manifest | Storage never sees filenames, sizes, or hashes. Only backup UUIDs and chunk object names are visible. | -| Locator uploaded last | If upload is interrupted, no `locator.json` marks the backup complete. | -| Identity required for verify/restore | Manifest is encrypted. You need the identity key to read it. This is a feature, not a bug. | -| SHA-256 at chunk + payload | Corruption is detected before and after decrypting each chunk. | -| Fixed 20MB chunk size | Prevents metadata bloat from tiny chunks and excessive memory usage. | +`repo.json` is the public completion marker with the format version, creation time, recipient keys, and chunk size parameters. Each recipient gets one wrapped master key under `keys/`. Encrypted chunks live under `data/`, keyed by the plaintext hash of each chunk. Snapshot manifests are encrypted under `snapshots/`, and checkpoint files under `index/` record which chunks already exist so a re-run skips them. --- ## Not built yet -Deferred intentionally: - | Feature | Reasoning | |---------|-----------| | Named storage profiles | `--remote my-backups` instead of typing URIs. | diff --git a/backup.go b/backup.go new file mode 100644 index 0000000..b089b98 --- /dev/null +++ b/backup.go @@ -0,0 +1,61 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/alecthomas/kong" + + "gloak/internal/flow" +) + +// BackupCmd backs up a file or directory into a gloak repository. +type BackupCmd struct { + ExistingFile string `arg:"" name:"path" help:"File or directory to back up"` + Identity string `short:"i" help:"Path to identity file (defaults to $GLOAK_DIR/identity.txt, or ~/.gloak/identity.txt)."` + Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak"` +} + +// Run backs up the selected path, using the default identity when none is +// specified, and sends an optional completion notification. Identity path +// expansion and backup errors are returned; notification failures are ignored. +func (c *BackupCmd) Run(ctx *kong.Context) error { + identityPath := c.Identity + if identityPath == "" { + identityPath = filepath.Join(getDefaultDir(), "identity.txt") + } + identityPath, err := expandHome(identityPath) + if err != nil { + return err + } + start := time.Now() + snapshotID, err := flow.RunBackup(c.ExistingFile, identityPath, c.Remote) + if err != nil { + return fmt.Errorf("backup failed: %w", err) + } + notifyCommand("backup", + "Path: "+c.ExistingFile, + "Snapshot: "+snapshotID, + "Duration: "+formatDuration(time.Since(start)), + ) + return nil +} + +// expandHome resolves ~ or a leading ~/ to the user's home directory, +// returning a home lookup error if expansion fails. Other paths are unchanged. +func expandHome(path string) (string, error) { + if path != "~" && !strings.HasPrefix(path, "~/") { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("failed to resolve home directory: %w", err) + } + if path == "~" { + return home, nil + } + return filepath.Join(home, path[2:]), nil +} diff --git a/check.go b/check.go new file mode 100644 index 0000000..0fc158c --- /dev/null +++ b/check.go @@ -0,0 +1,37 @@ +package main + +import ( + "path/filepath" + "time" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type CheckCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` +} + +// Run checks repository structure using the selected or default identity and +// sends an optional completion notification. Check errors are returned. +func (c *CheckCmd) Run() error { + pterm.Info.Println("Starting CHECK process...") + + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + pterm.Info.Printf("Using identity from: %s\n", ident) + } + + start := time.Now() + if err := flow.RunCheck(ident, c.Remote); err != nil { + return err + } + notifyCommand("check", + "Remote: "+c.Remote, + "Duration: "+formatDuration(time.Since(start)), + ) + return nil +} diff --git a/cleanup.go b/cleanup.go deleted file mode 100644 index 1cdeff0..0000000 --- a/cleanup.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "sort" - - "github.com/pterm/pterm" - - "gloak/internal/storage" -) - -type CleanupCmd struct { - Remote string `help:"Rclone remote URI (e.g., keraaaki:backup_folder)." required:"" short:"R"` - Yes bool `help:"Skip confirmation prompt." short:"y"` -} - -type lsjsonEntry struct { - Path string `json:"Path"` - Name string `json:"Name"` - IsDir bool `json:"IsDir"` - Size int64 `json:"Size"` - ModTime string `json:"ModTime"` -} - -func (c *CleanupCmd) Run() error { - pterm.Info.Println("Scanning remote for backup directories...") - - backend := storage.NewRcloneBackend(c.Remote) - - raw, err := backend.ListFiles(nil) - if err != nil { - return fmt.Errorf("failed to list remote: %w", err) - } - - var entries []lsjsonEntry - if err := json.Unmarshal(raw, &entries); err != nil { - return fmt.Errorf("failed to parse lsjson output: %w", err) - } - - // Collect top-level directories (potential backup folders) - var dirs []lsjsonEntry - for _, e := range entries { - if e.IsDir { - dirs = append(dirs, e) - } - } - - if len(dirs) == 0 { - pterm.Success.Println("No backup directories found. Nothing to clean up.") - return nil - } - - pterm.Info.Printf("Found %d backup directories. Checking for manifest.age...\n", len(dirs)) - - // Identify orphan directories (missing manifest.age) - var orphans []string - for _, d := range dirs { - filesInDir, err := backend.ListFiles([]string{d.Path}) - if err != nil { - pterm.Warning.Printf(" Could not list %s: %v — skipping\n", d.Path, err) - continue - } - - var innerEntries []lsjsonEntry - if err := json.Unmarshal(filesInDir, &innerEntries); err != nil { - pterm.Warning.Printf(" Could not parse listing for %s: %v — skipping\n", d.Path, err) - continue - } - - hasManifest := false - for _, ie := range innerEntries { - if !ie.IsDir && ie.Name == "manifest.age" { - hasManifest = true - break - } - } - - if !hasManifest { - orphans = append(orphans, d.Path) - } - } - - if len(orphans) == 0 { - pterm.Success.Println("All backup directories have a manifest.age. Nothing to clean up.") - return nil - } - - sort.Strings(orphans) - - pterm.Warning.Printf("Found %d orphan directorie(s) missing manifest.age:\n", len(orphans)) - for _, o := range orphans { - pterm.DefaultBasicText.Printf(" - %s\n", o) - } - - if !c.Yes { - fmt.Println() - result, _ := pterm.DefaultInteractiveConfirm. - WithDefaultText("Delete these directories?"). - Show() - if !result { - pterm.Info.Println("Aborted. No directories deleted.") - return nil - } - } - - return batchDelete(backend, orphans) -} - -func batchDelete(backend storage.Backend, orphans []string) error { - // Batch delete in chunks of up to 500 - const batchSize = 500 - deleted := 0 - var deleteErrors []string - - for i := 0; i < len(orphans); i += batchSize { - end := i + batchSize - if end > len(orphans) { - end = len(orphans) - } - batch := orphans[i:end] - - for _, path := range batch { - if err := backend.Delete(path); err != nil { - deleteErrors = append(deleteErrors, fmt.Sprintf("%s: %v", path, err)) - continue - } - deleted++ - pterm.Success.Printf(" Deleted: %s\n", path) - } - } - - fmt.Println() - pterm.Success.Printf("Cleanup complete: %d of %d orphan directorie(s) deleted.\n", deleted, len(orphans)) - - if len(deleteErrors) > 0 { - pterm.Warning.Printf("%d deletion(s) failed:\n", len(deleteErrors)) - for _, de := range deleteErrors { - pterm.Error.Printf(" %s\n", de) - } - return fmt.Errorf("%d deletion(s) failed", len(deleteErrors)) - } - - return nil -} diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md new file mode 100644 index 0000000..96ca1e0 --- /dev/null +++ b/docs/WORKFLOW.md @@ -0,0 +1,158 @@ +# Everyday gloak workflows + +This guide walks through the tasks most people actually do with a backup tool. Commands assume `gloak` is in your PATH. Every command that talks to storage needs `-R` (where backups live). The identity key is read from `$GLOAK_DIR/identity.txt` (default `~/.gloak/identity.txt`), or pass `-i` to use a different one. + +> [!WARNING] +> gloak is alpha. Keep another copy of anything critical somewhere else. + +## One-time setup + +Before the first backup, generate a key pair and create a repository. + +```bash +gloak keygen +gloak init -R myremote:backups +``` + +`keygen` writes two files into the gloak directory (`~/.gloak` unless `GLOAK_DIR` is set): + +- `identity.txt` is your private key. Treat it like a password. Anyone who has it can read every backup. +- `recipient.txt` is your public key. Safe to share; it only lets someone encrypt to you. + +`init` creates the storage layout on your remote. You only run it once per remote location. + +By default keys live under `~/.gloak`. Set `GLOAK_DIR` before you run `keygen` to keep them somewhere else; a leading `~` in `GLOAK_DIR` is expanded: + +```bash +export GLOAK_DIR=~/backups/gloak-keys +gloak keygen +gloak init -R myremote:backups +``` + +Prefer SSH keys for recipients? Pass the public key line directly: + +```bash +gloak init -R myremote:backups -r "ssh-ed25519 AAAA... you@laptop" +``` + +## Back up a folder + +Point `backup` at a file or a directory. Directories are walked recursively; empty folders are kept; symlinks are skipped. + +```bash +gloak backup ~/Documents -R myremote:backups +``` + +The command prints a snapshot ID when it finishes. Save it. That ID is how you list, verify, or restore this backup later. + +Backup of the same data twice uploads almost nothing the second time. Chunks that already exist on the remote are skipped automatically. + +## See what you have backed up + +```bash +gloak snapshots -R myremote:backups +``` + +You get a table like: + +```text +UUID KIND CREATED HOST DETAIL +aaaaaaaa-bbbb-4ccc-8ddd-000000000000 tree 2026-10-12T10:00:00Z laptop 12 files from Documents +``` + +`KIND` is `tree` for folders and `file` for a single-file backup. + +## Check a backup before you trust it + +Two levels: + +```bash +# quick structural check: every snapshot opens, every chunk is present +gloak check -R myremote:backups + +# deep check of one snapshot: downloads and decrypts every chunk +gloak verify -R myremote:backups +``` + +Run `check` occasionally. Run `verify` after a big backup, or before you delete the original files. + +## Restore + +Full restore of a folder snapshot: + +```bash +gloak restore -R myremote:backups -o ./restored +``` + +Files land under `./restored//`. To pull only one subfolder: + +```bash +gloak restore -R myremote:backups -o ./restored --path-filter subfolder +``` + +Single-file snapshots restore as one file under the output directory. + +## Clean up old backups + +Snapshots pile up. `prune` removes old ones and deletes chunks nothing needs anymore. + +```bash +# dry run: show what would be deleted, change nothing +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --keep-weekly 4 + +# actually delete +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --keep-weekly 4 --yes +``` + +How the keep flags work: + +- `--keep-last N` keeps the N newest snapshots. +- `--keep-daily N` keeps the newest snapshot of each of the last N days that have one. +- `--keep-weekly N` and `--keep-monthly N` work the same way for weeks and months. + +At least one keep flag is required. Without `--yes`, prune only prints the plan. + +## Optional: notify on Telegram + +gloak sends a short status report after backup, restore, verify, check, or prune. Put the credentials in `~/.gloak/config.toml` (or `$GLOAK_DIR/config.toml`): + +```toml +[telegram] +token = "123456:ABC..." +chat_id = "your-chat-id" +``` + +Environment variables override the file when set, which is handy for containers or one-off runs: + +```bash +export GLOAK_TELEGRAM_TOKEN="123456:ABC..." +export GLOAK_TELEGRAM_CHAT_ID="your-chat-id" +``` + +Example message after a backup: + +```text +gloak backup completed +Path: /home/you/Documents +Snapshot: 550e8400-e29b-41d4-a716-446655440000 +Duration: 42s +``` + +A notify failure only prints a warning. It never fails the backup itself. The token is never printed. + +## If something goes wrong + +- Wrong or missing identity: `identity cannot open this repository`. Confirm you are using the key that matches a recipient listed at init time. +- Snapshot not found: check the ID with `gloak snapshots`. +- Corruption during restore or verify: the error starts with `CHUNK CORRUPT` and names the bad chunk. gloak cannot repair a damaged object in place, and re-running backup will not replace it, because backup skips chunks that are already in the index. Restore from another copy of the data instead, and do not prune while `check` or `verify` still fails. +- Storage errors: confirm `rclone` is installed and can reach your remote. `gloak setup` installs rclone into `~/.gloak/bin/` if you need it. + +## Typical weekly routine + +```bash +gloak backup ~/Documents -R myremote:backups +gloak check -R myremote:backups +gloak prune -R myremote:backups --keep-last 7 --keep-daily 7 --yes +``` + +That is backup, sanity check, and cleanup in three commands. diff --git a/go.mod b/go.mod index 6555264..d02494a 100644 --- a/go.mod +++ b/go.mod @@ -4,15 +4,19 @@ go 1.26.3 require ( filippo.io/age v1.3.1 + github.com/BurntSushi/toml v1.6.0 github.com/alecthomas/kong v1.15.0 github.com/google/uuid v1.6.0 + github.com/klauspost/compress v1.20.1 github.com/pterm/pterm v0.12.83 + golang.org/x/crypto v0.58.0 ) require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect + filippo.io/edwards25519 v1.1.0 // indirect filippo.io/hpke v0.4.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/containerd/console v1.0.5 // indirect @@ -20,8 +24,7 @@ require ( github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/term v0.40.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect ) diff --git a/go.sum b/go.sum index 00c5311..3343a9e 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,12 @@ c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd h1:ZLsPO6WdZ5zatV4UfVpr7oAw c2sp.org/CCTV/age v0.0.0-20251208015420-e9274a7bdbfd/go.mod h1:SrHC2C7r5GkDk8R+NFVzYy/sdj0Ypg9htaPXQq5Cqeo= filippo.io/age v1.3.1 h1:hbzdQOJkuaMEpRCLSN1/C5DX74RPcNCk6oqhKMXmZi0= filippo.io/age v1.3.1/go.mod h1:EZorDTYUxt836i3zdori5IJX/v2Lj6kWFU0cfh6C0D4= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/hpke v0.4.0 h1:p575VVQ6ted4pL+it6M00V/f2qTZITO0zgmdKCkd5+A= filippo.io/hpke v0.4.0/go.mod h1:EmAN849/P3qdeK+PCMkDpDm83vRHM5cDipBJ8xbQLVY= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= github.com/MarvinJWendt/testza v0.2.1/go.mod h1:God7bhG8n6uQxwdScay+gjm9/LnO4D3kkcZX4hv9Rp8= github.com/MarvinJWendt/testza v0.2.8/go.mod h1:nwIcjmr0Zz+Rcwfh3/4UhBp7ePKVhuBExvZqnKYWlII= @@ -46,6 +50,8 @@ github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -82,8 +88,8 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -106,22 +112,22 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/init.go b/init.go new file mode 100644 index 0000000..19099df --- /dev/null +++ b/init.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "gloak/internal/flow" +) + +// InitCmd creates a new gloak repository. +type InitCmd struct { + Remote string `short:"R" required:"" help:"Repository remote, e.g. b2:my-bucket/gloak or local path"` + Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` + Force bool `short:"f" help:"Overwrite an existing repository (makes old snapshots unreadable)"` + ChunkMin int64 `help:"Minimum CDC chunk size in bytes (default 524288)."` + ChunkAvg int64 `help:"Average CDC chunk size in bytes (default 1048576)."` + ChunkMax int64 `help:"Maximum CDC chunk size in bytes (default 8388608)."` + PQ bool `help:"Reserve post-quantum hybrid recipients (not implemented yet)."` +} + +// Run initializes the repository with the resolved recipients and chunk bounds. +// It rejects PQ mode and returns recipient resolution or initialization errors. +func (c *InitCmd) Run() error { + if c.PQ { + return fmt.Errorf("--pq hybrid recipients are not implemented yet (reserved for a later phase)") + } + recipients, err := resolveInitRecipients(c.Recipient) + if err != nil { + return err + } + return flow.RunInit(recipients, c.Remote, c.ChunkMin, c.ChunkAvg, c.ChunkMax, c.Force) +} + +// resolveInitRecipients turns the --recipient value into a key list. An empty +// value uses recipient.txt in getDefaultDir(). A value that looks like keys +// (starts with age1, ssh-ed25519, ssh-rsa, or contains commas) is split on +// commas; anything else is treated as a file path with one key per line, +// skipping blanks and # comments. It returns file read errors and rejects +// empty lists; key validity is checked by callers. +func resolveInitRecipients(path string) ([]string, error) { + if path == "" { + path = filepath.Join(getDefaultDir(), "recipient.txt") + } + if looksLikeInlineKeys(path) { + var out []string + // SSH keys contain spaces; split only on commas. + for _, part := range strings.Split(path, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no recipients found (run 'gloak keygen' first)") + } + return out, nil + } + return readRecipientFile(path) +} + +// looksLikeInlineKeys reports whether s has a supported key prefix or a comma, +// so recipient resolution treats it as inline keys rather than a file path. +// It does not validate the keys. +func looksLikeInlineKeys(s string) bool { + return strings.HasPrefix(s, "age1") || + strings.HasPrefix(s, "ssh-ed25519 ") || + strings.HasPrefix(s, "ssh-rsa ") || + strings.Contains(s, ",") +} + +// readRecipientFile returns trimmed, nonempty lines other than # comments. +// It returns an error if the file cannot be read or contains no such lines; +// the returned keys are not validated. +func readRecipientFile(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read recipients from %s: %w (run 'gloak keygen' first)", path, err) + } + var out []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out = append(out, line) + } + if len(out) == 0 { + return nil, fmt.Errorf("no recipients found in %s (run 'gloak keygen' first)", path) + } + return out, nil +} diff --git a/init_pq_test.go b/init_pq_test.go new file mode 100644 index 0000000..4d4a4eb --- /dev/null +++ b/init_pq_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "strings" + "testing" +) + +func TestInitPQFlagNotImplemented(t *testing.T) { + cmd := InitCmd{Remote: "mem:", Recipient: "age1test", PQ: true} + err := cmd.Run() + if err == nil { + t.Fatal("InitCmd with --pq succeeded") + } + if !strings.Contains(err.Error(), "not implemented") { + t.Fatalf("error = %v", err) + } +} + +func TestResolveInitRecipientsAcceptsSSHEd25519(t *testing.T) { + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl test@host" + got, err := resolveInitRecipients(key) + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + if len(got) != 1 || got[0] != key { + t.Fatalf("got = %v", got) + } +} diff --git a/internal/chunker/chunker.go b/internal/chunker/chunker.go new file mode 100644 index 0000000..3531b77 --- /dev/null +++ b/internal/chunker/chunker.go @@ -0,0 +1,88 @@ +// Package chunker implements content-defined chunking with a gear hash. +package chunker + +import ( + "bufio" + "fmt" + "io" +) + +const windowSize = 64 + +// gear is filled once at init from a fixed seed so every binary builds the +// same table and CDC boundaries stay stable across runs and machines. +var gear [256]uint64 + +// init fills the gear table deterministically so chunk boundaries are stable +// across runs. +func init() { + x := uint64(0x9E3779B97F4A7C15) + for i := range gear { + x += 0x9E3779B97F4A7C15 + z := x + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9 + z = (z ^ (z >> 27)) * 0x94D049BB133111EB + gear[i] = z ^ (z >> 31) + } +} + +type CDC struct { + min int64 + avg int64 + max int64 +} + +// NewCDC creates a chunker with byte bounds min <= avg <= max, all positive, +// and rounds max up to a multiple of windowSize. avg controls the boundary +// frequency, not a guaranteed mean chunk size. Invalid bounds return an error. +func NewCDC(min, avg, max int64) (*CDC, error) { + if min <= 0 || avg <= 0 || max <= 0 { + return nil, fmt.Errorf("chunk size must be positive (min=%d avg=%d max=%d)", min, avg, max) + } + if max < avg || avg < min { + return nil, fmt.Errorf("chunk bounds require max >= avg >= min (min=%d avg=%d max=%d)", min, avg, max) + } + if max%windowSize != 0 { + max = ((max + windowSize - 1) / windowSize) * windowSize + } + return &CDC{min: min, avg: avg, max: max}, nil +} + +// Split reads r and calls fn once per chunk. Each chunk is a fresh slice the +// caller may retain. An empty reader produces no calls; the final chunk may +// be smaller than min. It stops on and returns reader or callback errors; +// on a non-EOF read error, the buffered partial chunk is not delivered. +func (c *CDC) Split(r io.Reader, fn func(chunk []byte) error) error { + br := bufio.NewReaderSize(r, 64*1024) + chunk := make([]byte, 0, c.max) + var h uint64 + for { + b, err := br.ReadByte() + if err == io.EOF { + if len(chunk) > 0 { + return fn(chunk) + } + return nil + } + if err != nil { + return err + } + chunk = append(chunk, b) + h = (h << 1) + gear[b] + if int64(len(chunk)) >= c.max { + if err := fn(chunk); err != nil { + return err + } + chunk = make([]byte, 0, c.max) + h = 0 + continue + } + if int64(len(chunk)) >= c.min && h%uint64(c.avg) == 0 { + if err := fn(chunk); err != nil { + return err + } + chunk = make([]byte, 0, c.max) + h = 0 + } + } +} diff --git a/internal/chunker/chunker_test.go b/internal/chunker/chunker_test.go new file mode 100644 index 0000000..cd4fb60 --- /dev/null +++ b/internal/chunker/chunker_test.go @@ -0,0 +1,135 @@ +package chunker + +import ( + "bytes" + "io" + "math/rand" + "testing" +) + +func mustCDC(t *testing.T, min, avg, max int64) *CDC { + t.Helper() + c, err := NewCDC(min, avg, max) + if err != nil { + t.Fatalf("NewCDC(%d,%d,%d) error: %v", min, avg, max, err) + } + return c +} + +func splitAll(t *testing.T, c *CDC, data []byte) [][]byte { + t.Helper() + var chunks [][]byte + err := c.Split(bytes.NewReader(data), func(chunk []byte) error { + cp := make([]byte, len(chunk)) + copy(cp, chunk) + chunks = append(chunks, cp) + return nil + }) + if err != nil { + t.Fatalf("Split() error: %v", err) + } + return chunks +} + +func TestNewCDCRejectsBadBounds(t *testing.T) { + cases := []struct{ min, avg, max int64 }{ + {0, 1, 2}, + {2, 1, 4}, + {2, 4, 3}, + {2, 2, 1}, + } + for _, c := range cases { + if _, err := NewCDC(c.min, c.avg, c.max); err == nil { + t.Fatalf("NewCDC(%d,%d,%d) succeeded", c.min, c.avg, c.max) + } + } +} + +func TestSplitEmptyReader(t *testing.T) { + c := mustCDC(t, 4, 8, 16) + chunks := splitAll(t, c, nil) + if len(chunks) != 0 { + t.Fatalf("chunks = %d, want 0", len(chunks)) + } +} + +func TestSplitSmallerThanMinIsOneChunk(t *testing.T) { + c := mustCDC(t, 64, 128, 256) + data := []byte("short") + chunks := splitAll(t, c, data) + if len(chunks) != 1 || !bytes.Equal(chunks[0], data) { + t.Fatalf("chunks = %d, want 1 equal to input", len(chunks)) + } +} + +func TestSplitRespectsMax(t *testing.T) { + c := mustCDC(t, 4, 64, 64) + data := bytes.Repeat([]byte{0xFF}, 200) + chunks := splitAll(t, c, data) + for i, ch := range chunks { + if int64(len(ch)) > 64 { + t.Fatalf("chunk %d length %d > max 64", i, len(ch)) + } + } + var total int + for _, ch := range chunks { + total += len(ch) + } + if total != len(data) { + t.Fatalf("total %d != %d", total, len(data)) + } +} + +func TestSplitDeterminism(t *testing.T) { + data := make([]byte, 50_000) + rng := rand.New(rand.NewSource(42)) + rng.Read(data) + a := splitAll(t, mustCDC(t, 512, 2048, 8192), data) + b := splitAll(t, mustCDC(t, 512, 2048, 8192), data) + if len(a) != len(b) { + t.Fatalf("chunk counts %d vs %d", len(a), len(b)) + } + for i := range a { + if !bytes.Equal(a[i], b[i]) { + t.Fatalf("chunk %d differs", i) + } + } + if len(a) < 2 { + t.Fatalf("expected multiple chunks, got %d", len(a)) + } +} + +func TestSplitRoundTripBytes(t *testing.T) { + data := make([]byte, 100_000) + rng := rand.New(rand.NewSource(7)) + rng.Read(data) + chunks := splitAll(t, mustCDC(t, 1024, 4096, 16384), data) + var out []byte + for _, ch := range chunks { + out = append(out, ch...) + } + if !bytes.Equal(out, data) { + t.Fatal("concatenated chunks != input") + } +} + +func TestSplitCallbackErrorPropagates(t *testing.T) { + c := mustCDC(t, 4, 8, 16) + sentinel := io.ErrClosedPipe + err := c.Split(bytes.NewReader([]byte("abcdefghij")), func([]byte) error { + return sentinel + }) + if err != sentinel { + t.Fatalf("err = %v, want sentinel", err) + } +} + +func TestMaxRoundedUpToWindowMultiple(t *testing.T) { + c := mustCDC(t, 4, 8, 100) + if c.max%64 != 0 { + t.Fatalf("max = %d, want multiple of 64", c.max) + } + if c.max < 100 { + t.Fatalf("max = %d, want >= 100", c.max) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..040ba85 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,48 @@ +// Package config loads the optional user configuration file. +// +// The file lives at /config.toml (GLOAK_DIR or ~/.gloak). A +// missing file is not an error; every value stays empty and callers fall +// back to their defaults. +package config + +import ( + "errors" + "os" + "path/filepath" + + "github.com/BurntSushi/toml" +) + +// Telegram holds Bot API credentials for optional notifications. +type Telegram struct { + Token string `toml:"token"` + ChatID string `toml:"chat_id"` +} + +// Config is the user configuration file. +type Config struct { + Telegram Telegram `toml:"telegram"` +} + +// Path returns the config file location for the given gloak directory. +func Path(dir string) string { + return filepath.Join(dir, "config.toml") +} + +// Load reads path and parses it. A missing file yields an empty Config and +// a nil error. Other read errors and TOML parsing errors are returned with an +// empty Config. +func Load(path string) (Config, error) { + var cfg Config + data, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return cfg, nil + } + return cfg, err + } + if err := toml.Unmarshal(data, &cfg); err != nil { + return Config{}, err + } + return cfg, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..dcbec66 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadMissingFileIsEmpty(t *testing.T) { + cfg, err := Load(filepath.Join(t.TempDir(), "config.toml")) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Telegram.Token != "" || cfg.Telegram.ChatID != "" { + t.Fatalf("cfg = %+v, want zero value", cfg) + } +} + +func TestLoadReadsTelegramSection(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + content := "# comment\n[telegram]\ntoken = \"abc\"\nchat_id = \"42\"\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if cfg.Telegram.Token != "abc" || cfg.Telegram.ChatID != "42" { + t.Fatalf("cfg = %+v", cfg) + } +} + +func TestLoadMalformedFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte("[telegram\ntoken="), 0600); err != nil { + t.Fatal(err) + } + + if _, err := Load(path); err == nil { + t.Fatal("Load() succeeded on malformed file") + } +} + +func TestPathUsesDir(t *testing.T) { + if got := Path("/x/gloak"); got != "/x/gloak/config.toml" { + t.Fatalf("Path() = %q", got) + } +} diff --git a/internal/core/manifest.go b/internal/core/manifest.go index 5c4c991..83e99a8 100644 --- a/internal/core/manifest.go +++ b/internal/core/manifest.go @@ -1,14 +1,12 @@ package core import ( - "bytes" - "encoding/json" - "fmt" - "time" - "github.com/google/uuid" +) - "gloak/internal/crypto" +const ( + KindFile = "file" + KindTree = "tree" ) type Manifest struct { @@ -22,57 +20,36 @@ type Manifest struct { Chunks []ChunkMeta `json:"chunks"` } -type Locator struct { - Version string `json:"version"` - UUID string `json:"uuid"` -} - -func GenerateUUID() string { - return uuid.New().String() -} - -func CreateManifest(id string, filename string, size int64, recipient string, payloadHash string, chunks []ChunkMeta) Manifest { - return Manifest{ - Version: "1.0", - UUID: id, - OriginalName: filename, - OriginalSize: size, - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Recipient: recipient, - PayloadSHA256: payloadHash, - Chunks: chunks, - } +type TreeEntry struct { + RelPath string `json:"rel_path"` + Type string `json:"type"` + Mode uint32 `json:"mode"` + Mtime string `json:"mtime"` + Size int64 `json:"size"` + Chunks []ChunkMeta `json:"chunks"` } -func CreateLocator(id string) Locator { - return Locator{ - Version: "1.0", - UUID: id, - } +type TreeManifest struct { + Version string `json:"version"` + UUID string `json:"uuid"` + Kind string `json:"kind"` + CreatedAt string `json:"created_at"` + Hostname string `json:"hostname"` + Paths []string `json:"paths"` + Entries []TreeEntry `json:"entries"` } -func EncryptManifest(pubKey string, manifest Manifest) ([]byte, error) { - jsonData, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal manifest json: %w", err) +// FileCount counts entries whose type is "file", including empty files. +func (m *TreeManifest) FileCount() int { + n := 0 + for _, e := range m.Entries { + if e.Type == "file" { + n++ + } } - - var buf bytes.Buffer - - encWriter, err := crypto.EncryptWriter(pubKey, &buf) - if err != nil { - return nil, fmt.Errorf("failed to prepare age writer: %w", err) - } - - _, err = encWriter.Write(jsonData) - if err != nil { - return nil, fmt.Errorf("failed to write JSON to encryptor: %w", err) - } - encWriter.Close() - - return buf.Bytes(), nil + return n } -func SerializeLocator(locator Locator) ([]byte, error) { - return json.MarshalIndent(locator, "", " ") +func GenerateUUID() string { + return uuid.New().String() } diff --git a/internal/core/manifest_test.go b/internal/core/manifest_test.go index 7fc53de..39b4c86 100644 --- a/internal/core/manifest_test.go +++ b/internal/core/manifest_test.go @@ -1,84 +1,14 @@ package core -import ( - "bytes" - "encoding/json" - "io" - "os" - "path/filepath" - "testing" - - appcrypto "gloak/internal/crypto" -) - -func TestCreateManifestAndLocatorVersions(t *testing.T) { - chunks := []ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 10, PlainSHA256: "plain", EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - manifest := CreateManifest("backup-id", "secret.tar", 123, "age1example", "payload-hash", chunks) - - if manifest.Version != "1.0" { - t.Fatalf("manifest version = %q, want 1.0", manifest.Version) - } - if manifest.UUID != "backup-id" || manifest.OriginalName != "secret.tar" || manifest.OriginalSize != 123 { - t.Fatalf("manifest fields not preserved: %+v", manifest) - } - if len(manifest.Chunks) != 1 || manifest.Chunks[0] != chunks[0] { - t.Fatalf("manifest chunks = %+v, want %+v", manifest.Chunks, chunks) - } - - locator := CreateLocator("backup-id") - if locator.Version != "1.0" || locator.UUID != "backup-id" { - t.Fatalf("locator = %+v, want version 1.0 and uuid backup-id", locator) - } -} - -func TestSerializeLocator(t *testing.T) { - data, err := SerializeLocator(CreateLocator("backup-id")) - if err != nil { - t.Fatalf("SerializeLocator() returned error: %v", err) - } - - var locator Locator - if err := json.Unmarshal(data, &locator); err != nil { - t.Fatalf("serialized locator is not valid JSON: %v", err) - } - if locator.Version != "1.0" || locator.UUID != "backup-id" { - t.Fatalf("locator = %+v, want version 1.0 and uuid backup-id", locator) - } -} - -func TestEncryptManifestCanBeDecrypted(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - identityPath := filepath.Join(t.TempDir(), "identity.txt") - if err := os.WriteFile(identityPath, []byte(identityFile), 0600); err != nil { - t.Fatalf("failed to write identity file: %v", err) - } - - manifest := CreateManifest("backup-id", "secret.tar", 123, pubKey, "payload-hash", nil) - encrypted, err := EncryptManifest(pubKey, manifest) - if err != nil { - t.Fatalf("EncryptManifest() returned error: %v", err) - } - if bytes.Contains(encrypted, []byte("secret.tar")) { - t.Fatalf("encrypted manifest contains plaintext filename") - } - - r, err := appcrypto.DecryptReader(identityPath, bytes.NewReader(encrypted)) - if err != nil { - t.Fatalf("DecryptReader() returned error: %v", err) - } - plain, err := io.ReadAll(r) - if err != nil { - t.Fatalf("failed to read decrypted manifest: %v", err) - } - - var got Manifest - if err := json.Unmarshal(plain, &got); err != nil { - t.Fatalf("decrypted manifest is not valid JSON: %v", err) - } - if got.UUID != manifest.UUID || got.OriginalName != manifest.OriginalName || got.PayloadSHA256 != manifest.PayloadSHA256 { - t.Fatalf("decrypted manifest = %+v, want %+v", got, manifest) +import "testing" + +func TestFileCount(t *testing.T) { + m := TreeManifest{Entries: []TreeEntry{ + {Type: "dir"}, + {Type: "file"}, + {Type: "file"}, + }} + if got := m.FileCount(); got != 2 { + t.Fatalf("FileCount() = %d, want 2", got) } } diff --git a/internal/core/path.go b/internal/core/path.go new file mode 100644 index 0000000..ba7e309 --- /dev/null +++ b/internal/core/path.go @@ -0,0 +1,33 @@ +package core + +import ( + "fmt" + "strings" +) + +// ValidateRelPath checks a slash-separated tree entry path without accessing +// the filesystem. It rejects empty paths, null bytes, backslashes, leading +// slashes, drive prefixes, and empty, . or .. segments. +func ValidateRelPath(rel string) error { + if rel == "" { + return fmt.Errorf("empty path") + } + if strings.ContainsRune(rel, 0) { + return fmt.Errorf("path contains null byte") + } + if strings.Contains(rel, `\`) { + return fmt.Errorf("path contains backslash: %s", rel) + } + if strings.HasPrefix(rel, "/") { + return fmt.Errorf("absolute path: %s", rel) + } + if len(rel) >= 2 && rel[1] == ':' { + return fmt.Errorf("drive letter path: %s", rel) + } + for _, seg := range strings.Split(rel, "/") { + if seg == "" || seg == "." || seg == ".." { + return fmt.Errorf("unsafe path segment in %s", rel) + } + } + return nil +} diff --git a/internal/core/path_test.go b/internal/core/path_test.go new file mode 100644 index 0000000..7924523 --- /dev/null +++ b/internal/core/path_test.go @@ -0,0 +1,19 @@ +package core + +import "testing" + +func TestValidateRelPathAcceptsNormal(t *testing.T) { + for _, p := range []string{"a", "a/b", "a/b/c.txt", "dir/empty"} { + if err := ValidateRelPath(p); err != nil { + t.Fatalf("ValidateRelPath(%q) error: %v", p, err) + } + } +} + +func TestValidateRelPathRejectsUnsafe(t *testing.T) { + for _, p := range []string{"", "/abs", "a//b", "a/../b", "..", ".", "a/.", `a\b`, "a\x00b", "C:/x", "c:\\x"} { + if err := ValidateRelPath(p); err == nil { + t.Fatalf("ValidateRelPath(%q) succeeded", p) + } + } +} diff --git a/internal/crypto/envelope.go b/internal/crypto/envelope.go new file mode 100644 index 0000000..44b471d --- /dev/null +++ b/internal/crypto/envelope.go @@ -0,0 +1,103 @@ +package crypto + +import ( + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/klauspost/compress/zstd" + "golang.org/x/crypto/chacha20poly1305" +) + +const chunkKeyInfo = "gloak-chunk-v1" + +// deriveChunkKey derives a 32-byte chunk key from masterKey using the decoded +// plaintext SHA-256 hash as salt. It rejects malformed hex or hashes that do +// not decode to 32 bytes and returns key derivation errors. +func deriveChunkKey(masterKey []byte, plainSHA256Hex string) ([]byte, error) { + salt, err := hex.DecodeString(plainSHA256Hex) + if err != nil || len(salt) != sha256.Size { + return nil, fmt.Errorf("invalid chunk hash: %q", plainSHA256Hex) + } + return hkdf.Key(sha256.New, masterKey, salt, chunkKeyInfo, chacha20poly1305.KeySize) +} + +// SealChunk compresses plain with zstd, then encrypts it with XChaCha20-Poly1305 +// under a key derived from masterKey. Output layout: 24-byte nonce || ciphertext. +// plainSHA256Hex must encode 32 bytes; its exact text is authenticated, but its +// value is not checked against plain. Hash decoding, key derivation, compression, +// and encryption setup errors are returned. +func SealChunk(masterKey []byte, plainSHA256Hex string, plain []byte) ([]byte, error) { + key, err := deriveChunkKey(masterKey, plainSHA256Hex) + if err != nil { + return nil, err + } + compressed, err := zstdEncode(plain) + if err != nil { + return nil, fmt.Errorf("failed to compress chunk: %w", err) + } + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, fmt.Errorf("failed to init cipher: %w", err) + } + nonce := make([]byte, chacha20poly1305.NonceSizeX) + if _, err := rand.Read(nonce); err != nil { + return nil, fmt.Errorf("failed to generate nonce: %w", err) + } + sealed := make([]byte, 0, len(nonce)+len(compressed)+aead.Overhead()) + sealed = append(sealed, nonce...) + sealed = aead.Seal(sealed, nonce, compressed, []byte(plainSHA256Hex)) + return sealed, nil +} + +// OpenChunk authenticates and decrypts a SealChunk envelope, then decompresses +// it. masterKey and the exact plainSHA256Hex string must match those used to +// seal it. It returns errors for malformed hashes, short envelopes, key or +// cipher setup failures, authentication failures, and invalid compressed data. +// Callers must check the returned plaintext's hash themselves. +func OpenChunk(masterKey []byte, plainSHA256Hex string, sealed []byte) ([]byte, error) { + key, err := deriveChunkKey(masterKey, plainSHA256Hex) + if err != nil { + return nil, err + } + aead, err := chacha20poly1305.NewX(key) + if err != nil { + return nil, fmt.Errorf("failed to init cipher: %w", err) + } + if len(sealed) < chacha20poly1305.NonceSizeX+aead.Overhead() { + return nil, fmt.Errorf("chunk too short: %d bytes", len(sealed)) + } + nonce := sealed[:chacha20poly1305.NonceSizeX] + ciphertext := sealed[chacha20poly1305.NonceSizeX:] + compressed, err := aead.Open(nil, nonce, ciphertext, []byte(plainSHA256Hex)) + if err != nil { + return nil, fmt.Errorf("chunk authentication failed: %w", err) + } + plain, err := zstdDecode(compressed) + if err != nil { + return nil, fmt.Errorf("failed to decompress chunk: %w", err) + } + return plain, nil +} + +// zstdEncode returns src compressed as a zstd frame, or an encoder setup error. +func zstdEncode(src []byte) ([]byte, error) { + enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) + if err != nil { + return nil, err + } + defer enc.Close() + return enc.EncodeAll(src, nil), nil +} + +// zstdDecode decompresses zstd data, returning decoder setup or decoding errors. +func zstdDecode(src []byte) ([]byte, error) { + dec, err := zstd.NewReader(nil) + if err != nil { + return nil, err + } + defer dec.Close() + return dec.DecodeAll(src, nil) +} diff --git a/internal/crypto/envelope_test.go b/internal/crypto/envelope_test.go new file mode 100644 index 0000000..44b47f8 --- /dev/null +++ b/internal/crypto/envelope_test.go @@ -0,0 +1,88 @@ +package crypto + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func TestSealOpenChunkRoundTrip(t *testing.T) { + master := make([]byte, 32) + if _, err := rand.Read(master); err != nil { + t.Fatalf("rand.Read: %v", err) + } + plain := []byte("payload with enough bytes to compress meaningfully, payload payload payload") + sum := sha256Hex(plain) + + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + got, err := OpenChunk(master, sum, sealed) + if err != nil { + t.Fatalf("OpenChunk() error: %v", err) + } + if !bytes.Equal(got, plain) { + t.Fatalf("OpenChunk() = %q, want %q", got, plain) + } +} + +func TestOpenChunkRejectsTamperedCiphertext(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + plain := []byte("important data") + sum := sha256Hex(plain) + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + sealed[len(sealed)-1] ^= 0xff + if _, err := OpenChunk(master, sum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with tampered ciphertext") + } +} + +func TestOpenChunkRejectsWrongChunkHash(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + plain := []byte("important data") + sealed, err := SealChunk(master, sha256Hex(plain), plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + wrongSum := sha256Hex([]byte("other data")) + if _, err := OpenChunk(master, wrongSum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with wrong chunk hash") + } +} + +func TestOpenChunkRejectsWrongMasterKey(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + other := []byte("fedcba9876543210fedcba9876543210") + plain := []byte("important data") + sum := sha256Hex(plain) + sealed, err := SealChunk(master, sum, plain) + if err != nil { + t.Fatalf("SealChunk() error: %v", err) + } + if _, err := OpenChunk(other, sum, sealed); err == nil { + t.Fatal("OpenChunk() succeeded with wrong master key") + } +} + +func TestSealChunkRejectsInvalidChunkHash(t *testing.T) { + master := []byte("0123456789abcdef0123456789abcdef") + _, err := SealChunk(master, "not-hex", []byte("x")) + if err == nil { + t.Fatal("SealChunk() succeeded with invalid chunk hash") + } + if !strings.Contains(err.Error(), "invalid chunk hash") { + t.Fatalf("SealChunk() error = %v, want invalid chunk hash", err) + } +} diff --git a/internal/crypto/multi.go b/internal/crypto/multi.go new file mode 100644 index 0000000..961003b --- /dev/null +++ b/internal/crypto/multi.go @@ -0,0 +1,60 @@ +package crypto + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "filippo.io/age" + "filippo.io/age/agessh" +) + +// ParseRecipient accepts age X25519 keys (age1...) and SSH public keys +// (ssh-ed25519 / ssh-rsa). It is the single entry point for repository +// recipients so init, upload, and sealing all accept the same formats. +func ParseRecipient(s string) (age.Recipient, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "ssh-ed25519 ") || strings.HasPrefix(s, "ssh-rsa ") { + return agessh.ParseRecipient(s) + } + return age.ParseX25519Recipient(s) +} + +// EncryptMulti encrypts plaintext to every recipient. Any matching identity +// can decrypt the result. It rejects an empty recipient list and returns +// recipient parsing or age encryption errors. +func EncryptMulti(recipientKeys []string, plaintext []byte) ([]byte, error) { + if len(recipientKeys) == 0 { + return nil, fmt.Errorf("no recipients given") + } + recipients := make([]age.Recipient, 0, len(recipientKeys)) + for _, key := range recipientKeys { + r, err := ParseRecipient(key) + if err != nil { + return nil, fmt.Errorf("invalid recipient %q: %w", key, err) + } + recipients = append(recipients, r) + } + + var buf bytes.Buffer + w, err := age.Encrypt(&buf, recipients...) + if err != nil { + return nil, fmt.Errorf("failed to init age encryptor: %w", err) + } + if _, err := w.Write(plaintext); err != nil { + _ = w.Close() + return nil, fmt.Errorf("failed to write plaintext: %w", err) + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to finish age encryption: %w", err) + } + return buf.Bytes(), nil +} + +// FingerprintRecipient returns a short stable identifier for a recipient key. +func FingerprintRecipient(recipient string) string { + sum := sha256.Sum256([]byte(recipient)) + return hex.EncodeToString(sum[:])[:16] +} diff --git a/internal/crypto/multi_test.go b/internal/crypto/multi_test.go new file mode 100644 index 0000000..d78d10e --- /dev/null +++ b/internal/crypto/multi_test.go @@ -0,0 +1,77 @@ +package crypto + +import ( + "bytes" + "io" + "os" + "path/filepath" + "testing" +) + +func generateTestKey(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func TestEncryptMultiOpensWithAnyRecipientIdentity(t *testing.T) { + pubA, identA := generateTestKey(t) + pubB, identB := generateTestKey(t) + _, identC := generateTestKey(t) + + plaintext := []byte(`{"hello":"world"}`) + sealed, err := EncryptMulti([]string{pubA, pubB}, plaintext) + if err != nil { + t.Fatalf("EncryptMulti() error: %v", err) + } + + for _, identPath := range []string{identA, identB} { + r, err := DecryptReader(identPath, bytes.NewReader(sealed)) + if err != nil { + t.Fatalf("DecryptReader(%s) error: %v", identPath, err) + } + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("decrypted = %q, want %q", got, plaintext) + } + } + + if r, err := DecryptReader(identC, bytes.NewReader(sealed)); err == nil { + if _, err := io.ReadAll(r); err == nil { + t.Fatal("DecryptReader succeeded with unrelated identity") + } + } +} + +func TestEncryptMultiRejectsNoRecipients(t *testing.T) { + if _, err := EncryptMulti(nil, []byte("x")); err == nil { + t.Fatal("EncryptMulti() succeeded with no recipients") + } +} + +func TestEncryptMultiRejectsInvalidRecipient(t *testing.T) { + if _, err := EncryptMulti([]string{"age1notavalidkey"}, []byte("x")); err == nil { + t.Fatal("EncryptMulti() succeeded with invalid recipient") + } +} + +func TestFingerprintRecipientIsStableAndShort(t *testing.T) { + pub, _ := generateTestKey(t) + fp := FingerprintRecipient(pub) + if fp != FingerprintRecipient(pub) { + t.Fatal("FingerprintRecipient() is not stable") + } + if len(fp) != 16 { + t.Fatalf("FingerprintRecipient() = %q, want 16 hex chars", fp) + } +} diff --git a/internal/crypto/parse_recipient_test.go b/internal/crypto/parse_recipient_test.go new file mode 100644 index 0000000..6a57611 --- /dev/null +++ b/internal/crypto/parse_recipient_test.go @@ -0,0 +1,39 @@ +package crypto + +import ( + "testing" + + "filippo.io/age" +) + +func TestParseRecipientAcceptsX25519(t *testing.T) { + pub, _, _, err := GenerateKey() + if err != nil { + t.Fatal(err) + } + r, err := ParseRecipient(pub) + if err != nil { + t.Fatalf("ParseRecipient() error: %v", err) + } + if _, ok := r.(*age.X25519Recipient); !ok { + t.Fatalf("type = %T", r) + } +} + +func TestParseRecipientAcceptsSSHEd25519(t *testing.T) { + // A well-formed ssh-ed25519 public key line (RFC 8709). + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl test@host" + r, err := ParseRecipient(key) + if err != nil { + t.Fatalf("ParseRecipient() error: %v", err) + } + if r == nil { + t.Fatal("nil recipient") + } +} + +func TestParseRecipientRejectsGarbage(t *testing.T) { + if _, err := ParseRecipient("not-a-key"); err == nil { + t.Fatal("ParseRecipient() accepted garbage") + } +} diff --git a/internal/flow/backup.go b/internal/flow/backup.go new file mode 100644 index 0000000..8e96d55 --- /dev/null +++ b/internal/flow/backup.go @@ -0,0 +1,207 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/google/uuid" + "github.com/pterm/pterm" + + "gloak/internal/chunker" + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +const indexCheckpointEvery = 64 + +type BackupResult struct { + SnapshotID string + TotalChunks int + UploadedChunks int + SkippedChunks int + TotalPlainBytes int64 +} + +// RunBackup backs up a file or directory into the repository on remote using +// identityPath and returns the new snapshot UUID. Directory backups skip +// symlinks and special files. It returns filesystem, repository, encryption, +// and storage errors; failed backups may leave uploaded chunks and index epochs. +func RunBackup(filePath, identityPath, remote string) (string, error) { + backend := storage.NewRcloneBackend(remote) + snapshotID := core.GenerateUUID() + + spinner, _ := pterm.DefaultSpinner.Start("Backing up...") + res, err := runBackupPath(filePath, identityPath, backend, snapshotID, func(done int) { + spinner.UpdateText(fmt.Sprintf("Backed up %d chunks", done)) + }) + if err != nil { + _ = spinner.Stop() + return "", err + } + spinner.Success("Backup complete") + pterm.Printf("Snapshot ID: %s\n", res.SnapshotID) + pterm.Printf("Chunks: %d total, %d uploaded, %d skipped\n", res.TotalChunks, res.UploadedChunks, res.SkippedChunks) + pterm.Printf("Data size: %d bytes\n", res.TotalPlainBytes) + return res.SnapshotID, nil +} + +// runBackupPath validates snapshotID as a UUID and dispatches directories to +// tree backup, all other paths to file backup. It returns stat or backup errors. +// A non-nil progress callback receives the cumulative number of processed chunks. +func runBackupPath(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) + } + info, err := os.Stat(filePath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to stat path: %w", err) + } + // Advisory shared lock: a concurrent exclusive prune must not run + // while this backup writes. + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + return BackupResult{}, err + } + defer release() + + if info.IsDir() { + return runTreeBackup(filePath, identityPath, backend, snapshotID, progress) + } + return runBackup(filePath, identityPath, backend, snapshotID, progress) +} + +// runBackup writes a file snapshot under snapshotID, reusing indexed chunks. +// progress, when non-nil, receives the cumulative chunk count including reused +// chunks. It returns statistics on success and a zero result on invalid UUIDs, +// filesystem, repository, encryption, or storage errors. Uploaded objects and +// saved index checkpoints remain after failures. +func runBackup(filePath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return BackupResult{}, fmt.Errorf("invalid snapshot id: %w", err) + } + + r, err := repo.Open(backend, identityPath) + if err != nil { + return BackupResult{}, err + } + + ix, err := index.Load(backend, identityPath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to load index: %w", err) + } + + cdc, err := chunker.NewCDC(r.Config.ChunkMin, r.Config.ChunkAvg, r.Config.ChunkMax) + if err != nil { + return BackupResult{}, fmt.Errorf("invalid chunk config: %w", err) + } + + file, err := os.Open(filePath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + + res := BackupResult{SnapshotID: snapshotID} + var metas []core.ChunkMeta + var sinceCheckpoint int + payloadHasher := sha256.New() + + err = cdc.Split(file, func(chunk []byte) error { + payloadHasher.Write(chunk) + hash := hashBytes(chunk) + + res.TotalChunks++ + res.TotalPlainBytes += int64(len(chunk)) + + dataPath := repo.DataPath(hash) + + if ix.Has(hash) { + res.SkippedChunks++ + } else { + sealed, err := crypto.SealChunk(r.MasterKey, hash, chunk) + if err != nil { + return fmt.Errorf("failed to seal chunk %s: %w", hash, err) + } + encSum := sha256.Sum256(sealed) + if err := backend.Upload(dataPath, bytes.NewReader(sealed)); err != nil { + return fmt.Errorf("failed to upload chunk %s: %w", hash, err) + } + ix.Add(index.Entry{ + PlainSHA256: hash, + PlainSize: int64(len(chunk)), + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + res.UploadedChunks++ + sinceCheckpoint++ + if sinceCheckpoint >= indexCheckpointEvery { + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return fmt.Errorf("failed to checkpoint index: %w", err) + } + sinceCheckpoint = 0 + } + } + + encEntry, _ := ix.Get(hash) + metas = append(metas, core.ChunkMeta{ + Index: len(metas), + Name: dataPath, + PlainSize: encEntry.PlainSize, + PlainSHA256: hash, + EncryptedSize: encEntry.EncryptedSize, + EncryptedSHA256: encEntry.EncryptedSHA256, + }) + + if progress != nil { + progress(res.TotalChunks) + } + return nil + }) + if err != nil { + return BackupResult{}, err + } + + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return BackupResult{}, fmt.Errorf("failed to save index: %w", err) + } + + recipient := "" + if len(r.Config.Recipients) > 0 { + recipient = r.Config.Recipients[0] + } + manifest := core.Manifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + OriginalName: filepath.Base(filePath), + OriginalSize: res.TotalPlainBytes, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Recipient: recipient, + PayloadSHA256: hex.EncodeToString(payloadHasher.Sum(nil)), + Chunks: metas, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to marshal manifest: %w", err) + } + sealed, err := crypto.EncryptMulti(r.Config.Recipients, manifestJSON) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to encrypt manifest: %w", err) + } + snapshotPath, err := repo.SnapshotPath(snapshotID) + if err != nil { + return BackupResult{}, err + } + if err := backend.Upload(snapshotPath, bytes.NewReader(sealed)); err != nil { + return BackupResult{}, fmt.Errorf("failed to upload snapshot: %w", err) + } + return res, nil +} diff --git a/internal/flow/backup_test.go b/internal/flow/backup_test.go new file mode 100644 index 0000000..3326eca --- /dev/null +++ b/internal/flow/backup_test.go @@ -0,0 +1,195 @@ +package flow + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func repoFixture(t *testing.T) (string, *memtest.MemoryBackend, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + identPath := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(identPath, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + backend := memtest.NewMemoryBackend() + // min=1 avg=8 max=64: tiny test payloads stay one chunk unless tests + // craft multi-chunk files themselves. + if _, _, err := repo.Init(backend, []string{pub}, 1, 8, 64, false); err != nil { + t.Fatalf("repo.Init() error: %v", err) + } + return identPath, backend, pub +} + +func writeTempFile(t *testing.T, data []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "payload.bin") + if err := os.WriteFile(path, data, 0600); err != nil { + t.Fatalf("write temp file: %v", err) + } + return path +} + +func TestBackupWritesSnapshotAndDataObjects(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + + res, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) + if err != nil { + t.Fatalf("runBackup() error: %v", err) + } + if res.TotalChunks < 1 || res.UploadedChunks != res.TotalChunks || res.SkippedChunks != 0 { + t.Fatalf("result = %+v", res) + } + if res.TotalPlainBytes != int64(len(payload)) { + t.Fatalf("TotalPlainBytes = %d", res.TotalPlainBytes) + } + if !backend.Has(repo.SnapshotsPrefix + "/" + res.SnapshotID + ".age") { + t.Fatal("snapshot file missing") + } + if got := backend.TotalUploads(repo.DataPrefix + "/"); got != res.UploadedChunks { + t.Fatalf("data uploads = %d, want %d", got, res.UploadedChunks) + } + if !backend.Has("index/000001.age") { + t.Fatal("index epoch missing") + } +} + +func TestBackupSkipsChunksAlreadyInIndex(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + + id1 := "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + if _, err := runBackup(filePath, ident, backend, id1, nil); err != nil { + t.Fatalf("first runBackup() error: %v", err) + } + dataUploadsBefore := backend.TotalUploads(repo.DataPrefix + "/") + + id2 := "aaaaaaaa-bbbb-4ccc-8ddd-ffffffffffff" + res, err := runBackup(filePath, ident, backend, id2, nil) + if err != nil { + t.Fatalf("second runBackup() error: %v", err) + } + if res.UploadedChunks != 0 || res.SkippedChunks != res.TotalChunks { + t.Fatalf("second result = %+v, want all skipped", res) + } + if backend.TotalUploads(repo.DataPrefix+"/") != dataUploadsBefore { + t.Fatal("second backup re-uploaded data objects") + } + if !backend.Has(repo.SnapshotsPrefix + "/" + id2 + ".age") { + t.Fatal("second snapshot missing") + } +} + +func TestBackupDedupsIdenticalContentAcrossFiles(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + fileA := writeTempFile(t, payload) + fileB := filepath.Join(t.TempDir(), "copy.bin") + if err := os.WriteFile(fileB, payload, 0600); err != nil { + t.Fatal(err) + } + + if _, err := runBackup(fileA, ident, backend, "11111111-2222-4333-8444-555555555555", nil); err != nil { + t.Fatalf("first runBackup() error: %v", err) + } + before := backend.TotalUploads(repo.DataPrefix + "/") + + res, err := runBackup(fileB, ident, backend, "11111111-2222-4333-8444-666666666666", nil) + if err != nil { + t.Fatalf("second runBackup() error: %v", err) + } + if res.UploadedChunks != 0 || res.SkippedChunks != res.TotalChunks { + t.Fatalf("second result = %+v, want all skipped", res) + } + if backend.TotalUploads(repo.DataPrefix+"/") != before { + t.Fatal("second backup re-uploaded data objects") + } +} + +func TestBackupFailsWithoutRepository(t *testing.T) { + ident, _, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("x")) + _, err := runBackup(filePath, ident, memtest.NewMemoryBackend(), "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil) + if err == nil { + t.Fatal("runBackup() succeeded without repository") + } + if !strings.Contains(err.Error(), "not a gloak repository") { + t.Fatalf("error = %v, want not a gloak repository", err) + } +} + +func TestBackupRejectsInvalidSnapshotID(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("x")) + _, err := runBackup(filePath, ident, backend, "../escape", nil) + if err == nil { + t.Fatal("runBackup() succeeded with invalid snapshot id") + } + if !strings.Contains(err.Error(), "invalid snapshot id") { + t.Fatalf("error = %v", err) + } +} + +func TestBackupObjectMatchesChunkHash(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + snapshotID := "99999999-8888-4777-8666-555555555555" + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { + t.Fatalf("runBackup() error: %v", err) + } + m, err := readManifest(backend, ident, snapshotID) + if err != nil { + t.Fatal(err) + } + if len(m.Chunks) == 0 { + t.Fatal("no chunks in manifest") + } + if !backend.Has(m.Chunks[0].Name) { + t.Fatalf("expected object at %s", m.Chunks[0].Name) + } +} + +func TestBackupCDCQualityAfterInsert(t *testing.T) { + ident, backend, _ := repoFixture(t) + base := make([]byte, 200_000) + for i := range base { + base[i] = byte(i * 31) + } + fileA := writeTempFile(t, base) + snapA := "aaaaaaaa-bbbb-4ccc-8ddd-000000000001" + resA, err := runBackup(fileA, ident, backend, snapA, nil) + if err != nil { + t.Fatal(err) + } + if resA.TotalChunks < 3 { + t.Fatalf("TotalChunks = %d, want >= 3", resA.TotalChunks) + } + + // Insert one byte near the start; most later chunk hashes should survive. + shifted := make([]byte, 0, len(base)+1) + shifted = append(shifted, base[:100]...) + shifted = append(shifted, 0xAB) + shifted = append(shifted, base[100:]...) + fileB := writeTempFile(t, shifted) + snapB := "aaaaaaaa-bbbb-4ccc-8ddd-000000000002" + resB, err := runBackup(fileB, ident, backend, snapB, nil) + if err != nil { + t.Fatal(err) + } + if resB.SkippedChunks == 0 { + t.Fatal("expected some chunks reused after single-byte insert") + } +} diff --git a/internal/flow/check.go b/internal/flow/check.go new file mode 100644 index 0000000..c40ba68 --- /dev/null +++ b/internal/flow/check.go @@ -0,0 +1,139 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// CheckResult is the structural consistency report for a repository. +type CheckResult struct { + Snapshots int + Referenced int + DataObjects int + Missing []string + Garbage []string +} + +// RunCheck opens the repository and prints a structural consistency report. +// It checks object presence without reading chunk contents. Missing referenced +// objects and repository or manifest errors fail the check; garbage alone does not. +func RunCheck(identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Checking repository...") + res, err := runCheck(identityPath, backend) + if err != nil { + _ = spinner.Stop() + return err + } + spinner.Success("Repository check passed") + pterm.Printf("Snapshots: %d\n", res.Snapshots) + pterm.Printf("Referenced chunks: %d\n", res.Referenced) + pterm.Printf("Data objects: %d\n", res.DataObjects) + if len(res.Garbage) > 0 { + pterm.Warning.Printf("Garbage objects: %d (run `gloak prune --yes` to remove them)\n", len(res.Garbage)) + } + return nil +} + +// runCheck compares hashes referenced by snapshots or the index with data +// object names, returning sorted missing and garbage hashes. It does not verify +// chunk contents. Snapshot listing failures are treated as an empty listing; +// repository, index, manifest, and data listing errors are returned. Missing +// objects return an error with the report; garbage alone is not an error. +func runCheck(identityPath string, backend storage.Backend) (CheckResult, error) { + var res CheckResult + + if _, err := repo.Open(backend, identityPath); err != nil { + return res, err + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return res, fmt.Errorf("failed to load index: %w", err) + } + + snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + // A fresh repo has no snapshots/ directory yet; only a confirmed + // missing path may be treated as empty, other failures are real. + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list snapshots: %w", err) + } + snapRaw = []byte("[]") + } + snapEntries, err := storage.ParseList(snapRaw) + if err != nil { + return res, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + + referenced := make(map[string]bool) + for _, entry := range snapEntries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) + if err != nil { + return res, fmt.Errorf("snapshot %s: %w", id, err) + } + res.Snapshots++ + if kind == core.KindTree { + for _, te := range treeM.Entries { + for _, meta := range te.Chunks { + referenced[meta.PlainSHA256] = true + } + } + } else { + for _, meta := range fileM.Chunks { + referenced[meta.PlainSHA256] = true + } + } + } + for _, h := range ix.AllHashes() { + referenced[h] = true + } + res.Referenced = len(referenced) + + dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) + if err != nil { + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list data objects: %w", err) + } + dataPaths = nil + } + present := make(map[string]bool, len(dataPaths)) + for _, p := range dataPaths { + h := path.Base(p) + if len(h) != 64 { + continue + } + present[h] = true + } + res.DataObjects = len(present) + + for h := range referenced { + if !present[h] { + res.Missing = append(res.Missing, h) + } + } + sort.Strings(res.Missing) + for h := range present { + if !referenced[h] { + res.Garbage = append(res.Garbage, h) + } + } + sort.Strings(res.Garbage) + + if len(res.Missing) > 0 { + return res, fmt.Errorf("check failed: %d referenced chunk(s) missing from storage", len(res.Missing)) + } + return res, nil +} diff --git a/internal/flow/check_test.go b/internal/flow/check_test.go new file mode 100644 index 0000000..ae9f0dc --- /dev/null +++ b/internal/flow/check_test.go @@ -0,0 +1,79 @@ +package flow + +import ( + "bytes" + "strings" + "testing" +) + +func TestRunCheckCleanRepository(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if res.Snapshots != 1 || len(res.Missing) != 0 { + t.Fatalf("result = %+v", res) + } + if res.DataObjects < 1 { + t.Fatalf("DataObjects = %d", res.DataObjects) + } +} + +func TestRunCheckReportsMissingObject(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + keys := backend.KeysWithPrefix("data/") + if len(keys) == 0 { + t.Fatal("no data objects") + } + backend.Delete(keys[0]) + + _, err := runCheck(ident, backend) + if err == nil { + t.Fatal("runCheck() succeeded with missing object") + } + if !strings.Contains(err.Error(), "missing") { + t.Fatalf("error = %v, want missing mention", err) + } +} + +func TestRunCheckReportsGarbageWithoutError(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + _ = backend.Upload("data/ff/ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", bytes.NewReader([]byte("orphan"))) + + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if len(res.Garbage) != 1 { + t.Fatalf("Garbage = %v, want one entry", res.Garbage) + } +} + +func TestRunCheckRejectsCorruptSnapshot(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("check me please")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + key := "snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age" + blob := backend.Get(key) + blob[len(blob)/2] ^= 0xff + backend.Put(key, blob) + + if _, err := runCheck(ident, backend); err == nil { + t.Fatal("runCheck() succeeded on corrupt snapshot") + } +} diff --git a/internal/flow/filemode.go b/internal/flow/filemode.go new file mode 100644 index 0000000..efd858b --- /dev/null +++ b/internal/flow/filemode.go @@ -0,0 +1,179 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +// isFileMode reports whether backupID/locator.json can be downloaded. +// It does not parse the locator, and any download error yields false. +func isFileMode(backend storage.Backend, backupID string) bool { + var buf bytes.Buffer + return backend.Download(backupID+"/locator.json", &buf) == nil +} + +// readFileManifest loads a standalone backup's locator and encrypted manifest. +// It checks both UUIDs, the manifest version, and that OriginalName equals its +// filepath.Base. Locator download failures become an incomplete-or-missing +// backup error; JSON, manifest download, identity, and decryption errors are returned. +func readFileManifest(backend storage.Backend, identityPath, backupID string) (fileManifest, error) { + var loc fileLocator + var locBuf bytes.Buffer + if err := backend.Download(backupID+"/locator.json", &locBuf); err != nil { + return fileManifest{}, fmt.Errorf("backup %s incomplete or missing: no locator.json", backupID) + } + if err := json.Unmarshal(locBuf.Bytes(), &loc); err != nil { + return fileManifest{}, fmt.Errorf("invalid locator: %w", err) + } + if loc.UUID != backupID { + return fileManifest{}, fmt.Errorf("locator uuid mismatch: got %s want %s", loc.UUID, backupID) + } + + var manBuf bytes.Buffer + if err := backend.Download(backupID+"/manifest.age", &manBuf); err != nil { + return fileManifest{}, fmt.Errorf("failed to download manifest: %w", err) + } + reader, err := crypto.DecryptReader(identityPath, &manBuf) + if err != nil { + return fileManifest{}, fmt.Errorf("failed to decrypt manifest: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return fileManifest{}, fmt.Errorf("failed to read manifest: %w", err) + } + var m fileManifest + if err := json.Unmarshal(plain, &m); err != nil { + return fileManifest{}, fmt.Errorf("failed to parse manifest: %w", err) + } + if m.UUID != backupID { + return fileManifest{}, fmt.Errorf("manifest uuid mismatch: got %s want %s", m.UUID, backupID) + } + if m.Version != fileFormatVersion { + return fileManifest{}, fmt.Errorf("unsupported manifest version: %s", m.Version) + } + if filepath.Base(m.OriginalName) != m.OriginalName { + return fileManifest{}, fmt.Errorf("invalid original filename in manifest: %s", m.OriginalName) + } + return m, nil +} + +// decodeFileMasterKey decodes the manifest's hex key, returning an error if it +// is malformed or does not contain exactly 32 bytes. +func decodeFileMasterKey(m fileManifest) ([]byte, error) { + key, err := hex.DecodeString(m.MasterKey) + if err != nil || len(key) != masterKeySize { + return nil, fmt.Errorf("invalid master key in manifest") + } + return key, nil +} + +// runFileRestore restores a standalone backup under outputDir and returns the +// output path. It creates parent directories and truncates an existing output +// file. Manifest, key, chunk path, download, decryption, checksum, and filesystem +// errors are returned; a failure can leave a partial file. The success path +// checks the output file's close error; close errors on failure paths are +// ignored. +func runFileRestore(backupID, identityPath, outputDir string, backend storage.Backend) (string, error) { + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + return "", err + } + masterKey, err := decodeFileMasterKey(m) + if err != nil { + return "", err + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + outputPath := filepath.Join(outputDir, m.OriginalName) + outFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return "", fmt.Errorf("failed to create output file: %w", err) + } + defer outFile.Close() + + payloadHasher := sha256.New() + for _, meta := range m.Chunks { + if err := validateFileChunkName(meta.Name); err != nil { + return "", fmt.Errorf("invalid chunk name in manifest: %w", err) + } + var buf bytes.Buffer + if err := backend.Download(backupID+"/"+meta.Name, &buf); err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + plain, err := crypto.OpenChunk(masterKey, meta.PlainSHA256, buf.Bytes()) + if err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + if _, err := outFile.Write(plain); err != nil { + return "", fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) + } + payloadHasher.Write(plain) + } + + if hex.EncodeToString(payloadHasher.Sum(nil)) != m.PayloadSHA256 { + return "", fmt.Errorf("fatal: overall payload hash does not match manifest") + } + if err := outFile.Close(); err != nil { + return "", fmt.Errorf("failed to close output file: %w", err) + } + return outputPath, nil +} + +// runFileVerify reads a standalone backup without writing restored files. +// It verifies encrypted and plaintext chunk sizes and hashes, authentication, +// and the overall payload hash, returning manifest, key, path, download, +// decryption, or integrity errors. +func runFileVerify(backupID, identityPath string, backend storage.Backend) error { + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + return err + } + masterKey, err := decodeFileMasterKey(m) + if err != nil { + return err + } + + payloadHasher := sha256.New() + for _, meta := range m.Chunks { + if err := validateFileChunkName(meta.Name); err != nil { + return fmt.Errorf("invalid chunk name in manifest: %w", err) + } + var buf bytes.Buffer + if err := backend.Download(backupID+"/"+meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) + } + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) + } + plain, err := crypto.OpenChunk(masterKey, meta.PlainSHA256, sealed) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + payloadHasher.Write(plain) + } + if hex.EncodeToString(payloadHasher.Sum(nil)) != m.PayloadSHA256 { + return fmt.Errorf("FATAL: overall payload hash differs from manifest") + } + return nil +} diff --git a/internal/flow/filemode_test.go b/internal/flow/filemode_test.go new file mode 100644 index 0000000..75a9de2 --- /dev/null +++ b/internal/flow/filemode_test.go @@ -0,0 +1,234 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage/memtest" +) + +func resealFileManifest(t *testing.T, backend *memtest.MemoryBackend, identityPath, uuid string, m fileManifest) { + t.Helper() + data, err := os.ReadFile(identityPath) + if err != nil { + t.Fatal(err) + } + var recipient string + for _, line := range bytes.Split(data, []byte("\n")) { + if bytes.HasPrefix(bytes.TrimSpace(line), []byte("# public key:")) { + recipient = string(bytes.TrimSpace(bytes.TrimPrefix(bytes.TrimSpace(line), []byte("# public key:")))) + break + } + } + if recipient == "" { + t.Fatal("no public key in identity file") + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + sealed, err := crypto.EncryptMulti([]string{recipient}, raw) + if err != nil { + t.Fatal(err) + } + backend.Put(uuid+"/manifest.age", sealed) +} + +func TestRunRestoreFileModeRoundTrip(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "secret.txt") + want := []byte("restore me please\n") + if err := os.WriteFile(src, want, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("upload: %v", err) + } + + outDir := t.TempDir() + gotPath, err := runRestore(backupID, outDir, identityPath, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + got, err := os.ReadFile(gotPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("restored %q, want %q", got, want) + } +} + +func TestRunRestoreFileModeEmptyFile(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "empty.bin") + if err := os.WriteFile(src, nil, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("upload: %v", err) + } + + outDir := t.TempDir() + gotPath, err := runRestore(backupID, outDir, identityPath, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + got, err := os.ReadFile(gotPath) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("restored %d bytes, want empty", len(got)) + } +} + +func TestRunRestoreFileModeEitherIdentity(t *testing.T) { + r1, id1 := testIdentityFile(t) + r2, id2 := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "m.bin") + if err := os.WriteFile(src, []byte("two keys"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{r1, r2}, backend) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{id1, id2} { + out := t.TempDir() + if _, err := runRestore(backupID, out, id, backend, ""); err != nil { + t.Fatalf("restore with %s: %v", id, err) + } + } +} + +func TestRunRestoreFileModeMissingLocator(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + backend.Delete(backupID + "/locator.json") + + // Without locator, isFileMode is false, so restore falls through to + // repository Open. Either "incomplete" or a repository error is fine; + // we only assert it fails. + if _, err := runRestore(backupID, t.TempDir(), identityPath, backend, ""); err == nil { + t.Fatal("expected error for missing locator") + } +} + +func TestRunRestoreFileModeRejectsTraversalManifest(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("data"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + m, err := readFileManifest(backend, identityPath, backupID) + if err != nil { + t.Fatal(err) + } + m.Chunks[0].Name = "../evil" + resealFileManifest(t, backend, identityPath, backupID, m) + + _, err = runRestore(backupID, t.TempDir(), identityPath, backend, "") + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("invalid chunk")) { + t.Fatalf("err = %v, want invalid chunk", err) + } +} + +func TestRunRestoreFileModeCorruptChunk(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "a.bin") + if err := os.WriteFile(src, []byte("abcdefgh"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + raw := backend.Get(backupID + "/chunk_00000") + raw[len(raw)/2] ^= 0xff + backend.Put(backupID+"/chunk_00000", raw) + + _, err = runRestore(backupID, t.TempDir(), identityPath, backend, "") + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("CHUNK CORRUPT")) { + t.Fatalf("err = %v, want CHUNK CORRUPT", err) + } +} + +func TestRunVerifyFileModeOK(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("verify me"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + if err := runVerify(backupID, identityPath, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) + } +} + +func TestRunVerifyFileModeCorrupt(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("verify me"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + raw := backend.Get(backupID + "/chunk_00000") + raw[0] ^= 0xff + backend.Put(backupID+"/chunk_00000", raw) + + err = runVerify(backupID, identityPath, backend) + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("CHUNK CORRUPT")) { + t.Fatalf("err = %v, want CHUNK CORRUPT", err) + } +} + +func TestRunVerifyFileModeMissingLocator(t *testing.T) { + recipient, identityPath := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "v.bin") + if err := os.WriteFile(src, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + backupID, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatal(err) + } + backend.Delete(backupID + "/locator.json") + + // Without locator, falls through to repository verify; must fail. + if err := runVerify(backupID, identityPath, backend); err == nil { + t.Fatal("expected error for missing locator") + } +} diff --git a/internal/flow/flow_test.go b/internal/flow/flow_test.go index 8916320..b8c6d34 100644 --- a/internal/flow/flow_test.go +++ b/internal/flow/flow_test.go @@ -3,383 +3,109 @@ package flow import ( "bytes" "encoding/json" - "fmt" - "io" "os" - "path/filepath" "strings" - "sync" "testing" + "gloak/internal/core" appcrypto "gloak/internal/crypto" - "gloak/internal/storage" + "gloak/internal/repo" ) -func TestUploadVerifyRestoreRoundTrip(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - identityPath := writeFlowIdentityFile(t, identityFile) - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - wantPayload := []byte("important backup payload\nwith enough bytes to test restore") - if err := os.WriteFile(sourcePath, wantPayload, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } +func TestBackupVerifyRestoreRoundTrip(t *testing.T) { + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - backend := newMemoryBackend() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - - if err := runUpload(sourcePath, pubKey, backend, backupID); err != nil { - t.Fatalf("runUpload() returned error: %v", err) - } - for _, path := range []string{backupID + "/chunk_00000", backupID + "/manifest.age", backupID + "/locator.json"} { - if !backend.has(path) { - t.Fatalf("uploaded path %q not found", path) - } + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { + t.Fatalf("runBackup() error: %v", err) } - - if err := runVerify(backupID, identityPath, backend); err != nil { - t.Fatalf("runVerify() returned error: %v", err) + if err := runVerify(snapshotID, ident, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) } - outputDir := t.TempDir() - if err := runRestore(backupID, identityPath, backend, outputDir); err != nil { - t.Fatalf("runRestore() returned error: %v", err) + outDir := t.TempDir() + restoredPath, err := runRestore(snapshotID, outDir, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) } - - gotPayload, err := os.ReadFile(filepath.Join(outputDir, "secret.txt")) + got, err := os.ReadFile(restoredPath) if err != nil { - t.Fatalf("failed to read restored file: %v", err) + t.Fatalf("read restored: %v", err) } - if !bytes.Equal(gotPayload, wantPayload) { - t.Fatalf("restored payload = %q, want %q", gotPayload, wantPayload) + if !bytes.Equal(got, payload) { + t.Fatalf("restored = %q, want %q", got, payload) } } func TestVerifyRejectsCorruptChunk(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) + ident, backend, _ := repoFixture(t) + payload := []byte("0123456789abcdefghij") + filePath := writeTempFile(t, payload) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" + if _, err := runBackup(filePath, ident, backend, snapshotID, nil); err != nil { + t.Fatalf("runBackup() error: %v", err) } - identityPath := writeFlowIdentityFile(t, identityFile) - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) + // Corrupt the first data object in place. + var dataKey string + for _, key := range backend.KeysWithPrefix(repo.DataPrefix + "/") { + dataKey = key + break } - - backend := newMemoryBackend() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - if err := runUpload(sourcePath, pubKey, backend, backupID); err != nil { - t.Fatalf("runUpload() returned error: %v", err) + if dataKey == "" { + t.Fatal("no data objects to corrupt") } + blob := backend.Get(dataKey) + blob[len(blob)-1] ^= 0xff + backend.Put(dataKey, blob) - backend.put(backupID+"/chunk_00000", []byte("tampered")) - err = runVerify(backupID, identityPath, backend) + err := runVerify(snapshotID, ident, backend) if err == nil { - t.Fatalf("runVerify() succeeded with corrupt chunk") + t.Fatal("runVerify() succeeded on corrupt chunk") } if !strings.Contains(err.Error(), "CHUNK CORRUPT") { - t.Fatalf("runVerify() error = %q, want chunk corrupt", err) - } -} - -func TestRunUploadWithStateDeletesStateAfterSuccess(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() + t.Fatalf("error = %v, want CHUNK CORRUPT", err) + } +} + +func TestRestoreRejectsEscapingChunkPath(t *testing.T) { + ident, backend, pub := repoFixture(t) + snapshotID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" + + manifest := core.Manifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + OriginalName: "payload.bin", + OriginalSize: 4, + CreatedAt: "2026-01-01T00:00:00Z", + Recipient: pub, + PayloadSHA256: hashBytes([]byte("data")), + Chunks: []core.ChunkMeta{{ + Index: 0, + Name: "../evil", + PlainSize: 4, + PlainSHA256: hashBytes([]byte("data")), + }}, + } + plain, err := json.Marshal(manifest) if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) + t.Fatalf("marshal manifest: %v", err) } - - backend := newMemoryBackend() - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - if err := runUploadWithState(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir); err != nil { - t.Fatalf("runUploadWithState() returned error: %v", err) - } - - if _, err := os.Stat(uploadStatePath(stateDir, backupID)); !os.IsNotExist(err) { - t.Fatalf("upload state should be deleted after success, stat error: %v", err) - } -} - -func TestRunUploadWithStateKeepsUploadedChunksAfterFailure(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() + sealed, err := appcrypto.EncryptMulti([]string{pub}, plain) if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) + t.Fatalf("EncryptMulti() error: %v", err) } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - backend := &failUploadBackend{ - memoryBackend: newMemoryBackend(), - failPath: backupID + "/manifest.age", - } - stateDir := t.TempDir() - err = runUploadWithState(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir) - if err == nil { - t.Fatalf("runUploadWithState() succeeded despite manifest upload failure") - } - if !strings.Contains(err.Error(), "failed to upload manifest.age") { - t.Fatalf("runUploadWithState() error = %q, want manifest upload failure", err) - } - - state, err := loadUploadState(stateDir, backupID) + snapshotPath, err := repo.SnapshotPath(snapshotID) if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if state.Remote != "memory:backups" || state.Recipient != pubKey { - t.Fatalf("state remote/recipient = %q/%q, want memory:backups/%q", state.Remote, state.Recipient, pubKey) + t.Fatalf("SnapshotPath() error: %v", err) } - absSourcePath, err := filepath.Abs(sourcePath) - if err != nil { - t.Fatalf("failed to resolve source path: %v", err) - } - if state.SourcePath != absSourcePath { - t.Fatalf("state SourcePath = %q, want %q", state.SourcePath, absSourcePath) - } - if len(state.UploadedChunks) != 1 { - t.Fatalf("uploaded chunks in state = %d, want 1", len(state.UploadedChunks)) - } - if state.UploadedChunks[0].Name != "chunk_00000" || state.UploadedChunks[0].PlainSize == 0 || state.UploadedChunks[0].PlainSHA256 == "" || state.UploadedChunks[0].EncryptedSize == 0 || state.UploadedChunks[0].EncryptedSHA256 == "" { - t.Fatalf("uploaded chunk metadata not recorded correctly: %+v", state.UploadedChunks[0]) - } -} - -func TestRunUploadWithStateStopsAtMidUploadFailure(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload large enough for several encrypted chunks"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - backend := &failUploadBackend{ - memoryBackend: newMemoryBackend(), - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - err = runUploadWithStateAndChunkSize(sourcePath, pubKey, backend, backupID, "memory:backups", stateDir, 8) - if err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - if !strings.Contains(err.Error(), "chunk 1 failed to process") { - t.Fatalf("runUploadWithStateAndChunkSize() error = %q, want chunk 1 failure", err) - } - - state, err := loadUploadState(stateDir, backupID) - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if state.ChunkSize != 8 { - t.Fatalf("state ChunkSize = %d, want 8", state.ChunkSize) - } - if len(state.UploadedChunks) != 1 { - t.Fatalf("uploaded chunks in state = %d, want 1", len(state.UploadedChunks)) - } - if state.UploadedChunks[0].Name != "chunk_00000" { - t.Fatalf("recorded chunk = %q, want chunk_00000", state.UploadedChunks[0].Name) - } - - if !backend.has(backupID + "/chunk_00000") { - t.Fatalf("chunk_00000 should exist after first chunk upload") - } - for _, path := range []string{backupID + "/chunk_00001", backupID + "/manifest.age", backupID + "/locator.json"} { - if backend.has(path) { - t.Fatalf("%s should not exist after mid-upload failure", path) - } - } -} - -func TestRunResumeCompletesInterruptedUpload(t *testing.T) { - pubKey, _, identityFile, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - identityPath := writeFlowIdentityFile(t, identityFile) - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - wantPayload := []byte("payload large enough for several encrypted chunks") - if err := os.WriteFile(sourcePath, wantPayload, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - memory := newMemoryBackend() - failing := &failUploadBackend{ - memoryBackend: memory, - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - if err := runUploadWithStateAndChunkSize(sourcePath, pubKey, failing, backupID, "memory:backups", stateDir, 8); err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - - if err := runResume(backupID, func(string) storage.Backend { return memory }, stateDir); err != nil { - t.Fatalf("runResume() returned error: %v", err) - } - if _, err := os.Stat(uploadStatePath(stateDir, backupID)); !os.IsNotExist(err) { - t.Fatalf("upload state should be deleted after resume, stat error: %v", err) - } - if memory.uploadCount(backupID+"/chunk_00000") != 1 { - t.Fatalf("chunk_00000 upload count = %d, want 1", memory.uploadCount(backupID+"/chunk_00000")) - } - if memory.uploadCount(backupID+"/chunk_00001") != 1 { - t.Fatalf("chunk_00001 upload count = %d, want 1", memory.uploadCount(backupID+"/chunk_00001")) - } - - if err := runVerify(backupID, identityPath, memory); err != nil { - t.Fatalf("runVerify() returned error after resume: %v", err) - } - - outputDir := t.TempDir() - if err := runRestore(backupID, identityPath, memory, outputDir); err != nil { - t.Fatalf("runRestore() returned error after resume: %v", err) - } - gotPayload, err := os.ReadFile(filepath.Join(outputDir, "secret.txt")) - if err != nil { - t.Fatalf("failed to read restored file: %v", err) - } - if !bytes.Equal(gotPayload, wantPayload) { - t.Fatalf("restored payload = %q, want %q", gotPayload, wantPayload) - } -} - -func TestRunResumeRejectsStateChunkPlainHashMismatch(t *testing.T) { - pubKey, _, _, err := appcrypto.GenerateKey() - if err != nil { - t.Fatalf("GenerateKey() returned error: %v", err) - } - - sourcePath := filepath.Join(t.TempDir(), "secret.txt") - if err := os.WriteFile(sourcePath, []byte("payload large enough for several encrypted chunks"), 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - memory := newMemoryBackend() - failing := &failUploadBackend{ - memoryBackend: memory, - failPath: backupID + "/chunk_00001", - } - stateDir := t.TempDir() - if err := runUploadWithStateAndChunkSize(sourcePath, pubKey, failing, backupID, "memory:backups", stateDir, 8); err == nil { - t.Fatalf("runUploadWithStateAndChunkSize() succeeded despite chunk upload failure") - } - - state, err := loadUploadState(stateDir, backupID) - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - state.UploadedChunks[0].PlainSHA256 = "bad-hash" - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - err = runResume(backupID, func(string) storage.Backend { return memory }, stateDir) - if err == nil { - t.Fatalf("runResume() succeeded with mismatched state chunk hash") - } - if !strings.Contains(err.Error(), "does not match source file") { - t.Fatalf("runResume() error = %q, want source mismatch error", err) - } -} - -type memoryBackend struct { - mu sync.Mutex - files map[string][]byte - uploads map[string]int -} - -type failUploadBackend struct { - *memoryBackend - failPath string -} - -func (f *failUploadBackend) Upload(remotePath string, in io.Reader) error { - if remotePath == f.failPath { - _, _ = io.Copy(io.Discard, in) - return fmt.Errorf("forced upload failure for %s", remotePath) - } - return f.memoryBackend.Upload(remotePath, in) -} - -func newMemoryBackend() *memoryBackend { - return &memoryBackend{files: make(map[string][]byte), uploads: make(map[string]int)} -} - -func (m *memoryBackend) Upload(remotePath string, in io.Reader) error { - data, err := io.ReadAll(in) - if err != nil { - return err - } - m.put(remotePath, data) - m.mu.Lock() - m.uploads[remotePath]++ - m.mu.Unlock() - return nil -} - -func (m *memoryBackend) Download(remotePath string, out io.Writer) error { - m.mu.Lock() - data, ok := m.files[remotePath] - m.mu.Unlock() - if !ok { - return fmt.Errorf("not found: %s", remotePath) - } - _, err := out.Write(data) - return err -} - -func (m *memoryBackend) ListFiles(prefix []string) ([]byte, error) { - return json.Marshal([]struct{}{}) -} - -func (m *memoryBackend) Delete(remotePath string) error { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.files, remotePath) - return nil -} - -func (m *memoryBackend) has(remotePath string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.files[remotePath] - return ok -} - -func (m *memoryBackend) put(remotePath string, data []byte) { - m.mu.Lock() - defer m.mu.Unlock() - m.files[remotePath] = append([]byte(nil), data...) -} - -func (m *memoryBackend) uploadCount(remotePath string) int { - m.mu.Lock() - defer m.mu.Unlock() - return m.uploads[remotePath] -} + backend.Put(snapshotPath, sealed) -func writeFlowIdentityFile(t *testing.T, content string) string { - t.Helper() - path := filepath.Join(t.TempDir(), "identity.txt") - if err := os.WriteFile(path, []byte(content), 0600); err != nil { - t.Fatalf("failed to write identity file: %v", err) + if _, err := runRestore(snapshotID, t.TempDir(), ident, backend, ""); err == nil { + t.Fatal("runRestore() succeeded with escaping chunk path") + } else if !strings.Contains(err.Error(), "escapes data prefix") { + t.Fatalf("error = %v, want escapes data prefix", err) } - return path } diff --git a/internal/flow/hash.go b/internal/flow/hash.go new file mode 100644 index 0000000..edb1e7e --- /dev/null +++ b/internal/flow/hash.go @@ -0,0 +1,11 @@ +package flow + +import ( + "crypto/sha256" + "encoding/hex" +) + +func hashBytes(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} diff --git a/internal/flow/init.go b/internal/flow/init.go new file mode 100644 index 0000000..786ed89 --- /dev/null +++ b/internal/flow/init.go @@ -0,0 +1,44 @@ +package flow + +import ( + "github.com/pterm/pterm" + + "gloak/internal/repo" + "gloak/internal/storage" +) + +// RunInit creates a new gloak repository on remote and prints the outcome. +// Zero min/avg/max select the package defaults; sizes are in bytes. With force +// it overwrites an existing repository, which makes old snapshots unreadable. +// Validation, encryption, and storage errors from repo.Init are returned. +func RunInit(recipients []string, remote string, min, avg, max int64, force bool) error { + if min == 0 { + min = repo.DefaultChunkMin + } + if avg == 0 { + avg = repo.DefaultChunkAvg + } + if max == 0 { + max = repo.DefaultChunkMax + } + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Initializing repository...") + + master, cfg, err := runInit(recipients, backend, min, avg, max, force) + if err != nil { + _ = spinner.Stop() + return err + } + spinner.Success("Repository initialized") + pterm.Printf("Location: %s\n", remote) + pterm.Printf("Recipients: %v\n", cfg.Recipients) + pterm.Printf("Chunk bounds: min=%d avg=%d max=%d bytes\n", cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) + pterm.Printf("Master key length: %d bytes (wrapped in %s)\n", len(master), repo.KeysPrefix+"/") + return nil +} + +// runInit delegates to repo.Init and returns its master key, config, and error. +// Chunk bounds are byte counts and are passed through without defaults. +func runInit(recipients []string, backend storage.Backend, min, avg, max int64, force bool) ([]byte, repo.Config, error) { + return repo.Init(backend, recipients, min, avg, max, force) +} diff --git a/internal/flow/init_test.go b/internal/flow/init_test.go new file mode 100644 index 0000000..675e86d --- /dev/null +++ b/internal/flow/init_test.go @@ -0,0 +1,91 @@ +package flow + +import ( + "bytes" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func TestRunInitCreatesRepository(t *testing.T) { + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + _ = fileContent + backend := memtest.NewMemoryBackend() + + master, cfg, err := runInit([]string{pub}, backend, 1, 8, 64, false) + if err != nil { + t.Fatalf("runInit() error: %v", err) + } + if len(master) != 32 { + t.Fatalf("master key length = %d", len(master)) + } + if !backend.Has(repo.ConfigPath) { + t.Fatal("repo.json missing") + } + if !backend.Has(repo.KeysPathFor(pub)) { + t.Fatal("wrapped key missing") + } + if cfg.ChunkMin != 1 || cfg.ChunkAvg != 8 || cfg.ChunkMax != 64 { + t.Fatalf("cfg chunk params = %d/%d/%d", cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) + } +} + +func TestRunInitRejectsNoRecipients(t *testing.T) { + if _, _, err := runInit(nil, memtest.NewMemoryBackend(), 1, 8, 64, false); err == nil { + t.Fatal("runInit() succeeded with no recipients") + } else if !strings.Contains(err.Error(), "recipient") { + t.Fatalf("runInit() error = %v, want recipient mention", err) + } +} + +func TestRunInitRejectsInvalidRecipient(t *testing.T) { + if _, _, err := runInit([]string{"age1bad"}, memtest.NewMemoryBackend(), 1, 8, 64, false); err == nil { + t.Fatal("runInit() succeeded with invalid recipient") + } +} + +func TestRunInitRejectsExistingRepository(t *testing.T) { + pub, _, _, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + backend := memtest.NewMemoryBackend() + + if _, _, err := runInit([]string{pub}, backend, 1, 8, 64, false); err != nil { + t.Fatalf("runInit() error: %v", err) + } + if _, _, err := runInit([]string{pub}, backend, 1, 8, 64, false); err == nil { + t.Fatal("runInit() succeeded on existing repository without force") + } else if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("runInit() error = %v, want already exists mention", err) + } +} + +func TestRunInitForceOverwrites(t *testing.T) { + pub, _, _, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + backend := memtest.NewMemoryBackend() + + first, _, err := runInit([]string{pub}, backend, 1, 8, 64, false) + if err != nil { + t.Fatalf("runInit() error: %v", err) + } + second, _, err := runInit([]string{pub}, backend, 1, 8, 64, true) + if err != nil { + t.Fatalf("runInit(force) error: %v", err) + } + if bytes.Equal(first, second) { + t.Fatal("runInit(force) kept the old master key") + } + if !backend.Has(repo.ConfigPath) { + t.Fatal("repo.json missing after force init") + } +} diff --git a/internal/flow/listerror_test.go b/internal/flow/listerror_test.go new file mode 100644 index 0000000..4d01e13 --- /dev/null +++ b/internal/flow/listerror_test.go @@ -0,0 +1,145 @@ +package flow + +import ( + "fmt" + "os/exec" + "strings" + "testing" + + "gloak/internal/repo" + "gloak/internal/storage" +) + +// listFailBackend wraps a backend and fails listings for selected prefixes, +// so error handling in check, snapshots, and prune can be exercised. +type listFailBackend struct { + storage.Backend + filesErr map[string]error + recurErr map[string]error +} + +func (b *listFailBackend) ListFiles(prefix []string) ([]byte, error) { + if err, ok := b.filesErr[strings.Join(prefix, "/")]; ok { + return nil, err + } + return b.Backend.ListFiles(prefix) +} + +func (b *listFailBackend) ListRecursive(prefix []string) ([]string, error) { + if err, ok := b.recurErr[strings.Join(prefix, "/")]; ok { + return nil, err + } + return b.Backend.ListRecursive(prefix) +} + +// missingPathErr builds the wrapped exit-code-3 error rclone produces for a +// directory that does not exist. +func missingPathErr(t *testing.T) error { + t.Helper() + err := exec.Command("sh", "-c", "exit 3").Run() + if err == nil { + t.Fatal("sh exited 0, want exit 3") + } + return fmt.Errorf("failed to list via rclone lsjson: %w", err) +} + +func TestRunCheckTreatsMissingSnapshotsDirAsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: missingPathErr(t)}, + } + if _, err := runCheck(ident, wrapped); err != nil { + t.Fatalf("runCheck() on fresh repo error: %v", err) + } +} + +func TestRunCheckPropagatesSnapshotListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runCheck(ident, wrapped) + if err == nil { + t.Fatal("runCheck() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runCheck() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunCheckPropagatesDataListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + recurErr: map[string]error{repo.DataPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runCheck(ident, wrapped) + if err == nil { + t.Fatal("runCheck() succeeded despite a data listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runCheck() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunSnapshotsPropagatesListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runSnapshots(ident, wrapped) + if err == nil { + t.Fatal("runSnapshots() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runSnapshots() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunSnapshotsTreatsMissingDirAsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: missingPathErr(t)}, + } + infos, err := runSnapshots(ident, wrapped) + if err != nil { + t.Fatalf("runSnapshots() on fresh repo error: %v", err) + } + if len(infos) != 0 { + t.Fatalf("runSnapshots() = %v, want empty", infos) + } +} + +func TestRunPrunePropagatesSnapshotListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + filesErr: map[string]error{repo.SnapshotsPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, false) + if err == nil { + t.Fatal("runPrune() succeeded despite a listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runPrune() error = %v, want it to wrap the listing failure", err) + } +} + +func TestRunPrunePropagatesDataListError(t *testing.T) { + ident, backend, _ := repoFixture(t) + wrapped := &listFailBackend{ + Backend: backend, + recurErr: map[string]error{repo.DataPrefix: fmt.Errorf("remote exploded")}, + } + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, false) + if err == nil { + t.Fatal("runPrune() succeeded despite a data listing failure") + } + if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("runPrune() error = %v, want it to wrap the listing failure", err) + } +} diff --git a/internal/flow/lock.go b/internal/flow/lock.go new file mode 100644 index 0000000..b569a1a --- /dev/null +++ b/internal/flow/lock.go @@ -0,0 +1,96 @@ +package flow + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "time" + + "gloak/internal/core" + "gloak/internal/storage" +) + +// Repository locks coordinate separate gloak processes through files under +// locks/ in the repository. Backups take a shared lock (several backups may +// run at once); prune takes an exclusive lock. The check-then-create +// sequence is not atomic on every backend, so the lock is advisory: it +// stops accidental overlap between backup and prune, not a determined race. + +const lockPrefix = "locks" + +// lockStaleAfter is how long a lock file may exist before other operations +// ignore it. A run that holds its lock longer than this can be overlapped. +const lockStaleAfter = 24 * time.Hour + +type repoLock struct { + Op string `json:"op"` + Hostname string `json:"hostname,omitempty"` + PID int `json:"pid,omitempty"` + CreatedAt string `json:"created_at"` +} + +// acquireRepoLock takes a lock on the repository. A shared acquire (backup) +// fails only while a live non-backup lock exists; an exclusive acquire +// (prune) fails while any live lock exists. Stale locks from processes that +// never released them are removed. The returned function releases the lock +// and is safe to call once. +func acquireRepoLock(backend storage.Backend, op string, shared bool) (func(), error) { + raw, err := backend.ListFiles([]string{lockPrefix}) + if err != nil { + if !storage.IsMissingPath(err) { + return nil, fmt.Errorf("failed to list repository locks: %w", err) + } + raw = []byte("[]") + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse lock listing: %w", err) + } + + now := time.Now().UTC() + for _, entry := range entries { + if entry.IsDir { + continue + } + name := lockPrefix + "/" + entry.Path + var buf bytes.Buffer + if err := backend.Download(name, &buf); err != nil { + return nil, fmt.Errorf("failed to read lock %s: %w", name, err) + } + var held repoLock + if err := json.Unmarshal(buf.Bytes(), &held); err != nil { + return nil, fmt.Errorf("unreadable lock file %s", name) + } + created, err := time.Parse(time.RFC3339, held.CreatedAt) + if err != nil { + return nil, fmt.Errorf("unreadable lock file %s", name) + } + if now.Sub(created) < lockStaleAfter { + if !shared || held.Op != "backup" { + return nil, fmt.Errorf("repository is busy: %s lock held by %s since %s", + held.Op, held.Hostname, held.CreatedAt) + } + continue + } + // The lock is stale; its holder never released it. Clearing it + // may race with another process doing the same, which is harmless. + _ = backend.Delete(name) + } + + host, _ := os.Hostname() + payload, err := json.Marshal(repoLock{ + Op: op, + Hostname: host, + PID: os.Getpid(), + CreatedAt: now.Format(time.RFC3339), + }) + if err != nil { + return nil, fmt.Errorf("failed to marshal lock: %w", err) + } + name := fmt.Sprintf("%s/%s-%s.json", lockPrefix, op, core.GenerateUUID()) + if err := backend.Upload(name, bytes.NewReader(payload)); err != nil { + return nil, fmt.Errorf("failed to create %s lock: %w", op, err) + } + return func() { _ = backend.Delete(name) }, nil +} diff --git a/internal/flow/lock_test.go b/internal/flow/lock_test.go new file mode 100644 index 0000000..9a54b51 --- /dev/null +++ b/internal/flow/lock_test.go @@ -0,0 +1,138 @@ +package flow + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "gloak/internal/storage/memtest" +) + +func putLock(t *testing.T, backend *memtest.MemoryBackend, name string, lk repoLock) { + t.Helper() + raw, err := json.Marshal(lk) + if err != nil { + t.Fatal(err) + } + backend.Put("locks/"+name, raw) +} + +func freshLock(op string) repoLock { + return repoLock{ + Op: op, + Hostname: "test-host", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } +} + +func TestAcquireRepoLockSharedAllowsOtherBackups(t *testing.T) { + backend := memtest.NewMemoryBackend() + release1, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("first shared acquire: %v", err) + } + release2, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("second shared acquire: %v", err) + } + release2() + release1() + if got := len(backend.KeysWithPrefix("locks/")); got != 0 { + t.Fatalf("locks left behind: %d", got) + } +} + +func TestAcquireRepoLockExclusiveBlocksShared(t *testing.T) { + backend := memtest.NewMemoryBackend() + release, err := acquireRepoLock(backend, "prune", false) + if err != nil { + t.Fatalf("exclusive acquire: %v", err) + } + defer release() + + if _, err := acquireRepoLock(backend, "backup", true); err == nil { + t.Fatal("shared acquire succeeded while an exclusive lock is held") + } else if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestAcquireRepoLockSharedBlocksExclusive(t *testing.T) { + backend := memtest.NewMemoryBackend() + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("shared acquire: %v", err) + } + defer release() + + if _, err := acquireRepoLock(backend, "prune", false); err == nil { + t.Fatal("exclusive acquire succeeded while a backup lock is held") + } +} + +func TestAcquireRepoLockBreaksStaleLock(t *testing.T) { + backend := memtest.NewMemoryBackend() + stale := freshLock("prune") + stale.CreatedAt = time.Now().UTC().Add(-lockStaleAfter - time.Hour).Format(time.RFC3339) + putLock(t, backend, "prune-stale.json", stale) + + release, err := acquireRepoLock(backend, "backup", true) + if err != nil { + t.Fatalf("shared acquire with stale lock present: %v", err) + } + release() + keys := backend.KeysWithPrefix("locks/") + if len(keys) != 0 { + t.Fatalf("stale lock not cleaned up: %v", keys) + } +} + +func TestRunPruneApplyBlockedByBackupLock(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "backup-live.json", freshLock("backup")) + + _, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, true) + if err == nil { + t.Fatal("runPrune(apply) succeeded while a backup lock is held") + } + if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestRunBackupPathBlockedByPruneLock(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "prune-live.json", freshLock("prune")) + + filePath := writeTempFile(t, []byte("locked-out")) + _, err := runBackupPath(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e997", nil) + if err == nil { + t.Fatal("runBackupPath() succeeded while a prune lock is held") + } + if !strings.Contains(err.Error(), "busy") { + t.Fatalf("error = %v, want it to report a busy repository", err) + } +} + +func TestRunPruneDryRunIgnoresLocks(t *testing.T) { + ident, backend, _ := repoFixture(t) + putLock(t, backend, "prune-live.json", freshLock("prune")) + + if _, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, false); err != nil { + t.Fatalf("dry-run runPrune() error: %v", err) + } +} + +func TestAcquireRepoLockRejectsUnreadableLock(t *testing.T) { + backend := memtest.NewMemoryBackend() + backend.Put("locks/broken.json", []byte("{not json")) + _, err := acquireRepoLock(backend, "backup", true) + if err == nil { + t.Fatal("acquireRepoLock() succeeded on an unreadable lock file") + } + if !strings.Contains(fmt.Sprint(err), "unreadable") { + t.Fatalf("error = %v, want it to mention the unreadable lock", err) + } +} diff --git a/internal/flow/manifest_read.go b/internal/flow/manifest_read.go new file mode 100644 index 0000000..4f7f1c1 --- /dev/null +++ b/internal/flow/manifest_read.go @@ -0,0 +1,126 @@ +package flow + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" + + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// readManifest downloads the snapshot for snapshotID and decrypts it with the +// identity that opened the repository. +func readManifest(backend storage.Backend, identityPath, snapshotID string) (core.Manifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return core.Manifest{}, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return core.Manifest{}, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + reader, err := crypto.DecryptReader(identityPath, &buf) + if err != nil { + return core.Manifest{}, fmt.Errorf("failed to decrypt snapshot: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return core.Manifest{}, fmt.Errorf("failed to read snapshot: %w", err) + } + var manifest core.Manifest + if err := json.Unmarshal(plain, &manifest); err != nil { + return core.Manifest{}, fmt.Errorf("failed to parse snapshot: %w", err) + } + return manifest, nil +} + +// readTreeManifest downloads the tree snapshot for snapshotID and decrypts +// it with the identity that opened the repository. +func readTreeManifest(backend storage.Backend, identityPath, snapshotID string) (core.TreeManifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return core.TreeManifest{}, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return core.TreeManifest{}, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + plain, err := decryptSnapshotBytes(identityPath, buf.Bytes()) + if err != nil { + return core.TreeManifest{}, err + } + var manifest core.TreeManifest + if err := json.Unmarshal(plain, &manifest); err != nil { + return core.TreeManifest{}, fmt.Errorf("failed to parse tree snapshot: %w", err) + } + return manifest, nil +} + +// readAnyManifest decrypts a snapshot and returns its kind. On success, kind is +// core.KindFile or core.KindTree and exactly one manifest pointer is non-nil. +// An absent or empty kind means file. Invalid snapshot IDs, unknown kinds, +// download, decryption, and JSON errors are returned. +func readAnyManifest(backend storage.Backend, identityPath, snapshotID string) (string, *core.Manifest, *core.TreeManifest, error) { + path, err := repo.SnapshotPath(snapshotID) + if err != nil { + return "", nil, nil, err + } + var buf bytes.Buffer + if err := backend.Download(path, &buf); err != nil { + return "", nil, nil, fmt.Errorf("snapshot %s not found: %w", snapshotID, err) + } + plain, err := decryptSnapshotBytes(identityPath, buf.Bytes()) + if err != nil { + return "", nil, nil, err + } + var probe struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(plain, &probe); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse snapshot: %w", err) + } + switch probe.Kind { + case core.KindTree: + var m core.TreeManifest + if err := json.Unmarshal(plain, &m); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse tree snapshot: %w", err) + } + return core.KindTree, nil, &m, nil + case "", core.KindFile: + var m core.Manifest + if err := json.Unmarshal(plain, &m); err != nil { + return "", nil, nil, fmt.Errorf("failed to parse file snapshot: %w", err) + } + return core.KindFile, &m, nil, nil + default: + return "", nil, nil, fmt.Errorf("unknown snapshot kind %q", probe.Kind) + } +} + +// decryptSnapshotBytes decrypts sealed age data using the identity file at +// identityPath. It returns identity loading, decryption, and plaintext read errors. +func decryptSnapshotBytes(identityPath string, sealed []byte) ([]byte, error) { + reader, err := crypto.DecryptReader(identityPath, bytes.NewReader(sealed)) + if err != nil { + return nil, fmt.Errorf("failed to decrypt snapshot: %w", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot: %w", err) + } + return plain, nil +} + +// validateChunkPath requires the data/ prefix and rejects any occurrence of +// "..". It performs no filesystem lookup or further path validation. +func validateChunkPath(path string) error { + if !strings.HasPrefix(path, repo.DataPrefix+"/") || strings.Contains(path, "..") { + return fmt.Errorf("chunk path escapes data prefix: %s", path) + } + return nil +} diff --git a/internal/flow/prune.go b/internal/flow/prune.go new file mode 100644 index 0000000..e744abe --- /dev/null +++ b/internal/flow/prune.go @@ -0,0 +1,257 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "gloak/internal/core" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// PruneResult summarizes what prune would do or did. +type PruneResult struct { + KeptSnapshots []string + DeadSnapshots []string + LiveChunks int + GarbageChunks int + DeletedSnapshots int + DeletedObjects int + CompactedEpochs bool +} + +// RunPrune opens the repository and runs the prune flow. Without apply, +// nothing is deleted; the result describes the plan only. It returns an error +// for an all-zero policy or a failed repository operation. An apply error may +// follow partial deletions, reflected in the returned counts. +func RunPrune(identityPath, remote string, policy RetentionPolicy, apply bool) (PruneResult, error) { + backend := storage.NewRcloneBackend(remote) + return runPrune(identityPath, backend, policy, apply) +} + +// runPrune plans retention and garbage collection, deleting dead snapshots and +// unreferenced chunks and compacting the index only when apply is true. +// An all-zero policy or unreadable snapshot returns an error. Data listing +// failures are treated as no additional objects; invalid snapshot timestamps +// use the zero time. Other repository, manifest, and deletion errors return +// the results accumulated so far, without undoing earlier deletions. +func runPrune(identityPath string, backend storage.Backend, policy RetentionPolicy, apply bool) (PruneResult, error) { + var res PruneResult + + if policy.KeepLast < 0 || policy.KeepDaily < 0 || policy.KeepWeekly < 0 || policy.KeepMonthly < 0 { + return res, fmt.Errorf("retention counts must be non-negative") + } + if policy.isEmpty() { + return res, fmt.Errorf("refusing to prune with no retention rules (pass at least one --keep-* flag)") + } + + if _, err := repo.Open(backend, identityPath); err != nil { + return res, err + } + if apply { + // Advisory exclusive lock: no backup (shared) may run while + // prune deletes, and no other prune may overlap. + release, err := acquireRepoLock(backend, "prune", false) + if err != nil { + return res, err + } + defer release() + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return res, fmt.Errorf("failed to load index: %w", err) + } + + // List snapshots. + snapRaw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list snapshots: %w", err) + } + // A fresh repo has no snapshots/ directory yet. + snapRaw = []byte("[]") + } + snapEntries, err := storage.ParseList(snapRaw) + if err != nil { + return res, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + + var snaps []snapshotMeta + for _, entry := range snapEntries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + if _, err := uuid.Parse(id); err != nil { + continue + } + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) + if err != nil { + // Undecryptable snapshot is an error; do not prune around it. + return res, fmt.Errorf("snapshot %s: %w", id, err) + } + var created time.Time + if kind == core.KindTree { + created, _ = time.Parse(time.RFC3339, treeM.CreatedAt) + } else { + created, _ = time.Parse(time.RFC3339, fileM.CreatedAt) + } + snaps = append(snaps, snapshotMeta{UUID: id, CreatedAt: created}) + } + + keep := selectKeepSnapshots(snaps, policy) + for _, s := range snaps { + if keep[s.UUID] { + res.KeptSnapshots = append(res.KeptSnapshots, s.UUID) + } else { + res.DeadSnapshots = append(res.DeadSnapshots, s.UUID) + } + } + sort.Strings(res.KeptSnapshots) + sort.Strings(res.DeadSnapshots) + + // Mark live chunks from kept snapshots. + live := make(map[string]bool) + for _, s := range snaps { + if !keep[s.UUID] { + continue + } + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, s.UUID) + if err != nil { + return res, fmt.Errorf("snapshot %s: %w", s.UUID, err) + } + if kind == core.KindTree { + for _, te := range treeM.Entries { + for _, meta := range te.Chunks { + live[meta.PlainSHA256] = true + } + } + } else { + for _, meta := range fileM.Chunks { + live[meta.PlainSHA256] = true + } + } + } + res.LiveChunks = len(live) + + // Garbage = index entries not live, plus data objects whose hash is + // not live. + allHashes := ix.AllHashes() + var garbage []string + for _, h := range allHashes { + if !live[h] { + garbage = append(garbage, h) + } + } + // Also scan data/ for objects not in the live set (orphans). + dataPaths, err := backend.ListRecursive([]string{repo.DataPrefix}) + if err != nil { + // A fresh repo has no data/ directory yet; anything else is a + // real listing failure and must not be pruned around. + if !storage.IsMissingPath(err) { + return res, fmt.Errorf("failed to list data objects: %w", err) + } + dataPaths = nil + } + seenGarbage := make(map[string]bool, len(garbage)) + for _, h := range garbage { + seenGarbage[h] = true + } + for _, p := range dataPaths { + h := path.Base(p) + if len(h) != 64 || live[h] || seenGarbage[h] { + continue + } + garbage = append(garbage, h) + seenGarbage[h] = true + } + sort.Strings(garbage) + res.GarbageChunks = len(garbage) + + if !apply { + return res, nil + } + + // Compact the index first: the new epoch drops hashes no kept + // snapshot references. If this fails, nothing has been deleted yet. + if err := compactIndex(backend, identityPath, ix, live); err != nil { + return res, err + } + res.CompactedEpochs = true + + // Delete dead snapshots. Their exclusive chunks are already gone from + // the index but the data objects are still present, so an interrupted + // prune leaves the repository consistent. + for _, id := range res.DeadSnapshots { + sp, err := repo.SnapshotPath(id) + if err != nil { + return res, err + } + if err := backend.Delete(sp); err != nil { + return res, fmt.Errorf("failed to delete snapshot %s: %w", id, err) + } + res.DeletedSnapshots++ + } + + // Delete garbage data objects last, once nothing references them. + for _, h := range garbage { + dp := repo.DataPath(h) + if err := backend.Delete(dp); err != nil { + return res, fmt.Errorf("failed to delete data object %s: %w", h, err) + } + res.DeletedObjects++ + } + + return res, nil +} + +// compactIndex writes a new epoch for known live hashes, if any, then attempts +// to delete prior epochs. Listing failures are treated as an empty listing and +// old-epoch deletion errors are ignored. Repository opening, listing parsing, +// and compaction errors are returned before deleting old epochs. +func compactIndex(backend storage.Backend, identityPath string, ix *index.Index, live map[string]bool) error { + r, err := repo.Open(backend, identityPath) + if err != nil { + return err + } + + // List current epochs before compacting so we can delete the old ones. + oldRaw, err := backend.ListFiles([]string{"index"}) + if err != nil { + if !storage.IsMissingPath(err) { + return fmt.Errorf("failed to list index epochs: %w", err) + } + oldRaw = []byte("[]") + } + oldEntries, err := storage.ParseList(oldRaw) + if err != nil { + return err + } + var oldEpochs []string + for _, e := range oldEntries { + if !e.IsDir && strings.HasSuffix(e.Path, ".age") { + oldEpochs = append(oldEpochs, e.Path) + } + } + + // Write one new epoch with live entries only. The epoch counter + // continues past existing names so the compacted file is new. + if _, err := ix.Compact(backend, r.Config.Recipients, live); err != nil { + return fmt.Errorf("failed to write compacted index: %w", err) + } + + // Delete every epoch that existed before compaction. A failure here + // leaves the old epochs in place, which Load merges harmlessly. + for _, name := range oldEpochs { + if err := backend.Delete("index/" + name); err != nil { + return fmt.Errorf("failed to delete old index epoch %s: %w", name, err) + } + } + return nil +} diff --git a/internal/flow/prune_test.go b/internal/flow/prune_test.go new file mode 100644 index 0000000..500f6d1 --- /dev/null +++ b/internal/flow/prune_test.go @@ -0,0 +1,184 @@ +package flow + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gloak/internal/storage" +) + +func TestSelectKeepSnapshotsLastOnly(t *testing.T) { + base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + snaps := []snapshotMeta{ + {UUID: "a", CreatedAt: base.Add(-72 * time.Hour)}, + {UUID: "b", CreatedAt: base.Add(-48 * time.Hour)}, + {UUID: "c", CreatedAt: base.Add(-24 * time.Hour)}, + {UUID: "d", CreatedAt: base}, + } + keep := selectKeepSnapshots(snaps, RetentionPolicy{KeepLast: 2}) + if len(keep) != 2 || !keep["d"] || !keep["c"] { + t.Fatalf("keep = %v", keep) + } +} + +func TestSelectKeepSnapshotsDaily(t *testing.T) { + // Three snapshots on same day, one older day, one older still. + day := time.Date(2026, 10, 10, 8, 0, 0, 0, time.UTC) + snaps := []snapshotMeta{ + {UUID: "old", CreatedAt: day.Add(-48 * time.Hour)}, + {UUID: "mid", CreatedAt: day.Add(-24 * time.Hour)}, + {UUID: "m1", CreatedAt: day}, + {UUID: "m2", CreatedAt: day.Add(2 * time.Hour)}, + } + keep := selectKeepSnapshots(snaps, RetentionPolicy{KeepDaily: 2}) + // Newest of day1 (m2) and newest of day0 (mid). + if !keep["m2"] || !keep["mid"] { + t.Fatalf("keep = %v", keep) + } + if keep["m1"] || keep["old"] { + t.Fatalf("unexpected keep = %v", keep) + } +} + +func TestSelectKeepSnapshotsEmptyPolicyKeepsNothing(t *testing.T) { + snaps := []snapshotMeta{{UUID: "a", CreatedAt: time.Now()}} + keep := selectKeepSnapshots(snaps, RetentionPolicy{}) + if len(keep) != 0 { + t.Fatalf("keep = %v", keep) + } +} + +func TestRunPruneRejectsEmptyPolicy(t *testing.T) { + ident, backend, _ := repoFixture(t) + _, err := runPrune(ident, backend, RetentionPolicy{}, false) + if err == nil { + t.Fatal("runPrune() succeeded with empty policy") + } +} + +func TestRunPruneRejectsNegativePolicy(t *testing.T) { + ident, backend, _ := repoFixture(t) + for _, policy := range []RetentionPolicy{ + {KeepLast: -1}, + {KeepDaily: -1}, + {KeepWeekly: -1}, + {KeepMonthly: -1, KeepLast: 1}, + } { + if _, err := runPrune(ident, backend, policy, false); err == nil { + t.Fatalf("runPrune() succeeded with negative policy %+v", policy) + } + } +} + +// failDeleteBackend fails Delete for paths under failPrefix. +type failDeleteBackend struct { + storage.Backend + failPrefix string +} + +func (b *failDeleteBackend) Delete(remotePath string) error { + if strings.HasPrefix(remotePath, b.failPrefix) { + return fmt.Errorf("delete %s failed", remotePath) + } + return b.Backend.Delete(remotePath) +} + +func TestRunPruneCompactsIndexBeforeDeletingAnything(t *testing.T) { + ident, backend, _ := repoFixture(t) + f1 := filepath.Join(t.TempDir(), "one.txt") + if err := os.WriteFile(f1, []byte("content-one-unique"), 0644); err != nil { + t.Fatal(err) + } + f2 := filepath.Join(t.TempDir(), "two.txt") + if err := os.WriteFile(f2, []byte("content-two-different"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackup(f1, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) + if _, err := runBackup(f2, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + + wrapped := &failDeleteBackend{Backend: backend, failPrefix: "index/"} + _, err := runPrune(ident, wrapped, RetentionPolicy{KeepLast: 1}, true) + if err == nil { + t.Fatal("runPrune() succeeded despite index epoch deletion failure") + } + // Compaction runs first: when it fails, no snapshot or object has + // been touched yet. + if !backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") { + t.Fatal("dead snapshot deleted before compaction finished") + } + if err := runVerify("8f91c6c7-7a0b-44aa-ae23-997b60e4e999", ident, backend); err != nil { + t.Fatalf("kept snapshot broken after failed prune: %v", err) + } +} + +func TestRunPruneDryRunChangesNothing(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("keep-me")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + before := backend.TotalUploads("") + snapBefore := backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") + + res, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, false) + if err != nil { + t.Fatalf("runPrune() error: %v", err) + } + if res.DeletedSnapshots != 0 || res.DeletedObjects != 0 { + t.Fatalf("dry-run deleted something: %+v", res) + } + if len(res.KeptSnapshots) != 1 || len(res.DeadSnapshots) != 0 { + t.Fatalf("plan = %+v", res) + } + if backend.TotalUploads("") != before || backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") != snapBefore { + t.Fatal("dry-run mutated backend") + } +} + +func TestRunPruneYesDeletesDead(t *testing.T) { + ident, backend, _ := repoFixture(t) + // Two snapshots with different content so second has unique chunks. + f1 := filepath.Join(t.TempDir(), "one.txt") + if err := os.WriteFile(f1, []byte("content-one-unique"), 0644); err != nil { + t.Fatal(err) + } + f2 := filepath.Join(t.TempDir(), "two.txt") + if err := os.WriteFile(f2, []byte("content-two-different"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackup(f1, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + // Distinct RFC3339 timestamps so keep-last is deterministic. + time.Sleep(1100 * time.Millisecond) + if _, err := runBackup(f2, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + + res, err := runPrune(ident, backend, RetentionPolicy{KeepLast: 1}, true) + if err != nil { + t.Fatalf("runPrune() error: %v", err) + } + if res.DeletedSnapshots != 1 { + t.Fatalf("DeletedSnapshots = %d, want 1", res.DeletedSnapshots) + } + if backend.Has("snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age") { + t.Fatal("dead snapshot still present") + } + if !backend.Has("snapshots/8f91c6c7-7a0b-44aa-ae23-997b60e4e999.age") { + t.Fatal("kept snapshot deleted") + } + // Live snapshot must still verify. + if err := runVerify("8f91c6c7-7a0b-44aa-ae23-997b60e4e999", ident, backend); err != nil { + t.Fatalf("verify after prune: %v", err) + } +} diff --git a/internal/flow/restore.go b/internal/flow/restore.go index 8c7c962..f073f0b 100644 --- a/internal/flow/restore.go +++ b/internal/flow/restore.go @@ -4,133 +4,329 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" - "io" "os" + "path" "path/filepath" + "sort" + "strings" + "time" "github.com/pterm/pterm" "gloak/internal/core" "gloak/internal/crypto" + "gloak/internal/repo" "gloak/internal/storage" ) -func RunRestore(backupID string, identityPath string, remote string, outputDir string) error { - return runRestore(backupID, identityPath, storage.NewRcloneBackend(remote), outputDir) -} - -func runRestore(backupID string, identityPath string, backend storage.Backend, outputDir string) error { - pterm.Info.Printf("Starting restore for UUID: %s\n", backupID) - - pterm.Info.Println("Downloading encrypted manifest...") - var manifestBuf bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/manifest.age", backupID), &manifestBuf) +// RunRestore restores a standalone file backup or repository snapshot to +// outputDir. For trees, pathFilter selects an exact relative path and its +// descendants; empty selects all entries. File backups ignore the filter. +// Access, decryption, integrity, and filesystem errors are returned; existing +// files may be truncated and partial output remains on failure. +func RunRestore(snapshotID, identityPath, remote, outputDir, pathFilter string) error { + backend := storage.NewRcloneBackend(remote) + outputPath, err := runRestore(snapshotID, outputDir, identityPath, backend, pathFilter) if err != nil { - return fmt.Errorf("failed to download manifest.age: %w", err) + return err } + pterm.Success.Println("Restore completed successfully! File saved to:", outputPath) + return nil +} - pterm.Info.Println("Decrypting manifest...") - manifestReader, err := crypto.DecryptReader(identityPath, &manifestBuf) +// runRestore restores a backup and returns the output file path or tree root. +// A downloadable locator selects standalone file mode; any locator download +// failure falls back to the repository flow. pathFilter applies only to trees. +// Repository, manifest, and restore errors are returned without cleaning up output. +func runRestore(snapshotID, outputDir, identityPath string, backend storage.Backend, pathFilter string) (string, error) { + if isFileMode(backend, snapshotID) { + return runFileRestore(snapshotID, identityPath, outputDir, backend) + } + r, err := repo.Open(backend, identityPath) if err != nil { - return fmt.Errorf("failed to decrypt manifest: %w", err) + return "", err } - - manifestBytes, err := io.ReadAll(manifestReader) + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, snapshotID) if err != nil { - return fmt.Errorf("failed to read manifest contents: %w", err) + return "", err } - - var manifest core.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return fmt.Errorf("failed to parse manifest JSON: %w", err) + if kind == core.KindTree { + return runTreeRestore(treeM, r, outputDir, pathFilter, backend) } + return runFileRestoreFromManifest(fileM, r, backend, outputDir) +} - pterm.Success.Printf("Manifest valid! Restoring file: %s (%d bytes)\n", manifest.OriginalName, manifest.OriginalSize) - - if err := os.MkdirAll(outputDir, 0755); err != nil { - return fmt.Errorf("failed to create output directory: %w", err) - } +// runFileRestoreFromManifest writes a repository file snapshot under outputDir +// and returns its path, truncating any existing file. It checks chunk plaintext +// sizes and hashes and the overall payload hash. Invalid names or chunk paths, +// filesystem, download, decryption, integrity, and close errors can leave +// partial output. +func runFileRestoreFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend, outputDir string) (string, error) { if filepath.Base(manifest.OriginalName) != manifest.OriginalName { - return fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) + return "", fmt.Errorf("invalid original filename in manifest: %s", manifest.OriginalName) + } + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) } outputPath := filepath.Join(outputDir, manifest.OriginalName) outFile, err := os.OpenFile(outputPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { - return fmt.Errorf("failed to create output file: %w", err) + return "", fmt.Errorf("failed to create output file: %w", err) } defer outFile.Close() - p, _ := pterm.DefaultProgressbar. - WithTotal(int(manifest.OriginalSize)). - WithTitle("Downloading & Decrypting..."). - WithRemoveWhenDone(false). - Start() - payloadHasher := sha256.New() - for _, chunk := range manifest.Chunks { - var encryptedChunk bytes.Buffer - if err := backend.Download(fmt.Sprintf("%s/%s", backupID, chunk.Name), &encryptedChunk); err != nil { - p.Stop() - return fmt.Errorf("failed to download chunk %s: %w", chunk.Name, err) + for _, meta := range manifest.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return "", err } - - encryptedBytes := encryptedChunk.Bytes() - if int64(len(encryptedBytes)) != chunk.EncryptedSize { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Size mismatch", chunk.Name) + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, buf.Bytes()) + if err != nil { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return "", fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) } - if hashBytes(encryptedBytes) != chunk.EncryptedSHA256 { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Hash mismatch", chunk.Name) + if _, err := outFile.Write(plain); err != nil { + return "", fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) } + payloadHasher.Write(plain) + } - decReader, err := crypto.DecryptReader(identityPath, bytes.NewReader(encryptedBytes)) - if err != nil { - p.Stop() - return fmt.Errorf("failed to decrypt chunk %s: %w", chunk.Name, err) + if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { + return "", fmt.Errorf("fatal: overall payload hash does not match manifest") + } + if err := outFile.Close(); err != nil { + return "", fmt.Errorf("failed to close output file: %w", err) + } + return outputPath, nil +} + +// runTreeRestore restores entries below a root named for the first source +// path, falling back to the snapshot UUID for an empty, . or root basename. +// pathFilter is cleaned and validated as a relative path first, then selects +// an exact slash-separated path and its descendants; empty selects all. +// It returns the destination root even when nothing matches. +// Missing source paths, unsafe entry paths, invalid filters, unknown selected +// entry types, and filesystem or chunk restore errors are returned; partial +// output remains. +func runTreeRestore(m *core.TreeManifest, r *repo.Repo, outputDir, pathFilter string, backend storage.Backend) (string, error) { + pathFilter, err := normalizePathFilter(pathFilter) + if err != nil { + return "", err + } + if len(m.Paths) == 0 { + return "", fmt.Errorf("tree snapshot has no source paths") + } + // The manifest is remote input: fall back to the snapshot UUID when + // the recorded root name is not a single safe path segment, and + // reject the manifest when even the UUID is unsafe. + rootName := filepath.Base(filepath.Clean(m.Paths[0])) + if err := core.ValidateRelPath(rootName); err != nil { + rootName = m.UUID + } + if err := core.ValidateRelPath(rootName); err != nil { + return "", fmt.Errorf("unsafe restore root name: %w", err) + } + if err := os.MkdirAll(outputDir, 0755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + destRoot := filepath.Join(outputDir, rootName) + // Defense in depth: the join above must stay under outputDir. + if rel, err := filepath.Rel(outputDir, destRoot); err != nil || + rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("restore root escapes output directory: %s", rootName) + } + + type dirSetting struct { + path string + mode os.FileMode + mtime time.Time + } + var dirSettings []dirSetting + var matched int + + for _, entry := range m.Entries { + if err := core.ValidateRelPath(entry.RelPath); err != nil { + return "", fmt.Errorf("unsafe path in manifest: %w", err) } - plainBytes, err := io.ReadAll(decReader) - if err != nil { - p.Stop() - return fmt.Errorf("failed to read decrypted chunk %s: %w", chunk.Name, err) + if pathFilter != "" { + if entry.RelPath != pathFilter && !strings.HasPrefix(entry.RelPath, pathFilter+"/") { + continue + } } - if int64(len(plainBytes)) != chunk.PlainSize { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Plain size mismatch", chunk.Name) + matched++ + target := filepath.Join(destRoot, filepath.FromSlash(entry.RelPath)) + // Defense in depth: ensure target stays under destRoot. + if !strings.HasPrefix(target, destRoot+string(filepath.Separator)) && target != destRoot { + return "", fmt.Errorf("restored path escapes output root: %s", entry.RelPath) } - if hashBytes(plainBytes) != chunk.PlainSHA256 { - p.Stop() - return fmt.Errorf("chunk %s corrupted/tampered! Plain hash mismatch", chunk.Name) + // The manifest is remote input: reject targets reached through a + // symlink below the destination root before creating or writing. + if err := rejectSymlinkRestorePath(destRoot, target); err != nil { + return "", err } - if _, err := outFile.Write(plainBytes); err != nil { - p.Stop() - return fmt.Errorf("failed to write restored chunk %s: %w", chunk.Name, err) + switch entry.Type { + case "dir": + // Keep the directory writable while children are restored; + // the recorded mode is applied after the loop. + if err := os.MkdirAll(target, 0700); err != nil { + return "", fmt.Errorf("failed to create directory %s: %w", entry.RelPath, err) + } + if err := os.Chmod(target, 0700); err != nil { + return "", fmt.Errorf("failed to open directory %s for restore: %w", entry.RelPath, err) + } + setting := dirSetting{path: target, mode: os.FileMode(entry.Mode) & 0777} + if mt, err := time.Parse(time.RFC3339, entry.Mtime); err == nil { + setting.mtime = mt + } + dirSettings = append(dirSettings, setting) + case "file": + if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil { + return "", fmt.Errorf("failed to create parent for %s: %w", entry.RelPath, err) + } + if err := restoreTreeFile(entry, r, backend, target); err != nil { + return "", err + } + default: + return "", fmt.Errorf("unknown tree entry type %q for %s", entry.Type, entry.RelPath) } - payloadHasher.Write(plainBytes) - p.Add(len(plainBytes)) + } + if pathFilter != "" && matched == 0 { + return "", fmt.Errorf("path filter %q matches nothing in snapshot %s", pathFilter, m.UUID) } - if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { - p.Stop() - return fmt.Errorf("fatal: overall payload hash does not match manifest") + // Apply directory modes after every child is written, deepest paths + // first so a read-only parent never blocks chmod on its children. + sort.Slice(dirSettings, func(i, j int) bool { + return len(dirSettings[i].path) > len(dirSettings[j].path) + }) + for _, d := range dirSettings { + if err := os.Chmod(d.path, d.mode); err != nil { + return "", fmt.Errorf("failed to set mode on %s: %w", d.path, err) + } + if !d.mtime.IsZero() { + _ = os.Chtimes(d.path, d.mtime, d.mtime) + } } + return destRoot, nil +} - p.Stop() - pterm.Success.Println("Restore completed successfully! File saved to:", outputPath) - return nil +// normalizePathFilter cleans a user-supplied tree restore filter and validates +// it as a relative path. Empty input stays empty and "." selects everything. +// Filters that are unsafe after cleaning are rejected. +func normalizePathFilter(pathFilter string) (string, error) { + if pathFilter == "" { + return "", nil + } + cleaned := path.Clean(filepath.ToSlash(pathFilter)) + if cleaned == "." { + return "", nil + } + if err := core.ValidateRelPath(cleaned); err != nil { + return "", fmt.Errorf("invalid path filter: %w", err) + } + return cleaned, nil } -type RestoreProgressReader struct { - io.Reader - pb *pterm.ProgressbarPrinter +// rejectSymlinkRestorePath returns an error when destRoot or any component of +// target below it already exists as a symlink. A string check on the joined +// path cannot see links planted under the destination, so every component is +// inspected before restore creates or writes through it. Components that do +// not exist yet are left for creation. +func rejectSymlinkRestorePath(destRoot, target string) error { + inspect := func(p string) error { + fi, err := os.Lstat(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("failed to inspect restore path %s: %w", p, err) + } + if fi.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore through symlink: %s", p) + } + return nil + } + if err := inspect(destRoot); err != nil { + return err + } + rel, err := filepath.Rel(destRoot, target) + if err != nil { + return fmt.Errorf("invalid restore path: %w", err) + } + cur := destRoot + for _, part := range strings.Split(rel, string(filepath.Separator)) { + if part == "." || part == "" { + continue + } + cur = filepath.Join(cur, part) + if err := inspect(cur); err != nil { + return err + } + } + return nil } -func (pr *RestoreProgressReader) Read(p []byte) (int, error) { - n, err := pr.Reader.Read(p) - if n > 0 { - pr.pb.Add(n) +// restoreTreeFile creates or truncates target, writes verified plaintext chunks, +// and attempts to restore its modification time. An existing symlink at the +// target is rejected. New files use entry.Mode's permission bits, subject to +// the umask. Path, download, decryption, checksum, file open/write, and close +// errors are returned and can leave a partial file. Invalid timestamps and +// timestamp update failures are ignored. +func restoreTreeFile(entry core.TreeEntry, r *repo.Repo, backend storage.Backend, target string) error { + mode := os.FileMode(entry.Mode) & 0777 + if fi, err := os.Lstat(target); err == nil && fi.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("refusing to restore %s over a symlink", entry.RelPath) + } + outFile, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + // An earlier restore may have left the file read-only; make it + // writable once and retry. + if chmodErr := os.Chmod(target, mode|0200); chmodErr != nil { + return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + } + outFile, err = os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("failed to create output file %s: %w", entry.RelPath, err) + } } - return n, err + defer outFile.Close() + + for _, meta := range entry.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err + } + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, buf.Bytes()) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + if _, err := outFile.Write(plain); err != nil { + return fmt.Errorf("failed to write chunk %d: %w", meta.Index, err) + } + } + if err := outFile.Close(); err != nil { + return fmt.Errorf("failed to close output file %s: %w", entry.RelPath, err) + } + // Re-applying the mode matters when the file already existed (a + // previous restore may have left it read-only or writable). + if err := os.Chmod(target, mode); err != nil { + return fmt.Errorf("failed to set mode on %s: %w", entry.RelPath, err) + } + if mt, err := time.Parse(time.RFC3339, entry.Mtime); err == nil { + _ = os.Chtimes(target, mt, mt) + } + return nil } diff --git a/internal/flow/resume.go b/internal/flow/resume.go deleted file mode 100644 index fdb6cbe..0000000 --- a/internal/flow/resume.go +++ /dev/null @@ -1,110 +0,0 @@ -package flow - -import ( - "bytes" - "fmt" - "os" - "strings" - - "github.com/pterm/pterm" - - "gloak/internal/core" - "gloak/internal/storage" -) - -func RunResumeValidation(backupID string) (UploadState, error) { - return runResumeValidation(backupID, defaultUploadStateDir()) -} - -func RunResume(backupID string) error { - return runResume(backupID, func(remote string) storage.Backend { - return storage.NewRcloneBackend(remote) - }, defaultUploadStateDir()) -} - -func runResume(backupID string, newBackend func(string) storage.Backend, stateDir string) error { - state, err := runResumeValidation(backupID, stateDir) - if err != nil { - return err - } - - file, err := os.Open(state.SourcePath) - if err != nil { - return fmt.Errorf("failed to open source file from upload state: %w", err) - } - defer file.Close() - - pterm.Success.Printf("Found upload state: %s\n", state.BackupID) - pterm.DefaultBasicText.Printf("Source file : %s\n", state.SourcePath) - pterm.DefaultBasicText.Printf("Destination : %s\n", state.Remote) - pterm.DefaultBasicText.Printf("Uploaded chunks : %d\n", len(state.UploadedChunks)) - - backend := newBackend(state.Remote) - p, _ := pterm.DefaultProgressbar. - WithTotal(int(state.OriginalSize)). - WithTitle("Resuming upload..."). - WithRemoveWhenDone(false). - Start() - - chunksMeta, payloadHash, err := uploadPlaintextChunks(file, state.Recipient, backend, backupID, &state, stateDir, state.ChunkSize, func(chunk core.ChunkMeta) { - pterm.Info.Printf("Skipping %s\n", chunk.Name) - }, func(n int) { - p.Add(n) - }) - if err != nil { - p.Stop() - return fmt.Errorf("resume upload failed: %w", err) - } - p.Stop() - - pterm.Info.Println("Building and encrypting manifest...") - manifest := core.CreateManifest(state.BackupID, state.OriginalName, state.OriginalSize, state.Recipient, payloadHash, chunksMeta) - encryptedManifestBytes, err := core.EncryptManifest(state.Recipient, manifest) - if err != nil { - return fmt.Errorf("failed to encrypt manifest: %w", err) - } - if err := backend.Upload(fmt.Sprintf("%s/manifest.age", state.BackupID), bytes.NewReader(encryptedManifestBytes)); err != nil { - return fmt.Errorf("failed to upload manifest.age: %w", err) - } - - locator := core.CreateLocator(state.BackupID) - locatorBytes, _ := core.SerializeLocator(locator) - if err := backend.Upload(fmt.Sprintf("%s/locator.json", state.BackupID), bytes.NewReader(locatorBytes)); err != nil { - return fmt.Errorf("failed to upload locator.json: %w", err) - } - if err := deleteUploadState(stateDir, state.BackupID); err != nil { - return fmt.Errorf("resume completed but failed to delete upload state: %w", err) - } - - pterm.Success.Println("Resume completed successfully!") - return nil -} - -func runResumeValidation(backupID string, stateDir string) (UploadState, error) { - state, err := loadUploadState(stateDir, backupID) - if err != nil { - return UploadState{}, err - } - - info, err := os.Stat(state.SourcePath) - if err != nil { - return UploadState{}, fmt.Errorf("failed to stat source file from upload state: %w", err) - } - if info.IsDir() { - return UploadState{}, fmt.Errorf("source path from upload state is a directory: %s", state.SourcePath) - } - if info.Size() != state.OriginalSize { - return UploadState{}, fmt.Errorf("source file size changed: got %d, want %d", info.Size(), state.OriginalSize) - } - if strings.TrimSpace(state.Recipient) == "" { - return UploadState{}, fmt.Errorf("upload state recipient is empty") - } - if state.Remote == "" { - return UploadState{}, fmt.Errorf("upload state remote is empty") - } - if state.ChunkSize <= 0 { - return UploadState{}, fmt.Errorf("upload state chunk size must be positive") - } - - return state, nil -} diff --git a/internal/flow/resume_test.go b/internal/flow/resume_test.go deleted file mode 100644 index bfb5587..0000000 --- a/internal/flow/resume_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package flow - -import ( - "os" - "path/filepath" - "strings" - "testing" - - "gloak/internal/core" -) - -func TestRunResumeValidationRejectsMissingState(t *testing.T) { - _, err := runResumeValidation("7f91c6c7-7a0b-44aa-ae23-997b60e4e998", t.TempDir()) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with missing state") - } - if !strings.Contains(err.Error(), "failed to read upload state") { - t.Fatalf("runResumeValidation() error = %q, want missing state error", err) - } -} - -func TestRunResumeValidationRejectsMissingSourceFile(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - state := NewUploadState(backupID, filepath.Join(t.TempDir(), "missing.txt"), "missing.txt", 7, "memory:backups", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with missing source file") - } - if !strings.Contains(err.Error(), "failed to stat source file from upload state") { - t.Fatalf("runResumeValidation() error = %q, want source stat error", err) - } -} - -func TestRunResumeValidationRejectsChangedSourceSize(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("current payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 5, "memory:backups", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with changed source size") - } - if !strings.Contains(err.Error(), "source file size changed") { - t.Fatalf("runResumeValidation() error = %q, want source size error", err) - } -} - -func TestRunResumeValidationRejectsEmptyRecipient(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 7, "memory:backups", " ") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - _, err := runResumeValidation(backupID, stateDir) - if err == nil { - t.Fatalf("runResumeValidation() succeeded with empty recipient") - } - if !strings.Contains(err.Error(), "upload state recipient is empty") { - t.Fatalf("runResumeValidation() error = %q, want empty recipient error", err) - } -} - -func TestRunResumeValidationAcceptsValidState(t *testing.T) { - stateDir := t.TempDir() - backupID := "7f91c6c7-7a0b-44aa-ae23-997b60e4e998" - sourcePath := writeResumeSource(t, []byte("payload")) - state := NewUploadState(backupID, sourcePath, filepath.Base(sourcePath), 7, "memory:backups", "age1recipient") - state.UploadedChunks = []core.ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 7, PlainSHA256: hashBytes([]byte("payload")), EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - got, err := runResumeValidation(backupID, stateDir) - if err != nil { - t.Fatalf("runResumeValidation() returned error: %v", err) - } - if got.BackupID != backupID || got.SourcePath != sourcePath || got.Remote != "memory:backups" { - t.Fatalf("validated state = %+v, want backup/source/remote preserved", got) - } - if len(got.UploadedChunks) != 1 || got.UploadedChunks[0].Name != "chunk_00000" { - t.Fatalf("validated uploaded chunks = %+v, want chunk_00000", got.UploadedChunks) - } -} - -func writeResumeSource(t *testing.T, content []byte) string { - t.Helper() - path := filepath.Join(t.TempDir(), "source.txt") - if err := os.WriteFile(path, content, 0600); err != nil { - t.Fatalf("failed to write source file: %v", err) - } - return path -} diff --git a/internal/flow/retention.go b/internal/flow/retention.go new file mode 100644 index 0000000..225669c --- /dev/null +++ b/internal/flow/retention.go @@ -0,0 +1,106 @@ +package flow + +import ( + "fmt" + "sort" + "time" +) + +// RetentionPolicy mirrors restic's keep rules. Zero means the rule does +// not apply. +type RetentionPolicy struct { + KeepLast int + KeepDaily int + KeepWeekly int + KeepMonthly int +} + +// isEmpty reports whether all retention counts are zero. +func (p RetentionPolicy) isEmpty() bool { + return p.KeepLast == 0 && p.KeepDaily == 0 && p.KeepWeekly == 0 && p.KeepMonthly == 0 +} + +// snapshotMeta is the subset of snapshot data retention needs. +type snapshotMeta struct { + UUID string + CreatedAt time.Time +} + +// selectKeepSnapshots returns the UUIDs that survive the retention policy. +// Snapshots are processed newest first. A snapshot kept by any rule stays. +// Daily, weekly, and monthly rules keep the newest snapshot in each of the +// latest occupied UTC days, ISO weeks, or months; nonpositive counts keep none. +// The input is sorted in place, with equal timestamps ordered by UUID. +func selectKeepSnapshots(snaps []snapshotMeta, p RetentionPolicy) map[string]bool { + keep := make(map[string]bool) + + sort.Slice(snaps, func(i, j int) bool { + if snaps[i].CreatedAt.Equal(snaps[j].CreatedAt) { + return snaps[i].UUID < snaps[j].UUID + } + return snaps[i].CreatedAt.After(snaps[j].CreatedAt) + }) + + // keep-last: the N newest. + for i := 0; i < p.KeepLast && i < len(snaps); i++ { + keep[snaps[i].UUID] = true + } + + // keep-daily: newest snapshot of each of the last N calendar days that + // have at least one snapshot. + if p.KeepDaily > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + day := s.CreatedAt.UTC().Format("2006-01-02") + if seen[day] { + continue + } + seen[day] = true + keep[s.UUID] = true + count++ + if count >= p.KeepDaily { + break + } + } + } + + // keep-weekly: newest snapshot of each of the last N ISO weeks. + if p.KeepWeekly > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + year, week := s.CreatedAt.UTC().ISOWeek() + key := fmt.Sprintf("%04d-W%02d", year, week) + if seen[key] { + continue + } + seen[key] = true + keep[s.UUID] = true + count++ + if count >= p.KeepWeekly { + break + } + } + } + + // keep-monthly: newest snapshot of each of the last N calendar months. + if p.KeepMonthly > 0 { + seen := make(map[string]bool) + count := 0 + for _, s := range snaps { + key := s.CreatedAt.UTC().Format("2006-01") + if seen[key] { + continue + } + seen[key] = true + keep[s.UUID] = true + count++ + if count >= p.KeepMonthly { + break + } + } + } + + return keep +} diff --git a/internal/flow/snapshots.go b/internal/flow/snapshots.go new file mode 100644 index 0000000..fc544b9 --- /dev/null +++ b/internal/flow/snapshots.go @@ -0,0 +1,107 @@ +package flow + +import ( + "fmt" + "path" + "sort" + "strings" + + "github.com/pterm/pterm" + + "gloak/internal/core" + "gloak/internal/repo" + "gloak/internal/storage" +) + +type SnapshotInfo struct { + UUID string + Kind string + CreatedAt string + Hostname string + Detail string +} + +// RunSnapshots prints the repository's snapshot summaries, or a message when +// the list is empty. It returns repository or snapshot loading errors and +// ignores table rendering errors. +func RunSnapshots(identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + infos, err := runSnapshots(identityPath, backend) + if err != nil { + return err + } + if len(infos) == 0 { + pterm.Info.Println("No snapshots found.") + return nil + } + table := pterm.DefaultTable. + WithHasHeader(). + WithData(append([][]string{{"UUID", "KIND", "CREATED", "HOST", "DETAIL"}}, snapshotRows(infos)...)) + _ = table.Render() + return nil +} + +// snapshotRows returns table rows in input order, with UUID, kind, creation +// time, hostname, and detail columns, without a header. +func snapshotRows(infos []SnapshotInfo) [][]string { + rows := make([][]string, 0, len(infos)) + for _, info := range infos { + rows = append(rows, []string{info.UUID, info.Kind, info.CreatedAt, info.Hostname, info.Detail}) + } + return rows +} + +// runSnapshots loads file and tree snapshot summaries, sorted by descending +// creation-time text and then ascending UUID. Snapshot listing failures yield +// an empty result; repository opening, listing parsing, and manifest errors +// are returned. +func runSnapshots(identityPath string, backend storage.Backend) ([]SnapshotInfo, error) { + if _, err := repo.Open(backend, identityPath); err != nil { + return nil, err + } + raw, err := backend.ListFiles([]string{repo.SnapshotsPrefix}) + if err != nil { + // Fresh repo with no snapshots directory; anything else is a + // real listing failure and must surface. + if storage.IsMissingPath(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to list snapshots: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse snapshot listing: %w", err) + } + var infos []SnapshotInfo + for _, entry := range entries { + if entry.IsDir || !strings.HasSuffix(entry.Path, ".age") { + continue + } + id := strings.TrimSuffix(path.Base(entry.Path), ".age") + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, id) + if err != nil { + return nil, fmt.Errorf("snapshot %s: %w", id, err) + } + info := SnapshotInfo{UUID: id, Kind: kind} + if kind == core.KindTree { + info.CreatedAt = treeM.CreatedAt + info.Hostname = treeM.Hostname + detail := fmt.Sprintf("%d files", treeM.FileCount()) + if len(treeM.Paths) > 0 { + detail += " from " + path.Base(strings.TrimRight(treeM.Paths[0], "/")) + } + info.Detail = detail + } else { + info.CreatedAt = fileM.CreatedAt + info.Detail = fileM.OriginalName + } + infos = append(infos, info) + } + sort.Slice(infos, func(i, j int) bool { + if infos[i].CreatedAt == infos[j].CreatedAt { + return infos[i].UUID < infos[j].UUID + } + return infos[i].CreatedAt > infos[j].CreatedAt + }) + return infos, nil +} diff --git a/internal/flow/snapshots_test.go b/internal/flow/snapshots_test.go new file mode 100644 index 0000000..6a4d8f4 --- /dev/null +++ b/internal/flow/snapshots_test.go @@ -0,0 +1,47 @@ +package flow + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRunSnapshotsListsBothKinds(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e998", nil); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "t.txt"), []byte("tree"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackupPath(root, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + infos, err := runSnapshots(ident, backend) + if err != nil { + t.Fatal(err) + } + if len(infos) != 2 { + t.Fatalf("len = %d, want 2", len(infos)) + } + kinds := map[string]bool{} + for _, info := range infos { + kinds[info.Kind] = true + } + if !kinds["file"] || !kinds["tree"] { + t.Fatalf("kinds = %v", kinds) + } +} + +func TestRunSnapshotsEmpty(t *testing.T) { + ident, backend, _ := repoFixture(t) + infos, err := runSnapshots(ident, backend) + if err != nil { + t.Fatal(err) + } + if len(infos) != 0 { + t.Fatalf("len = %d", len(infos)) + } +} diff --git a/internal/flow/tree_backup.go b/internal/flow/tree_backup.go new file mode 100644 index 0000000..42b0c98 --- /dev/null +++ b/internal/flow/tree_backup.go @@ -0,0 +1,200 @@ +package flow + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "time" + + "gloak/internal/chunker" + "gloak/internal/core" + "gloak/internal/crypto" + "gloak/internal/index" + "gloak/internal/repo" + "gloak/internal/storage" +) + +// runTreeBackup records directories and regular files in a tree snapshot, +// skipping symlinks and special files and reusing indexed chunks. A non-nil +// progress callback receives the cumulative processed chunk count. It returns +// statistics on success, or a zero result for path, filesystem, repository, +// encryption, or storage errors. Uploaded chunks and index checkpoints remain +// on failure; a failed hostname lookup leaves the hostname empty. +func runTreeBackup(dirPath, identityPath string, backend storage.Backend, snapshotID string, progress func(int)) (BackupResult, error) { + r, err := repo.Open(backend, identityPath) + if err != nil { + return BackupResult{}, err + } + ix, err := index.Load(backend, identityPath) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to load index: %w", err) + } + cdc, err := chunker.NewCDC(r.Config.ChunkMin, r.Config.ChunkAvg, r.Config.ChunkMax) + if err != nil { + return BackupResult{}, fmt.Errorf("invalid chunk config: %w", err) + } + + abs, err := filepath.Abs(dirPath) + if err != nil { + return BackupResult{}, err + } + // Resolve symlinks before the walk: walking a symlinked root never + // matches the root itself, and every relative path would start with + // ../, producing a silently empty snapshot. The manifest keeps the + // path the user passed so restore still uses the familiar name. + resolved, err := filepath.EvalSymlinks(abs) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to resolve %s: %w", abs, err) + } + + var entries []core.TreeEntry + var sinceCheckpoint int + res := BackupResult{SnapshotID: snapshotID} + + walkErr := filepath.WalkDir(resolved, func(p string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if p == resolved { + return nil + } + rel, err := filepath.Rel(resolved, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + info, err := d.Info() + if err != nil { + return err + } + mode := uint32(info.Mode().Perm()) + mtime := info.ModTime().UTC().Format(time.RFC3339) + + switch { + case d.IsDir(): + entries = append(entries, core.TreeEntry{ + RelPath: rel, + Type: "dir", + Mode: mode, + Mtime: mtime, + }) + return nil + case !info.Mode().IsRegular(): + // Symlinks and special files are skipped. + return nil + } + + file, err := os.Open(p) + if err != nil { + return err + } + defer file.Close() + + var chunks []core.ChunkMeta + payloadHasher := sha256.New() + splitErr := cdc.Split(file, func(chunk []byte) error { + payloadHasher.Write(chunk) + hash := hashBytes(chunk) + res.TotalChunks++ + res.TotalPlainBytes += int64(len(chunk)) + dataPath := repo.DataPath(hash) + + if ix.Has(hash) { + res.SkippedChunks++ + } else { + sealed, err := crypto.SealChunk(r.MasterKey, hash, chunk) + if err != nil { + return fmt.Errorf("failed to seal chunk %s: %w", hash, err) + } + encSum := sha256.Sum256(sealed) + if err := backend.Upload(dataPath, bytes.NewReader(sealed)); err != nil { + return fmt.Errorf("failed to upload chunk %s: %w", hash, err) + } + ix.Add(index.Entry{ + PlainSHA256: hash, + PlainSize: int64(len(chunk)), + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + res.UploadedChunks++ + sinceCheckpoint++ + if sinceCheckpoint >= indexCheckpointEvery { + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return fmt.Errorf("failed to checkpoint index: %w", err) + } + sinceCheckpoint = 0 + } + } + + encEntry, _ := ix.Get(hash) + chunks = append(chunks, core.ChunkMeta{ + Index: len(chunks), + Name: dataPath, + PlainSize: encEntry.PlainSize, + PlainSHA256: hash, + EncryptedSize: encEntry.EncryptedSize, + EncryptedSHA256: encEntry.EncryptedSHA256, + }) + if progress != nil { + progress(res.TotalChunks) + } + return nil + }) + if splitErr != nil { + return splitErr + } + entries = append(entries, core.TreeEntry{ + RelPath: rel, + Type: "file", + Mode: mode, + Mtime: mtime, + Size: info.Size(), + Chunks: chunks, + }) + return nil + }) + if walkErr != nil { + return BackupResult{}, walkErr + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].RelPath < entries[j].RelPath + }) + + if _, err := ix.Save(backend, r.Config.Recipients); err != nil { + return BackupResult{}, fmt.Errorf("failed to save index: %w", err) + } + + host, _ := os.Hostname() + manifest := core.TreeManifest{ + Version: repo.FormatVersion, + UUID: snapshotID, + Kind: core.KindTree, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Hostname: host, + Paths: []string{abs}, + Entries: entries, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to marshal tree manifest: %w", err) + } + sealed, err := crypto.EncryptMulti(r.Config.Recipients, manifestJSON) + if err != nil { + return BackupResult{}, fmt.Errorf("failed to encrypt tree manifest: %w", err) + } + snapshotPath, err := repo.SnapshotPath(snapshotID) + if err != nil { + return BackupResult{}, err + } + if err := backend.Upload(snapshotPath, bytes.NewReader(sealed)); err != nil { + return BackupResult{}, fmt.Errorf("failed to upload snapshot: %w", err) + } + return res, nil +} diff --git a/internal/flow/tree_backup_test.go b/internal/flow/tree_backup_test.go new file mode 100644 index 0000000..fb97326 --- /dev/null +++ b/internal/flow/tree_backup_test.go @@ -0,0 +1,134 @@ +package flow + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRunTreeBackupNestedDirs(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub", "empty"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "sub", "b.txt"), []byte("beta-beta-beta"), 0644); err != nil { + t.Fatal(err) + } + + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000010" + res, err := runBackupPath(root, ident, backend, snap, nil) + if err != nil { + t.Fatalf("runBackupPath() error: %v", err) + } + if res.TotalChunks < 2 { + t.Fatalf("TotalChunks = %d, want >= 2", res.TotalChunks) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.Kind != "tree" { + t.Fatalf("Kind = %q", m.Kind) + } + if m.FileCount() != 2 { + t.Fatalf("FileCount = %d", m.FileCount()) + } + var paths []string + for _, e := range m.Entries { + paths = append(paths, e.RelPath) + } + if paths[0] != "a.txt" { + t.Fatalf("first entry = %q, want a.txt", paths[0]) + } + foundEmpty := false + for _, e := range m.Entries { + if e.Type == "dir" && strings.HasSuffix(e.RelPath, "empty") { + foundEmpty = true + } + } + if !foundEmpty { + t.Fatalf("empty dir missing, entries = %v", paths) + } +} + +func TestRunTreeBackupSkipsSymlinks(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "real.txt"), []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink("real.txt", filepath.Join(root, "link")); err != nil { + t.Skip("symlink not supported") + } + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000011" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + for _, e := range m.Entries { + if e.RelPath == "link" { + t.Fatal("symlink recorded") + } + } +} + +func TestRunBackupPathDispatchesFile(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single")) + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000012" + if _, err := runBackupPath(filePath, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + m, err := readManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.OriginalName != "payload.bin" { + t.Fatalf("OriginalName = %q", m.OriginalName) + } +} + +func TestRunTreeBackupFollowsSymlinkedRoot(t *testing.T) { + ident, backend, _ := repoFixture(t) + real := t.TempDir() + if err := os.WriteFile(filepath.Join(real, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(real, link); err != nil { + t.Skip("symlink not supported") + } + + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000013" + res, err := runBackupPath(link, ident, backend, snap, nil) + if err != nil { + t.Fatalf("runBackupPath(symlink) error: %v", err) + } + if res.TotalChunks < 1 { + t.Fatalf("TotalChunks = %d, want >= 1", res.TotalChunks) + } + m, err := readTreeManifest(backend, ident, snap) + if err != nil { + t.Fatal(err) + } + if m.FileCount() != 1 { + t.Fatalf("FileCount = %d, want 1 (walk of a symlinked root must not be empty)", m.FileCount()) + } + found := false + for _, e := range m.Entries { + if e.RelPath == "a.txt" && e.Type == "file" { + found = true + } + } + if !found { + t.Fatalf("a.txt missing from entries: %+v", m.Entries) + } +} diff --git a/internal/flow/tree_restore_test.go b/internal/flow/tree_restore_test.go new file mode 100644 index 0000000..f9df77b --- /dev/null +++ b/internal/flow/tree_restore_test.go @@ -0,0 +1,438 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/core" + appcrypto "gloak/internal/crypto" + "gloak/internal/repo" + "gloak/internal/storage/memtest" +) + +func treeFixture(t *testing.T) (string, *memtest.MemoryBackend, string, string) { + t.Helper() + ident, backend, _ := repoFixture(t) + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "sub", "empty"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "sub", "b.txt"), []byte("beta"), 0644); err != nil { + t.Fatal(err) + } + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000020" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + return root, backend, ident, snap +} + +func TestRunTreeRestoreFull(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + if base := filepath.Base(got); base == "" { + t.Fatal("empty restore root") + } + a, err := os.ReadFile(filepath.Join(got, "a.txt")) + if err != nil { + t.Fatal(err) + } + if string(a) != "alpha" { + t.Fatalf("a.txt = %q", a) + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "beta" { + t.Fatalf("b.txt = %q", b) + } + if fi, err := os.Stat(filepath.Join(got, "sub", "empty")); err != nil || !fi.IsDir() { + t.Fatal("empty dir not restored") + } +} + +func TestRunTreeRestorePathFilter(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "sub") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + if _, err := os.Stat(filepath.Join(got, "a.txt")); err == nil { + t.Fatal("a.txt restored despite filter") + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil { + t.Fatal(err) + } + if string(b) != "beta" { + t.Fatalf("b.txt = %q", b) + } +} + +func TestRunTreeRestoreRejectsUnsafeRelPath(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000030", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "../escape", Type: "file", Mode: 0644, Mtime: "2026-10-12T00:00:00Z", Size: 0}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + _, err := runRestore(m.UUID, t.TempDir(), ident, backend, "") + if err == nil { + t.Fatal("runRestore() accepted traversal RelPath") + } + if !strings.Contains(err.Error(), "unsafe") && !strings.Contains(err.Error(), "..") { + t.Fatalf("error = %v", err) + } +} + +func TestRunTreeRestoreKeepsUnsafeRootNameInsideOutput(t *testing.T) { + ident, backend, _ := repoFixture(t) + // Base(Clean("/")) is "/", which is not a safe single path segment, + // so the restore root falls back to the snapshot UUID. + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000031", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/"}, + Entries: nil, + } + seedTreeSnapshot(t, backend, ident, m) + + out := t.TempDir() + got, err := runRestore(m.UUID, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + want := filepath.Join(out, m.UUID) + if got != want { + t.Fatalf("restore root = %q, want %q", got, want) + } +} + +func TestRunTreeRestoreRejectsUnsafeRootAndUUID(t *testing.T) { + ident, backend, _ := repoFixture(t) + // The path id stays a valid UUID while the manifest content is + // hostile: root name and UUID both escape the output directory. + validID := "aaaaaaaa-bbbb-4ccc-8ddd-000000000032" + m := core.TreeManifest{ + Version: "2.0", + UUID: "../evil", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{".."}, + Entries: nil, + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + r, err := repo.Open(backend, ident) + if err != nil { + t.Fatal(err) + } + sealed, err := appcrypto.EncryptMulti(r.Config.Recipients, raw) + if err != nil { + t.Fatal(err) + } + if err := backend.Upload("snapshots/"+validID+".age", bytes.NewReader(sealed)); err != nil { + t.Fatal(err) + } + + if _, err := runRestore(validID, t.TempDir(), ident, backend, ""); err == nil { + t.Fatal("runRestore() accepted a hostile root name and uuid") + } else if !strings.Contains(err.Error(), "unsafe") { + t.Fatalf("error = %v, want it to reject the unsafe root", err) + } +} + +func TestRunTreeRestoreFilterMatchingNothingFails(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + _, err := runRestore(snap, t.TempDir(), ident, backend, "does/not/exist") + if err == nil { + t.Fatal("runRestore() succeeded with a filter that matches nothing") + } + if !strings.Contains(err.Error(), "matches nothing") { + t.Fatalf("error = %v, want it to report the unmatched filter", err) + } +} + +func TestRunTreeRestoreAppliesDirModesAfterChildren(t *testing.T) { + ident, backend, _ := repoFixture(t) + root := t.TempDir() + locked := filepath.Join(root, "locked") + if err := os.MkdirAll(locked, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "f.txt"), []byte("inside"), 0444); err != nil { + t.Fatal(err) + } + if err := os.Chmod(locked, 0555); err != nil { + t.Fatal(err) + } + // Restore write access so t.TempDir can clean up after the test. + t.Cleanup(func() { _ = os.Chmod(locked, 0755) }) + snap := "aaaaaaaa-bbbb-4ccc-8ddd-000000000033" + if _, err := runBackupPath(root, ident, backend, snap, nil); err != nil { + t.Fatal(err) + } + + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, "") + if err != nil { + t.Fatalf("runRestore() error: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Join(got, "locked"), 0755) }) + fi, err := os.Stat(filepath.Join(got, "locked")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0555 { + t.Fatalf("dir mode = %o, want 555", fi.Mode().Perm()) + } + fi, err = os.Stat(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0444 { + t.Fatalf("file mode = %o, want 444", fi.Mode().Perm()) + } + data, err := os.ReadFile(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if string(data) != "inside" { + t.Fatalf("content = %q", data) + } + + // Restoring again over the read-only tree must succeed and keep the + // recorded modes. + if _, err := runRestore(snap, out, ident, backend, ""); err != nil { + t.Fatalf("re-restore over read-only tree: %v", err) + } + fi, err = os.Stat(filepath.Join(got, "locked")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0555 { + t.Fatalf("dir mode after re-restore = %o, want 555", fi.Mode().Perm()) + } + fi, err = os.Stat(filepath.Join(got, "locked", "f.txt")) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0444 { + t.Fatalf("file mode after re-restore = %o, want 444", fi.Mode().Perm()) + } +} + +func TestNormalizePathFilter(t *testing.T) { + cases := []struct { + in string + want string + wantErr bool + }{ + {"", "", false}, + {"sub", "sub", false}, + {"sub/", "sub", false}, + {"./sub", "sub", false}, + {"sub//b.txt", "sub/b.txt", false}, + {"sub/../sub", "sub", false}, + {".", "", false}, + {"..", "", true}, + {"../x", "", true}, + {"a/../../x", "", true}, + {"/abs", "", true}, + {`back\slash`, "", true}, + } + for _, c := range cases { + got, err := normalizePathFilter(c.in) + if c.wantErr { + if err == nil { + t.Errorf("normalizePathFilter(%q) = %q, want error", c.in, got) + } + continue + } + if err != nil { + t.Errorf("normalizePathFilter(%q) error: %v", c.in, err) + continue + } + if got != c.want { + t.Errorf("normalizePathFilter(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestRejectSymlinkRestorePath(t *testing.T) { + base := t.TempDir() + destRoot := filepath.Join(base, "dest") + // A destination root that does not exist yet is fine. + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "a.txt")); err != nil { + t.Fatalf("missing destRoot: %v", err) + } + if err := os.MkdirAll(filepath.Join(destRoot, "sub"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(destRoot, "a.txt"), []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "sub", "b.txt")); err != nil { + t.Fatalf("regular path: %v", err) + } + victim := filepath.Join(base, "victim.txt") + if err := os.WriteFile(victim, []byte("keep"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, filepath.Join(destRoot, "link.txt")); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "link.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("final component symlink: err = %v, want symlink rejection", err) + } + outside := filepath.Join(base, "outside") + if err := os.MkdirAll(outside, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(destRoot, "evil")); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(destRoot, filepath.Join(destRoot, "evil", "x.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("intermediate symlink: err = %v, want symlink rejection", err) + } + linkRoot := filepath.Join(base, "linkroot") + if err := os.Symlink(outside, linkRoot); err != nil { + t.Fatal(err) + } + if err := rejectSymlinkRestorePath(linkRoot, filepath.Join(linkRoot, "x.txt")); err == nil || + !strings.Contains(err.Error(), "symlink") { + t.Fatalf("destRoot symlink: err = %v, want symlink rejection", err) + } +} + +func TestRestoreTreeFileRejectsSymlinkTarget(t *testing.T) { + dir := t.TempDir() + victim := filepath.Join(dir, "victim.txt") + if err := os.WriteFile(victim, []byte("untouched"), 0644); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "link.txt") + if err := os.Symlink(victim, link); err != nil { + t.Fatal(err) + } + entry := core.TreeEntry{RelPath: "link.txt", Type: "file", Mode: 0644} + // The symlink guard runs before any repo or backend access, so nil is safe. + err := restoreTreeFile(entry, nil, nil, link) + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } +} + +func TestRunTreeRestoreFilterNormalizesSlashes(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + for _, filter := range []string{"sub/", "./sub"} { + out := t.TempDir() + got, err := runRestore(snap, out, ident, backend, filter) + if err != nil { + t.Fatalf("filter %q: %v", filter, err) + } + if _, err := os.Stat(filepath.Join(got, "a.txt")); err == nil { + t.Fatalf("filter %q: a.txt restored despite filter", filter) + } + b, err := os.ReadFile(filepath.Join(got, "sub", "b.txt")) + if err != nil || string(b) != "beta" { + t.Fatalf("filter %q: sub/b.txt = %q, err = %v", filter, b, err) + } + } +} + +func TestRunTreeRestoreRejectsSymlinkTarget(t *testing.T) { + root, backend, ident, snap := treeFixture(t) + out := t.TempDir() + dest := filepath.Join(out, filepath.Base(root)) + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + victim := filepath.Join(t.TempDir(), "victim.txt") + if err := os.WriteFile(victim, []byte("untouched"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(victim, filepath.Join(dest, "a.txt")); err != nil { + t.Fatal(err) + } + _, err := runRestore(snap, out, ident, backend, "") + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } + got, err := os.ReadFile(victim) + if err != nil || string(got) != "untouched" { + t.Fatalf("victim = %q, err = %v, want it untouched", got, err) + } +} + +func TestRunTreeRestoreRejectsSymlinkDirectory(t *testing.T) { + root, backend, ident, snap := treeFixture(t) + out := t.TempDir() + dest := filepath.Join(out, filepath.Base(root)) + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.Symlink(outside, filepath.Join(dest, "sub")); err != nil { + t.Fatal(err) + } + _, err := runRestore(snap, out, ident, backend, "") + if err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatalf("err = %v, want symlink rejection", err) + } + if _, err := os.Stat(filepath.Join(outside, "b.txt")); err == nil { + t.Fatal("restore escaped through a symlinked directory") + } +} + +func seedTreeSnapshot(t *testing.T, backend *memtest.MemoryBackend, ident string, m core.TreeManifest) { + t.Helper() + r, err := repo.Open(backend, ident) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + sealed, err := appcrypto.EncryptMulti(r.Config.Recipients, raw) + if err != nil { + t.Fatal(err) + } + p, err := repo.SnapshotPath(m.UUID) + if err != nil { + t.Fatal(err) + } + if err := backend.Upload(p, bytes.NewReader(sealed)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/flow/tree_verify_test.go b/internal/flow/tree_verify_test.go new file mode 100644 index 0000000..5d650a1 --- /dev/null +++ b/internal/flow/tree_verify_test.go @@ -0,0 +1,102 @@ +package flow + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/core" +) + +func TestRunVerifyTreeSnapshot(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + if err := runVerify(snap, ident, backend); err != nil { + t.Fatalf("runVerify() error: %v", err) + } +} + +func TestRunVerifyTreeCorruptChunk(t *testing.T) { + _, backend, ident, snap := treeFixture(t) + keys := backend.KeysWithPrefix("data/") + if len(keys) == 0 { + t.Fatal("no data") + } + blob := backend.Get(keys[0]) + blob[len(blob)/2] ^= 0xff + backend.Put(keys[0], blob) + err := runVerify(snap, ident, backend) + if err == nil { + t.Fatal("runVerify() succeeded on corrupt chunk") + } + if !strings.Contains(err.Error(), "CHUNK CORRUPT") { + t.Fatalf("error = %v", err) + } +} + +func TestRunVerifyTreeRejectsUnknownEntryType(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000040", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "weird", Type: "socket", Mode: 0644, Mtime: "2026-10-12T00:00:00Z"}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + err := runVerify(m.UUID, ident, backend) + if err == nil || !strings.Contains(err.Error(), "unknown tree entry type") { + t.Fatalf("err = %v, want unknown entry type rejection", err) + } +} + +func TestRunVerifyTreeRejectsUnsafeDirPath(t *testing.T) { + ident, backend, _ := repoFixture(t) + m := core.TreeManifest{ + Version: "2.0", + UUID: "aaaaaaaa-bbbb-4ccc-8ddd-000000000041", + Kind: core.KindTree, + CreatedAt: "2026-10-12T00:00:00Z", + Hostname: "test", + Paths: []string{"/tmp/x"}, + Entries: []core.TreeEntry{ + {RelPath: "../escape", Type: "dir", Mode: 0755, Mtime: "2026-10-12T00:00:00Z"}, + }, + } + seedTreeSnapshot(t, backend, ident, m) + + err := runVerify(m.UUID, ident, backend) + if err == nil || !strings.Contains(err.Error(), "unsafe") { + t.Fatalf("err = %v, want unsafe path rejection", err) + } +} + +func TestRunCheckWithTreeAndFileSnapshots(t *testing.T) { + ident, backend, _ := repoFixture(t) + filePath := writeTempFile(t, []byte("single-file-payload")) + if _, err := runBackup(filePath, ident, backend, "7f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "t.txt"), []byte("tree-payload"), 0644); err != nil { + t.Fatal(err) + } + if _, err := runBackupPath(root, ident, backend, "8f91c6c7-7a0b-44aa-ae23-997b60e4e999", nil); err != nil { + t.Fatal(err) + } + res, err := runCheck(ident, backend) + if err != nil { + t.Fatalf("runCheck() error: %v", err) + } + if res.Snapshots != 2 { + t.Fatalf("Snapshots = %d, want 2", res.Snapshots) + } + if len(res.Missing) != 0 { + t.Fatalf("Missing = %v", res.Missing) + } +} diff --git a/internal/flow/upload.go b/internal/flow/upload.go index 065d377..4ca0236 100644 --- a/internal/flow/upload.go +++ b/internal/flow/upload.go @@ -2,12 +2,16 @@ package flow import ( "bytes" + "crypto/rand" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "os" "path/filepath" + "regexp" + "time" "github.com/pterm/pterm" @@ -16,105 +20,67 @@ import ( "gloak/internal/storage" ) -func RunUpload(filePath string, recipientKey string, remote string) error { - return runUploadWithStateAndChunkSize(filePath, recipientKey, storage.NewRcloneBackend(remote), core.GenerateUUID(), remote, defaultUploadStateDir(), core.ChunkSize) -} - -func runUpload(filePath string, recipientKey string, backend storage.Backend, backupID string) error { - return runUploadWithState(filePath, recipientKey, backend, backupID, "", "") -} +const fileFormatVersion = "file-2" -func runUploadWithState(filePath string, recipientKey string, backend storage.Backend, backupID string, remote string, stateDir string) error { - return runUploadWithStateAndChunkSize(filePath, recipientKey, backend, backupID, remote, stateDir, core.ChunkSize) -} +const masterKeySize = 32 -func runUploadWithStateAndChunkSize(filePath string, recipientKey string, backend storage.Backend, backupID string, remote string, stateDir string, chunkSize int64) error { - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("failed to open source file: %w", err) - } - defer file.Close() +var fileChunkNameRe = regexp.MustCompile(`^chunk_\d{5,}$`) - stat, err := file.Stat() - if err != nil { - return fmt.Errorf("failed to stat file: %w", err) - } - originalSize := stat.Size() - originalName := filepath.Base(filePath) - state := UploadState{} - if stateDir != "" { - absPath, err := filepath.Abs(filePath) - if err != nil { - return fmt.Errorf("failed to resolve source path: %w", err) - } - state = newUploadStateWithChunkSize(backupID, absPath, originalName, originalSize, remote, recipientKey, chunkSize) - if err := saveUploadState(stateDir, state); err != nil { - return err - } - } - - pterm.Success.Printf("Backup ID: %s\n", backupID) - - p, _ := pterm.DefaultProgressbar. - WithTotal(int(originalSize)). - WithTitle("Encrypting & Uploading..."). - WithRemoveWhenDone(false). - Start() - - chunksMeta, payloadHash, err := uploadPlaintextChunks(file, recipientKey, backend, backupID, &state, stateDir, chunkSize, nil, func(n int) { - p.Add(n) - }) - - if err != nil { - p.Stop() - return fmt.Errorf("chunking/upload process failed: %w", err) - } - - p.Stop() - - pterm.Info.Println("Building and encrypting manifest...") - manifest := core.CreateManifest(backupID, originalName, originalSize, recipientKey, payloadHash, chunksMeta) +type fileManifest struct { + Version string `json:"version"` + UUID string `json:"uuid"` + OriginalName string `json:"original_name"` + OriginalSize int64 `json:"original_size"` + CreatedAt string `json:"created_at"` + PayloadSHA256 string `json:"payload_sha256"` + MasterKey string `json:"master_key"` + Chunks []core.ChunkMeta `json:"chunks"` +} - encryptedManifestBytes, err := core.EncryptManifest(recipientKey, manifest) - if err != nil { - return fmt.Errorf("failed to encrypt manifest: %w", err) - } +type fileLocator struct { + Version string `json:"version"` + UUID string `json:"uuid"` +} - err = backend.Upload(fmt.Sprintf("%s/manifest.age", backupID), bytes.NewReader(encryptedManifestBytes)) - if err != nil { - return fmt.Errorf("failed to upload manifest.age: %w", err) - } +// fileChunkName formats an index as chunk_ followed by a decimal number padded +// to at least five digits. +func fileChunkName(index int) string { + return fmt.Sprintf("chunk_%05d", index) +} - locator := core.CreateLocator(backupID) - locatorBytes, _ := core.SerializeLocator(locator) - err = backend.Upload(fmt.Sprintf("%s/locator.json", backupID), bytes.NewReader(locatorBytes)) - if err != nil { - return fmt.Errorf("failed to upload locator.json: %w", err) +// validateFileChunkName accepts only chunk_ names with at least five digits. +func validateFileChunkName(name string) error { + if !fileChunkNameRe.MatchString(name) { + return fmt.Errorf("invalid chunk name: %s", name) } - if stateDir != "" { - if err := deleteUploadState(stateDir, backupID); err != nil { - return fmt.Errorf("backup completed but failed to delete upload state: %w", err) - } - } - - pterm.Success.Println("Backup completed successfully!") - pterm.DefaultBasicText.Printf("Save this UUID for restoration: %s\n", backupID) return nil } -func uploadPlaintextChunks(file *os.File, recipientKey string, backend storage.Backend, backupID string, state *UploadState, stateDir string, chunkSize int64, onSkip func(core.ChunkMeta), onProgress func(int)) ([]core.ChunkMeta, string, error) { - if chunkSize <= 0 { - return nil, "", fmt.Errorf("chunk size must be positive") +// RunUpload seals filePath into a self-contained {uuid}/ folder on backend and +// returns that UUID. Chunks contain up to core.ChunkSize bytes. It writes the +// locator last as a completion marker. Empty recipients, file read, encryption, +// and upload errors are returned; uploaded objects remain after a failure. +func RunUpload(filePath string, recipients []string, backend storage.Backend) (string, error) { + if len(recipients) == 0 { + return "", fmt.Errorf("at least one recipient is required") + } + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("failed to open source file: %w", err) } + defer file.Close() - existingChunks := make(map[int]core.ChunkMeta, len(state.UploadedChunks)) - for _, chunk := range state.UploadedChunks { - existingChunks[chunk.Index] = chunk + backupID := core.GenerateUUID() + masterKey := make([]byte, masterKeySize) + if _, err := rand.Read(masterKey); err != nil { + return "", fmt.Errorf("failed to generate master key: %w", err) } + spinner, _ := pterm.DefaultSpinner.Start("Encrypting and uploading...") payloadHasher := sha256.New() - chunksMeta := make([]core.ChunkMeta, 0, len(state.UploadedChunks)) - buffer := make([]byte, int(chunkSize)) + buffer := make([]byte, int(core.ChunkSize)) + var metas []core.ChunkMeta + var total int64 for index := 0; ; index++ { n, readErr := io.ReadFull(file, buffer) @@ -122,93 +88,80 @@ func uploadPlaintextChunks(file *os.File, recipientKey string, backend storage.B break } if readErr != nil && readErr != io.ErrUnexpectedEOF { - return nil, "", fmt.Errorf("failed to read source chunk %d: %w", index, readErr) + _ = spinner.Stop() + return "", fmt.Errorf("failed to read chunk %d: %w", index, readErr) } - - plainChunk := append([]byte(nil), buffer[:n]...) - payloadHasher.Write(plainChunk) - plainHash := hashBytes(plainChunk) - chunkName := fmt.Sprintf("chunk_%05d", index) - - if existing, ok := existingChunks[index]; ok { - if existing.Name != chunkName { - return nil, "", fmt.Errorf("upload state chunk %d name mismatch: got %s, want %s", index, existing.Name, chunkName) - } - if existing.PlainSize != int64(n) || existing.PlainSHA256 != plainHash { - return nil, "", fmt.Errorf("upload state chunk %d does not match source file", index) - } - chunksMeta = append(chunksMeta, existing) - if onSkip != nil { - onSkip(existing) - } - } else { - encryptedChunk, err := encryptChunk(recipientKey, plainChunk) - if err != nil { - return nil, "", fmt.Errorf("failed to encrypt chunk %d: %w", index, err) - } - - chunk := core.ChunkMeta{ - Index: index, - Name: chunkName, - PlainSize: int64(n), - PlainSHA256: plainHash, - EncryptedSize: int64(len(encryptedChunk)), - EncryptedSHA256: hashBytes(encryptedChunk), - } - - if err := backend.Upload(fmt.Sprintf("%s/%s", backupID, chunkName), bytes.NewReader(encryptedChunk)); err != nil { - return nil, "", fmt.Errorf("chunk %d failed to process: %w", index, err) - } - chunksMeta = append(chunksMeta, chunk) - if stateDir != "" { - state.UploadedChunks = append(state.UploadedChunks, chunk) - if err := saveUploadState(stateDir, *state); err != nil { - return nil, "", fmt.Errorf("chunk %d callback failed: %w", index, err) - } - } + if n == 0 { + break } - if onProgress != nil { - onProgress(n) + plain := buffer[:n] + payloadHasher.Write(plain) + plainHash := hashBytes(plain) + sealed, err := crypto.SealChunk(masterKey, plainHash, plain) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to seal chunk %d: %w", index, err) + } + encSum := sha256.Sum256(sealed) + name := fileChunkName(index) + remotePath := backupID + "/" + name + if err := backend.Upload(remotePath, bytes.NewReader(sealed)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload chunk %d: %w", index, err) } + metas = append(metas, core.ChunkMeta{ + Index: index, + Name: name, + PlainSize: int64(n), + PlainSHA256: plainHash, + EncryptedSize: int64(len(sealed)), + EncryptedSHA256: hex.EncodeToString(encSum[:]), + }) + total += int64(n) + spinner.UpdateText(fmt.Sprintf("Uploaded %d chunks", len(metas))) + if readErr == io.ErrUnexpectedEOF { break } } - return chunksMeta, hex.EncodeToString(payloadHasher.Sum(nil)), nil -} - -func encryptChunk(recipientKey string, plainChunk []byte) ([]byte, error) { - var encrypted bytes.Buffer - encWriter, err := crypto.EncryptWriter(recipientKey, &encrypted) + manifest := fileManifest{ + Version: fileFormatVersion, + UUID: backupID, + OriginalName: filepath.Base(filePath), + OriginalSize: total, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + PayloadSHA256: hex.EncodeToString(payloadHasher.Sum(nil)), + MasterKey: hex.EncodeToString(masterKey), + Chunks: metas, + } + manifestJSON, err := json.Marshal(manifest) if err != nil { - return nil, err + _ = spinner.Stop() + return "", fmt.Errorf("failed to marshal manifest: %w", err) } - if _, err := encWriter.Write(plainChunk); err != nil { - _ = encWriter.Close() - return nil, err + sealedManifest, err := crypto.EncryptMulti(recipients, manifestJSON) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to encrypt manifest: %w", err) } - if err := encWriter.Close(); err != nil { - return nil, err + if err := backend.Upload(backupID+"/manifest.age", bytes.NewReader(sealedManifest)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload manifest: %w", err) } - return encrypted.Bytes(), nil -} -func hashBytes(data []byte) string { - hash := sha256.Sum256(data) - return hex.EncodeToString(hash[:]) -} - -type ProgressReader struct { - io.Reader - pb *pterm.ProgressbarPrinter -} - -func (pr *ProgressReader) Read(p []byte) (int, error) { - n, err := pr.Reader.Read(p) - if n > 0 { - pr.pb.Add(n) + locator := fileLocator{Version: fileFormatVersion, UUID: backupID} + locatorJSON, err := json.Marshal(locator) + if err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to marshal locator: %w", err) + } + if err := backend.Upload(backupID+"/locator.json", bytes.NewReader(locatorJSON)); err != nil { + _ = spinner.Stop() + return "", fmt.Errorf("failed to upload locator: %w", err) } - return n, err + + spinner.Success(fmt.Sprintf("Upload complete: %s (%d chunks, %d bytes)", backupID, len(metas), total)) + return backupID, nil } diff --git a/internal/flow/upload_state.go b/internal/flow/upload_state.go deleted file mode 100644 index 8596e71..0000000 --- a/internal/flow/upload_state.go +++ /dev/null @@ -1,109 +0,0 @@ -package flow - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "gloak/internal/core" -) - -const uploadStateVersion = "1.0" - -type UploadState struct { - Version string `json:"version"` - BackupID string `json:"backup_id"` - SourcePath string `json:"source_path"` - OriginalName string `json:"original_name"` - OriginalSize int64 `json:"original_size"` - Remote string `json:"remote"` - Recipient string `json:"recipient"` - ChunkSize int64 `json:"chunk_size"` - UploadedChunks []core.ChunkMeta `json:"uploaded_chunks"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} - -func NewUploadState(backupID string, sourcePath string, originalName string, originalSize int64, remote string, recipient string) UploadState { - return newUploadStateWithChunkSize(backupID, sourcePath, originalName, originalSize, remote, recipient, core.ChunkSize) -} - -func newUploadStateWithChunkSize(backupID string, sourcePath string, originalName string, originalSize int64, remote string, recipient string, chunkSize int64) UploadState { - now := uploadStateTimestamp() - return UploadState{ - Version: uploadStateVersion, - BackupID: backupID, - SourcePath: sourcePath, - OriginalName: originalName, - OriginalSize: originalSize, - Remote: remote, - Recipient: recipient, - ChunkSize: chunkSize, - CreatedAt: now, - UpdatedAt: now, - } -} - -func uploadStatePath(stateDir string, backupID string) string { - return filepath.Join(stateDir, backupID+".json") -} - -func defaultUploadStateDir() string { - home, err := os.UserHomeDir() - if err != nil { - home = "." - } - return filepath.Join(home, ".gloak", "uploads") -} - -func saveUploadState(stateDir string, state UploadState) error { - if err := os.MkdirAll(stateDir, 0700); err != nil { - return fmt.Errorf("failed to create upload state directory: %w", err) - } - - state.UpdatedAt = uploadStateTimestamp() - data, err := json.MarshalIndent(state, "", " ") - if err != nil { - return fmt.Errorf("failed to encode upload state: %w", err) - } - - path := uploadStatePath(stateDir, state.BackupID) - if err := os.WriteFile(path, data, 0600); err != nil { - return fmt.Errorf("failed to write upload state: %w", err) - } - - return nil -} - -func loadUploadState(stateDir string, backupID string) (UploadState, error) { - path := uploadStatePath(stateDir, backupID) - data, err := os.ReadFile(path) - if err != nil { - return UploadState{}, fmt.Errorf("failed to read upload state: %w", err) - } - - var state UploadState - if err := json.Unmarshal(data, &state); err != nil { - return UploadState{}, fmt.Errorf("failed to parse upload state: %w", err) - } - if state.BackupID != backupID { - return UploadState{}, fmt.Errorf("upload state backup id mismatch: got %s, want %s", state.BackupID, backupID) - } - - return state, nil -} - -func deleteUploadState(stateDir string, backupID string) error { - path := uploadStatePath(stateDir, backupID) - if err := os.Remove(path); err != nil { - return fmt.Errorf("failed to delete upload state: %w", err) - } - - return nil -} - -func uploadStateTimestamp() string { - return time.Now().UTC().Format(time.RFC3339Nano) -} diff --git a/internal/flow/upload_state_test.go b/internal/flow/upload_state_test.go deleted file mode 100644 index 535d176..0000000 --- a/internal/flow/upload_state_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package flow - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "gloak/internal/core" -) - -func TestNewUploadState(t *testing.T) { - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - - if state.Version != "1.0" { - t.Fatalf("Version = %q, want 1.0", state.Version) - } - if state.BackupID != "backup-id" || state.SourcePath != "/tmp/secret.tar" || state.OriginalName != "secret.tar" { - t.Fatalf("state identity fields not preserved: %+v", state) - } - if state.OriginalSize != 123 || state.Remote != "remote:path" || state.Recipient != "age1recipient" { - t.Fatalf("state backup fields not preserved: %+v", state) - } - if state.ChunkSize != core.ChunkSize { - t.Fatalf("ChunkSize = %d, want %d", state.ChunkSize, core.ChunkSize) - } - if state.CreatedAt == "" || state.UpdatedAt == "" { - t.Fatalf("timestamps should be set: %+v", state) - } - if _, err := time.Parse(time.RFC3339Nano, state.CreatedAt); err != nil { - t.Fatalf("CreatedAt is not RFC3339Nano: %v", err) - } -} - -func TestUploadStateSaveLoadRoundTrip(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - state.UploadedChunks = []core.ChunkMeta{{Index: 0, Name: "chunk_00000", PlainSize: 10, PlainSHA256: "plain", EncryptedSize: 12, EncryptedSHA256: "encrypted"}} - - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - path := uploadStatePath(stateDir, "backup-id") - info, err := os.Stat(path) - if err != nil { - t.Fatalf("state file was not written: %v", err) - } - if info.Mode().Perm() != 0600 { - t.Fatalf("state file permissions = %v, want 0600", info.Mode().Perm()) - } - - got, err := loadUploadState(stateDir, "backup-id") - if err != nil { - t.Fatalf("loadUploadState() returned error: %v", err) - } - if got.BackupID != state.BackupID || got.SourcePath != state.SourcePath || got.Remote != state.Remote { - t.Fatalf("loaded state identity fields = %+v, want %+v", got, state) - } - if len(got.UploadedChunks) != 1 || got.UploadedChunks[0] != state.UploadedChunks[0] { - t.Fatalf("loaded chunks = %+v, want %+v", got.UploadedChunks, state.UploadedChunks) - } - createdAt, err := time.Parse(time.RFC3339Nano, got.CreatedAt) - if err != nil { - t.Fatalf("loaded CreatedAt is not RFC3339Nano: %v", err) - } - updatedAt, err := time.Parse(time.RFC3339Nano, got.UpdatedAt) - if err != nil { - t.Fatalf("loaded UpdatedAt is not RFC3339Nano: %v", err) - } - if updatedAt.Before(createdAt) { - t.Fatalf("UpdatedAt %s is before CreatedAt %s", got.UpdatedAt, got.CreatedAt) - } -} - -func TestUploadStatePathUsesBackupID(t *testing.T) { - stateDir := t.TempDir() - got := uploadStatePath(stateDir, "backup-id") - want := filepath.Join(stateDir, "backup-id.json") - if got != want { - t.Fatalf("uploadStatePath() = %q, want %q", got, want) - } -} - -func TestLoadUploadStateRejectsInvalidJSON(t *testing.T) { - stateDir := t.TempDir() - path := uploadStatePath(stateDir, "backup-id") - if err := os.WriteFile(path, []byte("{"), 0600); err != nil { - t.Fatalf("failed to write invalid state: %v", err) - } - - _, err := loadUploadState(stateDir, "backup-id") - if err == nil { - t.Fatalf("loadUploadState() succeeded with invalid JSON") - } - if !strings.Contains(err.Error(), "failed to parse upload state") { - t.Fatalf("loadUploadState() error = %q, want parse error", err) - } -} - -func TestLoadUploadStateRejectsBackupIDMismatch(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("other-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - if err := os.Rename(uploadStatePath(stateDir, "other-id"), uploadStatePath(stateDir, "backup-id")); err != nil { - t.Fatalf("failed to rename state file: %v", err) - } - - _, err := loadUploadState(stateDir, "backup-id") - if err == nil { - t.Fatalf("loadUploadState() succeeded with mismatched backup id") - } - if !strings.Contains(err.Error(), "upload state backup id mismatch") { - t.Fatalf("loadUploadState() error = %q, want backup id mismatch", err) - } -} - -func TestDeleteUploadState(t *testing.T) { - stateDir := t.TempDir() - state := NewUploadState("backup-id", "/tmp/secret.tar", "secret.tar", 123, "remote:path", "age1recipient") - if err := saveUploadState(stateDir, state); err != nil { - t.Fatalf("saveUploadState() returned error: %v", err) - } - - if err := deleteUploadState(stateDir, "backup-id"); err != nil { - t.Fatalf("deleteUploadState() returned error: %v", err) - } - if _, err := os.Stat(uploadStatePath(stateDir, "backup-id")); !os.IsNotExist(err) { - t.Fatalf("state file still exists or stat failed unexpectedly: %v", err) - } -} diff --git a/internal/flow/upload_test.go b/internal/flow/upload_test.go new file mode 100644 index 0000000..8ae0ff7 --- /dev/null +++ b/internal/flow/upload_test.go @@ -0,0 +1,122 @@ +package flow + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "filippo.io/age" + + "gloak/internal/core" + "gloak/internal/storage/memtest" +) + +func testIdentityFile(t *testing.T) (recipient, identityPath string) { + t.Helper() + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatalf("generate identity: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + content := "# created: now\n# public key: " + id.Recipient().String() + "\n" + id.String() + "\n" + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return id.Recipient().String(), path +} + +func TestRunUploadWritesLocatorLast(t *testing.T) { + recipient, _ := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "data.bin") + if err := os.WriteFile(src, []byte("hello gloak"), 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + + uuid, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("RunUpload() error: %v", err) + } + if uuid == "" { + t.Fatal("RunUpload() returned empty uuid") + } + if !backend.Has(uuid + "/locator.json") { + t.Fatalf("missing locator at %s/locator.json", uuid) + } + if !backend.Has(uuid + "/manifest.age") { + t.Fatal("missing manifest.age") + } + if !backend.Has(uuid + "/chunk_00000") { + t.Fatal("missing chunk_00000") + } + + var loc fileLocator + if err := json.Unmarshal(backend.Get(uuid+"/locator.json"), &loc); err != nil { + t.Fatalf("locator json: %v", err) + } + if loc.Version != fileFormatVersion || loc.UUID != uuid { + t.Fatalf("locator = %+v, want version %q uuid %q", loc, fileFormatVersion, uuid) + } + if backend.UploadCount(uuid+"/locator.json") != 1 { + t.Fatal("locator uploaded more than once") + } +} + +func TestRunUploadRejectsEmptyRecipients(t *testing.T) { + src := filepath.Join(t.TempDir(), "data.bin") + if err := os.WriteFile(src, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + _, err := RunUpload(src, nil, memtest.NewMemoryBackend()) + if err == nil || !strings.Contains(err.Error(), "recipient") { + t.Fatalf("err = %v, want recipient error", err) + } +} + +func TestFileChunkNameAndValidation(t *testing.T) { + if got := fileChunkName(0); got != "chunk_00000" { + t.Fatalf("fileChunkName(0) = %q", got) + } + if err := validateFileChunkName("chunk_00001"); err != nil { + t.Fatalf("valid name rejected: %v", err) + } + for _, name := range []string{"chunk_100000", "chunk_000001"} { + if err := validateFileChunkName(name); err != nil { + t.Fatalf("validateFileChunkName(%q) rejected: %v", name, err) + } + } + bad := []string{ + "chunk_0000", + "chunk_00000x", + "../evil", + "chunk_00000/../x", + "aaaa-bbbb-cccc-dddd-eeeeeeeeeeee/chunk_00000", + "/etc/passwd", + } + for _, name := range bad { + if err := validateFileChunkName(name); err == nil { + t.Fatalf("validateFileChunkName(%q) succeeded, want error", name) + } + } +} + +func TestRunUploadMultiChunk(t *testing.T) { + recipient, _ := testIdentityFile(t) + src := filepath.Join(t.TempDir(), "big.bin") + // ChunkSize is 20MiB; write slightly over one chunk to force two chunks. + data := bytes.Repeat([]byte("a"), int(core.ChunkSize)+10) + if err := os.WriteFile(src, data, 0600); err != nil { + t.Fatal(err) + } + backend := memtest.NewMemoryBackend() + uuid, err := RunUpload(src, []string{recipient}, backend) + if err != nil { + t.Fatalf("RunUpload() error: %v", err) + } + if !backend.Has(uuid+"/chunk_00000") || !backend.Has(uuid+"/chunk_00001") { + t.Fatal("expected two chunks") + } +} diff --git a/internal/flow/verify.go b/internal/flow/verify.go index 72b917c..0a4693f 100644 --- a/internal/flow/verify.go +++ b/internal/flow/verify.go @@ -4,103 +4,127 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" - "io" "github.com/pterm/pterm" "gloak/internal/core" "gloak/internal/crypto" + "gloak/internal/repo" "gloak/internal/storage" ) -func RunVerify(backupID string, identityPath string, remote string) error { - return runVerify(backupID, identityPath, storage.NewRcloneBackend(remote)) -} - -func runVerify(backupID string, identityPath string, backend storage.Backend) error { - pterm.Info.Printf("Starting verification for UUID: %s\n", backupID) - - var manifestBuf bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/manifest.age", backupID), &manifestBuf) +// RunVerify verifies a standalone file backup or repository snapshot without +// writing restored files. Access, decryption, and integrity errors are returned. +func RunVerify(snapshotID, identityPath, remote string) error { + backend := storage.NewRcloneBackend(remote) + spinner, _ := pterm.DefaultSpinner.Start("Verifying backup...") + err := runVerify(snapshotID, identityPath, backend) if err != nil { - return fmt.Errorf("failed to download manifest.age: %w", err) + _ = spinner.Stop() + return err } + spinner.Success("Backup verified successfully") + return nil +} - pterm.Info.Println("Decrypting manifest...") - manifestReader, err := crypto.DecryptReader(identityPath, &manifestBuf) +// runVerify selects standalone verification when a locator can be downloaded, +// otherwise it opens the repository and verifies the snapshot's file or tree +// contents. It returns repository, manifest, and verification errors. +func runVerify(snapshotID, identityPath string, backend storage.Backend) error { + if isFileMode(backend, snapshotID) { + return runFileVerify(snapshotID, identityPath, backend) + } + r, err := repo.Open(backend, identityPath) if err != nil { - return fmt.Errorf("failed to decrypt manifest (invalid key or corrupted file): %w", err) + return err } - - manifestBytes, err := io.ReadAll(manifestReader) + kind, fileM, treeM, err := readAnyManifest(backend, identityPath, snapshotID) if err != nil { - return fmt.Errorf("failed to read manifest contents: %w", err) + return err } - - var manifest core.Manifest - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return fmt.Errorf("failed to parse manifest JSON: %w", err) + if kind == core.KindTree { + return runTreeVerify(treeM, r, backend) } + return runFileVerifyFromManifest(fileM, r, backend) +} - pterm.Success.Printf("Manifest valid! Verifying %d chunk...\n", len(manifest.Chunks)) - - p, _ := pterm.DefaultProgressbar. - WithTotal(len(manifest.Chunks)). - WithTitle("Verifying Chunks..."). - WithRemoveWhenDone(false). - Start() - +// runFileVerifyFromManifest verifies encrypted and plaintext chunk sizes and +// hashes, authentication, and the overall payload hash without restoring files. +// It returns chunk path, download, decryption, and integrity errors. +func runFileVerifyFromManifest(manifest *core.Manifest, r *repo.Repo, backend storage.Backend) error { payloadHasher := sha256.New() - - for _, chunk := range manifest.Chunks { - var encryptedChunk bytes.Buffer - err := backend.Download(fmt.Sprintf("%s/%s", backupID, chunk.Name), &encryptedChunk) - if err != nil { - p.Stop() - return fmt.Errorf("failed to download chunk %s: %w", chunk.Name, err) + for _, meta := range manifest.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err } - - encryptedBytes := encryptedChunk.Bytes() - if int64(len(encryptedBytes)) != chunk.EncryptedSize { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Size %s mismatch!", chunk.Name) + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) } - if hashBytes(encryptedBytes) != chunk.EncryptedSHA256 { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Hash %s mismatch!", chunk.Name) + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) } - - plainReader, err := crypto.DecryptReader(identityPath, bytes.NewReader(encryptedBytes)) - if err != nil { - p.Stop() - return fmt.Errorf("failed to decrypt chunk %s: %w", chunk.Name, err) + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) } - plainBytes, err := io.ReadAll(plainReader) + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) if err != nil { - p.Stop() - return fmt.Errorf("failed to read decrypted chunk %s: %w", chunk.Name, err) - } - if int64(len(plainBytes)) != chunk.PlainSize { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Plain size %s mismatch!", chunk.Name) + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) } - if hashBytes(plainBytes) != chunk.PlainSHA256 { - p.Stop() - return fmt.Errorf("CHUNK CORRUPT: Plain hash %s mismatch!", chunk.Name) + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) } - payloadHasher.Write(plainBytes) - - p.Add(1) + payloadHasher.Write(plain) } - p.Stop() - if hex.EncodeToString(payloadHasher.Sum(nil)) != manifest.PayloadSHA256 { - return fmt.Errorf("FATAL: Overall payload hash differs from manifest! Data has been modified") + return fmt.Errorf("FATAL: overall payload hash differs from manifest") } + return nil +} - pterm.Success.Println("VERIFIED! All chunks are healthy and checksums match 100%.") +// runTreeVerify verifies paths and encrypted and plaintext chunk sizes and +// hashes for file entries. Directory entries only have their paths checked. +// Unknown entry types and unsafe paths are rejected the same way restore +// rejects them. It returns path, download, decryption, and integrity errors +// without writing restored files. +func runTreeVerify(m *core.TreeManifest, r *repo.Repo, backend storage.Backend) error { + for _, entry := range m.Entries { + if err := core.ValidateRelPath(entry.RelPath); err != nil { + return err + } + switch entry.Type { + case "file": + for _, meta := range entry.Chunks { + if err := validateChunkPath(meta.Name); err != nil { + return err + } + var buf bytes.Buffer + if err := backend.Download(meta.Name, &buf); err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): missing object: %w", meta.Index, meta.Name, err) + } + sealed := buf.Bytes() + if int64(len(sealed)) != meta.EncryptedSize { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted size mismatch", meta.Index, meta.Name) + } + if hashBytes(sealed) != meta.EncryptedSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): encrypted sha256 mismatch", meta.Index, meta.Name) + } + plain, err := crypto.OpenChunk(r.MasterKey, meta.PlainSHA256, sealed) + if err != nil { + return fmt.Errorf("CHUNK CORRUPT %d (%s): %w", meta.Index, meta.Name, err) + } + if int64(len(plain)) != meta.PlainSize || hashBytes(plain) != meta.PlainSHA256 { + return fmt.Errorf("CHUNK CORRUPT %d (%s): plaintext mismatch", meta.Index, meta.Name) + } + } + case "dir": + // Directories carry no chunks to verify. + default: + return fmt.Errorf("unknown tree entry type %q for %s", entry.Type, entry.RelPath) + } + } return nil } diff --git a/internal/index/index.go b/internal/index/index.go new file mode 100644 index 0000000..88689f4 --- /dev/null +++ b/internal/index/index.go @@ -0,0 +1,235 @@ +// Package index tracks which chunks already exist in the repository so +// repeat backups upload nothing. +package index + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + "strconv" + "strings" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +const indexPrefix = "index" + +type Entry struct { + PlainSHA256 string `json:"plain_sha256"` + PlainSize int64 `json:"plain_size"` + EncryptedSize int64 `json:"encrypted_size"` + EncryptedSHA256 string `json:"encrypted_sha256"` +} + +type Index struct { + epoch int + pending map[string]Entry + all map[string]Entry +} + +type epochFile struct { + Entries []Entry `json:"entries"` +} + +// New returns an empty index with no pending entries and an epoch counter of zero. +func New() *Index { + return &Index{ + pending: make(map[string]Entry), + all: make(map[string]Entry), + } +} + +// Load merges readable index files in ascending name order, with later entries +// replacing earlier entries for the same plaintext hash. Listing failures yield +// an empty index; malformed listing JSON returns an error. Files that cannot be +// downloaded, decrypted, read, or parsed are skipped. Skipped entries are not +// queued for Save. The epoch counter is restored from the highest recognized +// epoch name, including skipped files. +func Load(backend storage.Backend, identityPath string) (*Index, error) { + ix := New() + + raw, err := backend.ListFiles([]string{indexPrefix}) + if err != nil { + // A fresh repo has no index/ directory yet (rclone lsjson exits 3 + // on a missing directory); that is normal, so treat the listing as + // empty and start from epoch 0. Any other listing failure is a real + // error and must not be mistaken for an empty index. + if storage.IsMissingPath(err) { + return ix, nil + } + return nil, fmt.Errorf("failed to list index: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse index listing: %w", err) + } + + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir { + names = append(names, entry.Path) + } + } + sort.Strings(names) + + for _, name := range names { + path := fmt.Sprintf("%s/%s", indexPrefix, name) + // Count the epoch number even when the file's contents end up + // skipped, so a later Save never reuses an existing name. + if epoch, ok := parseEpoch(path); ok && epoch > ix.epoch { + ix.epoch = epoch + } + var epochBuf bytes.Buffer + if err := backend.Download(path, &epochBuf); err != nil { + continue + } + reader, err := crypto.DecryptReader(identityPath, &epochBuf) + if err != nil { + continue + } + plain, err := io.ReadAll(reader) + if err != nil { + continue + } + var file epochFile + if err := json.Unmarshal(plain, &file); err != nil { + continue + } + for _, e := range file.Entries { + ix.all[e.PlainSHA256] = e + } + } + return ix, nil +} + +// parseEpoch extracts the epoch number from a path like "index/000001.age". +// ok is false when the path does not match the zero-padded epoch pattern. +func parseEpoch(path string) (int, bool) { + prefix := indexPrefix + "/" + suffix := ".age" + if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) { + return 0, false + } + digits := path[len(prefix) : len(path)-len(suffix)] + if len(digits) != 6 { + return 0, false + } + for _, r := range digits { + if r < '0' || r > '9' { + return 0, false + } + } + epoch, err := strconv.Atoi(digits) + if err != nil { + return 0, false + } + return epoch, true +} + +// Has reports whether a plaintext hash is known, including pending entries. +func (ix *Index) Has(hash string) bool { + _, ok := ix.Get(hash) + return ok +} + +// AllHashes returns every known plaintext hash, sorted. Pending entries that +// were never saved are included so check sees work in progress. +func (ix *Index) AllHashes() []string { + seen := make(map[string]bool, len(ix.all)+len(ix.pending)) + for h := range ix.all { + seen[h] = true + } + for h := range ix.pending { + seen[h] = true + } + out := make([]string, 0, len(seen)) + for h := range seen { + out = append(out, h) + } + sort.Strings(out) + return out +} + +// Get looks up a plaintext hash, preferring a pending entry. It returns the +// zero entry and false when the hash is unknown. +func (ix *Index) Get(hash string) (Entry, bool) { + if e, ok := ix.pending[hash]; ok { + return e, true + } + e, ok := ix.all[hash] + return e, ok +} + +// Add records an entry by its plaintext hash and queues it for the next Save, +// replacing any entry with the same hash. +func (ix *Index) Add(entry Entry) { + ix.pending[entry.PlainSHA256] = entry + ix.all[entry.PlainSHA256] = entry +} + +// Compact writes a new epoch containing known entries marked true in liveHashes +// and returns the number written, or zero without writing if none match. +// It leaves ix unchanged and returns encryption or upload errors from Save. +// Callers delete old epochs separately. +func (ix *Index) Compact(backend storage.Backend, recipients []string, liveHashes map[string]bool) (int, error) { + // Reuse Save by replacing all and pending with the live subset, then + // force a save even if pending is empty (when there is something to write). + next := New() + next.epoch = ix.epoch + count := 0 + for h, e := range ix.all { + if liveHashes[h] { + next.all[h] = e + next.pending[h] = e + count++ + } + } + if count == 0 { + return 0, nil + } + return next.Save(backend, recipients) +} + +// Save writes only the pending entries as the next encrypted epoch and returns +// the number written. It is a no-op when nothing is pending. Pending entries +// are cleared and the epoch counter advances only after a successful upload; +// serialization, encryption, and upload errors leave them unchanged. +func (ix *Index) Save(backend storage.Backend, recipients []string) (int, error) { + if len(ix.pending) == 0 { + return 0, nil + } + + hashes := make([]string, 0, len(ix.pending)) + for hash := range ix.pending { + hashes = append(hashes, hash) + } + sort.Strings(hashes) + + file := epochFile{Entries: make([]Entry, 0, len(hashes))} + for _, hash := range hashes { + file.Entries = append(file.Entries, ix.pending[hash]) + } + + plain, err := json.Marshal(file) + if err != nil { + return 0, fmt.Errorf("failed to marshal index epoch: %w", err) + } + sealed, err := crypto.EncryptMulti(recipients, plain) + if err != nil { + return 0, fmt.Errorf("failed to encrypt index epoch: %w", err) + } + + name := fmt.Sprintf("%s/%06d.age", indexPrefix, ix.epoch+1) + if err := backend.Upload(name, bytes.NewReader(sealed)); err != nil { + return 0, fmt.Errorf("failed to upload index epoch: %w", err) + } + + for _, hash := range hashes { + delete(ix.pending, hash) + } + ix.epoch++ + return len(hashes), nil +} diff --git a/internal/index/index_test.go b/internal/index/index_test.go new file mode 100644 index 0000000..273d46f --- /dev/null +++ b/internal/index/index_test.go @@ -0,0 +1,260 @@ +package index + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage" + "gloak/internal/storage/memtest" +) + +// listErrorBackend fails every listing with err while staying usable for +// anything else, so Load error handling can be exercised. +type listErrorBackend struct { + storage.Backend + err error +} + +func (b *listErrorBackend) ListFiles(prefix []string) ([]byte, error) { + return nil, b.err +} + +func identityFixture(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func entryFixture(hash string) Entry { + return Entry{ + PlainSHA256: hash, + PlainSize: 100, + EncryptedSize: 120, + EncryptedSHA256: hash + "-enc", + } +} + +func TestAddHasGet(t *testing.T) { + ix := New() + ix.Add(entryFixture("aaa")) + if !ix.Has("aaa") { + t.Fatal("Has() = false after Add") + } + if e, ok := ix.Get("aaa"); !ok || e.PlainSize != 100 { + t.Fatalf("Get() = %+v, %v", e, ok) + } + if ix.Has("bbb") { + t.Fatal("Has() = true for unknown hash") + } +} + +func TestAllHashesMergesPendingAndAll(t *testing.T) { + ix := New() + ix.Add(entryFixture("bbb")) + ix.Add(entryFixture("aaa")) + got := ix.AllHashes() + want := []string{"aaa", "bbb"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("AllHashes() = %v, want %v", got, want) + } +} + +func TestSaveAndLoadAcrossEpochs(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + + ix := New() + ix.Add(entryFixture("aaa")) + ix.Add(entryFixture("bbb")) + n, err := ix.Save(backend, []string{pub}) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + if n != 2 { + t.Fatalf("Save() = %d, want 2", n) + } + + ix.Add(entryFixture("ccc")) + if n, err := ix.Save(backend, []string{pub}); err != nil || n != 1 { + t.Fatalf("second Save() = %d, %v, want 1, nil", n, err) + } + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + for _, hash := range []string{"aaa", "bbb", "ccc"} { + if !loaded.Has(hash) { + t.Fatalf("Load() missing %s", hash) + } + } +} + +func TestSaveSkipsEmptyEpoch(t *testing.T) { + pub, _ := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + before := len(backend.KeysWithPrefix("index/")) + n, err := ix.Save(backend, []string{pub}) + if err != nil { + t.Fatalf("Save() error: %v", err) + } + if n != 0 || len(backend.KeysWithPrefix("index/")) != before { + t.Fatalf("empty Save() wrote files: n=%d files=%d", n, len(backend.KeysWithPrefix("index/"))) + } +} + +func TestLoadSkipsUnreadableEpoch(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + ix.Add(entryFixture("bbb")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("second Save() error: %v", err) + } + // Corrupt the third epoch with garbage that is not valid age ciphertext, + // after both valid epochs are already written. + backend.Put("index/000003.age", []byte("garbage-not-age")) + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if !loaded.Has("aaa") || !loaded.Has("bbb") { + t.Fatalf("Load() lost entries: %+v", loaded.all) + } +} + +func TestLoadRestoresEpochCounter(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + ix.Add(entryFixture("bbb")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("second Save() error: %v", err) + } + + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + loaded.Add(entryFixture("ccc")) + if _, err := loaded.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() after Load error: %v", err) + } + + keys := backend.KeysWithPrefix("index/") + want := []string{"index/000001.age", "index/000002.age", "index/000003.age"} + if len(keys) != len(want) { + t.Fatalf("index files = %v, want %v", keys, want) + } + for i, k := range want { + if keys[i] != k { + t.Fatalf("index files = %v, want %v", keys, want) + } + } + + reloaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() after save error: %v", err) + } + for _, hash := range []string{"aaa", "bbb", "ccc"} { + if !reloaded.Has(hash) { + t.Fatalf("reloaded index missing %s", hash) + } + } +} + +func TestLoadOnEmptyBackendIsClean(t *testing.T) { + _, ident := identityFixture(t) + loaded, err := Load(memtest.NewMemoryBackend(), ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(loaded.all) != 0 { + t.Fatalf("Load() on empty backend = %+v", loaded.all) + } +} + +func TestLoadTreatsMissingPathAsEmpty(t *testing.T) { + _, ident := identityFixture(t) + // rclone exits 3 when the remote path is missing; that is a fresh repo. + exitErr := exec.Command("sh", "-c", "exit 3").Run() + backend := &listErrorBackend{ + Backend: memtest.NewMemoryBackend(), + err: fmt.Errorf("failed to list index: %w", exitErr), + } + loaded, err := Load(backend, ident) + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(loaded.AllHashes()) != 0 { + t.Fatalf("Load() on missing index = %v, want empty", loaded.AllHashes()) + } +} + +func TestLoadPropagatesListingErrors(t *testing.T) { + _, ident := identityFixture(t) + backend := &listErrorBackend{ + Backend: memtest.NewMemoryBackend(), + err: errors.New("remote exploded"), + } + if _, err := Load(backend, ident); err == nil { + t.Fatal("Load() succeeded on a real listing error") + } else if !strings.Contains(err.Error(), "remote exploded") { + t.Fatalf("Load() error = %v, want it to wrap the listing failure", err) + } +} + +func TestSaveOutputIsEncryptedJSON(t *testing.T) { + pub, ident := identityFixture(t) + backend := memtest.NewMemoryBackend() + ix := New() + ix.Add(entryFixture("aaa")) + if _, err := ix.Save(backend, []string{pub}); err != nil { + t.Fatalf("Save() error: %v", err) + } + var buf bytes.Buffer + if err := backend.Download("index/000001.age", &buf); err != nil { + t.Fatalf("Download() error: %v", err) + } + reader, err := crypto.DecryptReader(ident, &buf) + if err != nil { + t.Fatalf("index file is not age-encrypted with identity: %v", err) + } + plain, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + if !bytes.Contains(plain, []byte("aaa")) { + t.Fatalf("decrypted index missing entry hash: %s", plain) + } +} diff --git a/internal/notify/telegram.go b/internal/notify/telegram.go new file mode 100644 index 0000000..fba3e95 --- /dev/null +++ b/internal/notify/telegram.go @@ -0,0 +1,90 @@ +package notify + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + "unicode/utf8" +) + +// TelegramConfig holds Bot API credentials. Empty token disables notify. +type TelegramConfig struct { + Token string + ChatID string + // BaseURL overrides the Bot API root for tests. + BaseURL string +} + +// FromEnv reads GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID. +func FromEnv() TelegramConfig { + return TelegramConfig{ + Token: os.Getenv("GLOAK_TELEGRAM_TOKEN"), + ChatID: os.Getenv("GLOAK_TELEGRAM_CHAT_ID"), + } +} + +// Enabled reports whether notify should attempt a send. +func (c TelegramConfig) Enabled() bool { + return c.Token != "" && c.ChatID != "" +} + +// Send posts message to Telegram, truncating it to 4000 bytes with an ellipsis. +// It does nothing when credentials are incomplete. It uses a 15-second HTTP +// timeout and returns request errors or errors for non-200 responses. +func (c TelegramConfig) Send(message string) error { + if !c.Enabled() { + return nil + } + base := c.BaseURL + if base == "" { + base = "https://api.telegram.org" + } + endpoint := fmt.Sprintf("%s/bot%s/sendMessage", base, c.Token) + + form := url.Values{} + form.Set("chat_id", c.ChatID) + form.Set("text", truncate(message, 4000)) + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Post(endpoint, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) + if err != nil { + return fmt.Errorf("telegram send: %s", redact(err.Error(), c.Token)) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var body struct { + Description string `json:"description"` + } + _ = json.NewDecoder(resp.Body).Decode(&body) + return fmt.Errorf("telegram send: status %d: %s", resp.StatusCode, redact(body.Description, c.Token)) + } + return nil +} + +// redact removes the bot token from a message. Transport errors embed the +// full request URL, which contains the token. +func redact(msg, token string) string { + if token == "" { + return msg + } + return strings.ReplaceAll(msg, token, "[redacted]") +} + +// truncate limits s to n bytes, ending shortened text with "...". The cut +// lands on a rune boundary so the result stays valid UTF-8, and n >= 3 is +// required when truncation is needed. +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + cut := n - 3 + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + return s[:cut] + "..." +} diff --git a/internal/notify/telegram_test.go b/internal/notify/telegram_test.go new file mode 100644 index 0000000..9ba6e4d --- /dev/null +++ b/internal/notify/telegram_test.go @@ -0,0 +1,101 @@ +package notify + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "unicode/utf8" +) + +func TestSendDisabledNoops(t *testing.T) { + c := TelegramConfig{} + if c.Enabled() { + t.Fatal("empty config should be disabled") + } + if err := c.Send("hello"); err != nil { + t.Fatalf("Send() error: %v", err) + } +} + +func TestSendPostsMessage(t *testing.T) { + var gotBody string + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer srv.Close() + + c := TelegramConfig{Token: "TESTTOKEN", ChatID: "42", BaseURL: srv.URL} + if err := c.Send("backup ok"); err != nil { + t.Fatalf("Send() error: %v", err) + } + if !strings.Contains(gotPath, "botTESTTOKEN/sendMessage") { + t.Fatalf("path = %q", gotPath) + } + vals, err := url.ParseQuery(gotBody) + if err != nil { + t.Fatal(err) + } + if vals.Get("chat_id") != "42" || vals.Get("text") != "backup ok" { + t.Fatalf("body = %q", gotBody) + } +} + +func TestSendHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"description":"bad request"}`)) + })) + defer srv.Close() + + c := TelegramConfig{Token: "T", ChatID: "1", BaseURL: srv.URL} + if err := c.Send("x"); err == nil { + t.Fatal("Send() succeeded on HTTP error") + } +} + +func TestSendRedactsTokenFromTransportError(t *testing.T) { + // Nothing listens on this port, so the transport error embeds the + // request URL, which contains the bot token. + c := TelegramConfig{ + Token: "SECRETTOKEN123", + ChatID: "1", + BaseURL: "http://127.0.0.1:1", + } + err := c.Send("x") + if err == nil { + t.Fatal("Send() succeeded against an unreachable server") + } + if strings.Contains(err.Error(), "SECRETTOKEN123") { + t.Fatalf("error leaks the bot token: %v", err) + } + if !strings.Contains(err.Error(), "[redacted]") { + t.Fatalf("error = %v, want the token placeholder", err) + } +} + +func TestTruncateKeepsUTF8(t *testing.T) { + // "€" occupies bytes 2-4; a plain byte cut at n-3 would land inside it. + s := "aa€" + strings.Repeat("b", 10) + got := truncate(s, 6) + if got != "aa..." { + t.Fatalf("truncate() = %q, want %q", got, "aa...") + } + if !utf8.ValidString(got) { + t.Fatalf("truncate() = %q is not valid UTF-8", got) + } +} + +func TestTruncateShortInputUnchanged(t *testing.T) { + s := "aa€bb" + if got := truncate(s, len(s)); got != s { + t.Fatalf("truncate() = %q, want %q", got, s) + } +} diff --git a/internal/repo/config.go b/internal/repo/config.go new file mode 100644 index 0000000..2e02f4b --- /dev/null +++ b/internal/repo/config.go @@ -0,0 +1,24 @@ +package repo + +const ( + FormatVersion = "2.0" + ConfigPath = "repo.json" + KeysPrefix = "keys" + SnapshotsPrefix = "snapshots" + DataPrefix = "data" + + DefaultChunkMin = 512 * 1024 + DefaultChunkAvg = 1024 * 1024 + DefaultChunkMax = 8 * 1024 * 1024 +) + +// Config is the plaintext repository descriptor. It carries no user data: +// version, creation time, recipient public keys, and CDC chunk parameters. +type Config struct { + Version string `json:"version"` + CreatedAt string `json:"created_at"` + Recipients []string `json:"recipients"` + ChunkMin int64 `json:"chunk_min"` + ChunkAvg int64 `json:"chunk_avg"` + ChunkMax int64 `json:"chunk_max"` +} diff --git a/internal/repo/repo.go b/internal/repo/repo.go new file mode 100644 index 0000000..2167d56 --- /dev/null +++ b/internal/repo/repo.go @@ -0,0 +1,201 @@ +package repo + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "time" + + "github.com/google/uuid" + + "gloak/internal/crypto" + "gloak/internal/storage" +) + +const masterKeySize = 32 + +type Repo struct { + Backend storage.Backend + Config Config + MasterKey []byte +} + +// KeysPathFor returns the remote path of the wrapped master key for a recipient. +func KeysPathFor(recipient string) string { + return fmt.Sprintf("%s/%s.age", KeysPrefix, crypto.FingerprintRecipient(recipient)) +} + +// DataPath returns the content-addressed object path for a plaintext chunk +// hash. Inputs shorter than 2 bytes cannot be sharded, so it returns +// data/invalid instead of panicking. +func DataPath(plainSHA256Hex string) string { + if len(plainSHA256Hex) < 2 { + return DataPrefix + "/invalid" + } + return fmt.Sprintf("%s/%s/%s", DataPrefix, plainSHA256Hex[:2], plainSHA256Hex) +} + +// SnapshotPath returns the snapshot manifest path, rejecting anything that is +// not a valid UUID so a crafted id can never escape the snapshots/ prefix. +func SnapshotPath(snapshotID string) (string, error) { + if _, err := uuid.Parse(snapshotID); err != nil { + return "", fmt.Errorf("invalid snapshot id: %w", err) + } + return fmt.Sprintf("%s/%s.age", SnapshotsPrefix, snapshotID), nil +} + +// Init creates the repository on the backend: one wrapped master key per +// recipient first, repo.json last as the completion marker. It refuses to +// overwrite a readable repo.json unless force is set; download errors during +// that existence check are treated as absence. A fresh master key makes old +// snapshots unreadable. min, avg, and max are positive, ordered byte counts. +// It returns the new master key and config, or a validation, encryption, or +// storage error. Failed initialization may leave uploaded objects behind. +func Init(backend storage.Backend, recipients []string, min, avg, max int64, force bool) ([]byte, Config, error) { + if len(recipients) == 0 { + return nil, Config{}, fmt.Errorf("at least one recipient is required") + } + for _, r := range recipients { + if _, err := crypto.ParseRecipient(r); err != nil { + return nil, Config{}, fmt.Errorf("invalid recipient %q: %w", r, err) + } + } + if min <= 0 || avg <= 0 || max <= 0 || max < avg || avg < min { + return nil, Config{}, fmt.Errorf("chunk bounds require 0 < min <= avg <= max (min=%d avg=%d max=%d)", min, avg, max) + } + + if !force { + if err := backend.Download(ConfigPath, io.Discard); err == nil { + return nil, Config{}, fmt.Errorf("repository already exists at this location (use force to overwrite)") + } + } + + master := make([]byte, masterKeySize) + if _, err := rand.Read(master); err != nil { + return nil, Config{}, fmt.Errorf("failed to generate master key: %w", err) + } + + cfg := Config{ + Version: FormatVersion, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Recipients: recipients, + ChunkMin: min, + ChunkAvg: avg, + ChunkMax: max, + } + + for _, recipient := range recipients { + wrapped, err := crypto.EncryptMulti([]string{recipient}, master) + if err != nil { + return nil, Config{}, fmt.Errorf("failed to wrap master key: %w", err) + } + if err := backend.Upload(KeysPathFor(recipient), bytes.NewReader(wrapped)); err != nil { + return nil, Config{}, fmt.Errorf("failed to upload wrapped key: %w", err) + } + } + + // Force re-init (or a retry after a crashed init) can leave wrapped + // keys for recipients that are no longer part of the repository. Remove + // them before repo.json, the completion marker, is replaced, so a + // removed recipient cannot keep opening the repository. + if err := deleteStaleKeys(backend, recipients); err != nil { + return nil, Config{}, err + } + + cfgBytes, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, Config{}, fmt.Errorf("failed to marshal repo config: %w", err) + } + if err := backend.Upload(ConfigPath, bytes.NewReader(cfgBytes)); err != nil { + return nil, Config{}, fmt.Errorf("failed to upload repo.json: %w", err) + } + return master, cfg, nil +} + +// deleteStaleKeys removes wrapped keys under keys/ whose fingerprint is +// not part of recipients. A missing keys/ directory is normal on first +// init; any other listing failure is returned. +func deleteStaleKeys(backend storage.Backend, recipients []string) error { + keep := make(map[string]bool, len(recipients)) + for _, recipient := range recipients { + keep[KeysPathFor(recipient)] = true + } + raw, err := backend.ListFiles([]string{KeysPrefix}) + if err != nil { + if storage.IsMissingPath(err) { + return nil + } + return fmt.Errorf("failed to list keys: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return fmt.Errorf("failed to parse keys listing: %w", err) + } + for _, entry := range entries { + if entry.IsDir { + continue + } + full := KeysPrefix + "/" + entry.Path + if keep[full] { + continue + } + if err := backend.Delete(full); err != nil { + return fmt.Errorf("failed to delete stale key %s: %w", full, err) + } + } + return nil +} + +// Open loads repo.json and unwraps a 32-byte master key using identityPath. +// Config download, parsing, format, chunk-bound, and key-listing errors are +// returned. Individual key download, decryption, and read failures are skipped; +// if no key opens, it returns an identity-cannot-open error. +func Open(backend storage.Backend, identityPath string) (*Repo, error) { + var buf bytes.Buffer + if err := backend.Download(ConfigPath, &buf); err != nil { + return nil, fmt.Errorf("not a gloak repository (%s missing): %w", ConfigPath, err) + } + + var cfg Config + if err := json.Unmarshal(buf.Bytes(), &cfg); err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", ConfigPath, err) + } + if cfg.Version != FormatVersion { + return nil, fmt.Errorf("unsupported repository format %q (want %q)", cfg.Version, FormatVersion) + } + if cfg.ChunkMin <= 0 || cfg.ChunkAvg <= 0 || cfg.ChunkMax <= 0 || + cfg.ChunkMax < cfg.ChunkAvg || cfg.ChunkAvg < cfg.ChunkMin { + return nil, fmt.Errorf("invalid chunk parameters in %s (min=%d avg=%d max=%d)", ConfigPath, cfg.ChunkMin, cfg.ChunkAvg, cfg.ChunkMax) + } + + raw, err := backend.ListFiles([]string{KeysPrefix}) + if err != nil { + return nil, fmt.Errorf("failed to list keys: %w", err) + } + entries, err := storage.ParseList(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse keys listing: %w", err) + } + + for _, entry := range entries { + if entry.IsDir { + continue + } + var keyBuf bytes.Buffer + if err := backend.Download(fmt.Sprintf("%s/%s", KeysPrefix, entry.Path), &keyBuf); err != nil { + continue + } + reader, err := crypto.DecryptReader(identityPath, &keyBuf) + if err != nil { + continue + } + master, err := io.ReadAll(reader) + if err != nil || len(master) != masterKeySize { + continue + } + return &Repo{Backend: backend, Config: cfg, MasterKey: master}, nil + } + return nil, fmt.Errorf("identity cannot open this repository") +} diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go new file mode 100644 index 0000000..fee674e --- /dev/null +++ b/internal/repo/repo_test.go @@ -0,0 +1,185 @@ +package repo + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gloak/internal/crypto" + "gloak/internal/storage/memtest" +) + +func testRecipient(t *testing.T) (string, string) { + t.Helper() + pub, _, fileContent, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("GenerateKey() error: %v", err) + } + path := filepath.Join(t.TempDir(), "identity.txt") + if err := os.WriteFile(path, []byte(fileContent), 0600); err != nil { + t.Fatalf("write identity: %v", err) + } + return pub, path +} + +func TestInitWritesKeysAndConfig(t *testing.T) { + pubA, _ := testRecipient(t) + pubB, _ := testRecipient(t) + backend := memtest.NewMemoryBackend() + + master, cfg, err := Init(backend, []string{pubA, pubB}, 1, 8, 64, false) + if err != nil { + t.Fatalf("Init() error: %v", err) + } + if len(master) != 32 { + t.Fatalf("master key length = %d, want 32", len(master)) + } + if cfg.Version != FormatVersion || cfg.ChunkMin != 1 || cfg.ChunkAvg != 8 || cfg.ChunkMax != 64 { + t.Fatalf("cfg = %+v", cfg) + } + if !backend.Has(ConfigPath) { + t.Fatal("repo.json not uploaded") + } + for _, pub := range []string{pubA, pubB} { + path := KeysPathFor(pub) + if !backend.Has(path) { + t.Fatalf("key file %s not uploaded", path) + } + } +} + +func TestInitRemovesStaleKeysOnReinit(t *testing.T) { + pubA, identA := testRecipient(t) + pubB, identB := testRecipient(t) + backend := memtest.NewMemoryBackend() + + if _, _, err := Init(backend, []string{pubA}, 1, 8, 64, false); err != nil { + t.Fatalf("Init() error: %v", err) + } + // A wrapped key for a recipient that is about to be removed. + stalePath := KeysPrefix + "/0000000000000000.age" + backend.Put(stalePath, []byte("stale")) + + if _, _, err := Init(backend, []string{pubB}, 1, 8, 64, true); err != nil { + t.Fatalf("Init(force) error: %v", err) + } + if backend.Has(stalePath) { + t.Fatal("stale key survived a force re-init") + } + if backend.Has(KeysPathFor(pubA)) { + t.Fatal("removed recipient still has a wrapped key") + } + if !backend.Has(KeysPathFor(pubB)) { + t.Fatal("new recipient key missing") + } + if _, err := Open(backend, identB); err != nil { + t.Fatalf("Open() with the new identity: %v", err) + } + if _, err := Open(backend, identA); err == nil { + t.Fatal("removed recipient can still open the repository") + } +} + +func TestOpenWithEachRecipientIdentity(t *testing.T) { + pubA, identA := testRecipient(t) + pubB, identB := testRecipient(t) + backend := memtest.NewMemoryBackend() + master, _, err := Init(backend, []string{pubA, pubB}, 1, 8, 64, false) + if err != nil { + t.Fatalf("Init() error: %v", err) + } + + for _, identPath := range []string{identA, identB} { + r, err := Open(backend, identPath) + if err != nil { + t.Fatalf("Open(%s) error: %v", identPath, err) + } + if string(r.MasterKey) != string(master) { + t.Fatalf("Open(%s) master key mismatch", identPath) + } + } +} + +func TestOpenRejectsWrongIdentity(t *testing.T) { + pubA, _ := testRecipient(t) + _, identOther := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pubA}, 1, 8, 64, false); err != nil { + t.Fatalf("Init() error: %v", err) + } + _, err := Open(backend, identOther) + if err == nil { + t.Fatal("Open() succeeded with unrelated identity") + } + if !strings.Contains(err.Error(), "cannot open") { + t.Fatalf("Open() error = %v, want cannot open", err) + } +} + +func TestOpenRejectsMissingConfig(t *testing.T) { + _, ident := testRecipient(t) + _, err := Open(memtest.NewMemoryBackend(), ident) + if err == nil { + t.Fatal("Open() succeeded on empty backend") + } + if !strings.Contains(err.Error(), ConfigPath) { + t.Fatalf("Open() error = %v, want repo.json mention", err) + } +} + +func TestInitRejectsBadChunkBounds(t *testing.T) { + pub, _ := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pub}, 8, 1, 64, false); err == nil { + t.Fatal("Init() succeeded with avg < min") + } +} + +func TestOpenRejectsMissingCDCParams(t *testing.T) { + pub, ident := testRecipient(t) + backend := memtest.NewMemoryBackend() + if _, _, err := Init(backend, []string{pub}, 1, 8, 64, false); err != nil { + t.Fatal(err) + } + // Simulate a Phase 1 repo.json that only had chunk_size. + raw := []byte(`{"version":"2.0","created_at":"now","recipients":["` + pub + `"],"chunk_size":4096}`) + backend.Put(ConfigPath, raw) + _, err := Open(backend, ident) + if err == nil { + t.Fatal("Open() succeeded without CDC params") + } + if !strings.Contains(err.Error(), "chunk") { + t.Fatalf("error = %v, want chunk param mention", err) + } +} + +func TestDataPathUsesHashSharding(t *testing.T) { + got := DataPath(strings.Repeat("ab", 32)) + want := "data/ab/" + strings.Repeat("ab", 32) + if got != want { + t.Fatalf("DataPath() = %q, want %q", got, want) + } +} + +func TestDataPathRejectsShortHash(t *testing.T) { + for _, in := range []string{"a", ""} { + got := DataPath(in) + if got != "data/invalid" { + t.Fatalf("DataPath(%q) = %q, want %q", in, got, "data/invalid") + } + } +} + +func TestSnapshotPathRejectsInvalidUUID(t *testing.T) { + if _, err := SnapshotPath("../escape"); err == nil { + t.Fatal("SnapshotPath() accepted invalid uuid") + } + got, err := SnapshotPath("7f91c6c7-7a0b-44aa-ae23-997b60e4e998") + if err != nil { + t.Fatalf("SnapshotPath() error: %v", err) + } + if got != "snapshots/7f91c6c7-7a0b-44aa-ae23-997b60e4e998.age" { + t.Fatalf("SnapshotPath() = %q", got) + } +} diff --git a/internal/storage/list.go b/internal/storage/list.go new file mode 100644 index 0000000..cc193dc --- /dev/null +++ b/internal/storage/list.go @@ -0,0 +1,19 @@ +package storage + +import "encoding/json" + +// ListEntry is the subset of rclone lsjson output that gloak uses. +type ListEntry struct { + Path string `json:"Path"` + IsDir bool `json:"IsDir"` +} + +// ParseList decodes rclone lsjson output, returning a JSON decoding error for +// malformed or incompatible input. Unknown fields are ignored. +func ParseList(raw []byte) ([]ListEntry, error) { + var entries []ListEntry + if err := json.Unmarshal(raw, &entries); err != nil { + return nil, err + } + return entries, nil +} diff --git a/internal/storage/memtest/memory.go b/internal/storage/memtest/memory.go new file mode 100644 index 0000000..73b02ec --- /dev/null +++ b/internal/storage/memtest/memory.go @@ -0,0 +1,159 @@ +// Package memtest provides an in-memory storage.Backend for tests. +package memtest + +import ( + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "sync" +) + +type MemoryBackend struct { + mu sync.Mutex + files map[string][]byte + uploads map[string]int +} + +func NewMemoryBackend() *MemoryBackend { + return &MemoryBackend{files: make(map[string][]byte), uploads: make(map[string]int)} +} + +func (m *MemoryBackend) Upload(remotePath string, in io.Reader) error { + data, err := io.ReadAll(in) + if err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + m.files[remotePath] = append([]byte(nil), data...) + m.uploads[remotePath]++ + return nil +} + +func (m *MemoryBackend) Download(remotePath string, out io.Writer) error { + m.mu.Lock() + data, ok := m.files[remotePath] + m.mu.Unlock() + if !ok { + return fmt.Errorf("not found: %s", remotePath) + } + _, err := out.Write(data) + return err +} + +// ListFiles mimics rclone lsjson: Path is relative to the listed prefix, +// only direct children are returned, directories are marked IsDir. +func (m *MemoryBackend) ListFiles(prefix []string) ([]byte, error) { + p := strings.Trim(strings.Join(prefix, "/"), "/") + m.mu.Lock() + defer m.mu.Unlock() + + children := map[string]bool{} + for key := range m.files { + var rel string + switch { + case p == "": + rel = key + case strings.HasPrefix(key, p+"/"): + rel = strings.TrimPrefix(key, p+"/") + default: + continue + } + if idx := strings.Index(rel, "/"); idx >= 0 { + children[rel[:idx]] = true + } else { + children[rel] = false + } + } + + names := make([]string, 0, len(children)) + for name := range children { + names = append(names, name) + } + sort.Strings(names) + + entries := make([]map[string]any, 0, len(names)) + for _, name := range names { + entries = append(entries, map[string]any{ + "Path": name, + "Name": name, + "IsDir": children[name], + "Size": 0, + }) + } + return json.Marshal(entries) +} + +// ListRecursive returns every stored file path under prefix, sorted. +func (m *MemoryBackend) ListRecursive(prefix []string) ([]string, error) { + p := strings.Trim(strings.Join(prefix, "/"), "/") + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for key := range m.files { + if p == "" || strings.HasPrefix(key, p+"/") { + out = append(out, key) + } + } + sort.Strings(out) + return out, nil +} + +func (m *MemoryBackend) Delete(remotePath string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.files, remotePath) + return nil +} + +func (m *MemoryBackend) Has(path string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.files[path] + return ok +} + +func (m *MemoryBackend) Put(path string, data []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.files[path] = append([]byte(nil), data...) +} + +func (m *MemoryBackend) Get(path string) []byte { + m.mu.Lock() + defer m.mu.Unlock() + return append([]byte(nil), m.files[path]...) +} + +func (m *MemoryBackend) UploadCount(path string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.uploads[path] +} + +func (m *MemoryBackend) TotalUploads(prefix string) int { + m.mu.Lock() + defer m.mu.Unlock() + total := 0 + for path, n := range m.uploads { + if strings.HasPrefix(path, prefix) { + total += n + } + } + return total +} + +func (m *MemoryBackend) KeysWithPrefix(prefix string) []string { + m.mu.Lock() + defer m.mu.Unlock() + var keys []string + for key := range m.files { + if strings.HasPrefix(key, prefix) { + keys = append(keys, key) + } + } + sort.Strings(keys) + return keys +} diff --git a/internal/storage/memtest/memory_test.go b/internal/storage/memtest/memory_test.go new file mode 100644 index 0000000..f4a6b71 --- /dev/null +++ b/internal/storage/memtest/memory_test.go @@ -0,0 +1,27 @@ +package memtest + +import ( + "bytes" + "testing" +) + +func TestListRecursiveReturnsNestedFiles(t *testing.T) { + m := NewMemoryBackend() + _ = m.Upload("data/ab/abhash", bytes.NewReader([]byte("x"))) + _ = m.Upload("data/cd/cdhash", bytes.NewReader([]byte("y"))) + _ = m.Upload("snapshots/s1.age", bytes.NewReader([]byte("z"))) + + got, err := m.ListRecursive([]string{"data"}) + if err != nil { + t.Fatal(err) + } + want := []string{"data/ab/abhash", "data/cd/cdhash"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got %v, want %v", got, want) + } + } +} diff --git a/internal/storage/rclone.go b/internal/storage/rclone.go index 2f72e77..555aacc 100644 --- a/internal/storage/rclone.go +++ b/internal/storage/rclone.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "errors" "fmt" "io" "os/exec" @@ -9,7 +10,8 @@ import ( ) type RcloneBackend struct { - Remote string + Remote string + // ConfigPath is optional. When empty, rclone uses its own default config. ConfigPath string } @@ -17,20 +19,44 @@ type Backend interface { Upload(remotePath string, in io.Reader) error Download(remotePath string, out io.Writer) error ListFiles(prefix []string) ([]byte, error) + ListRecursive(prefix []string) ([]string, error) Delete(remotePath string) error } +// NewRcloneBackend uses remote as the storage root and leaves rclone's default +// configuration in effect until ConfigPath is set. func NewRcloneBackend(remote string) *RcloneBackend { - return &RcloneBackend{ - Remote: remote, - ConfigPath: "/home/keraki/.config/rclone/rclone.conf", + return &RcloneBackend{Remote: remote} +} + +// rcloneArgs copies base and appends a --config option only when configPath is +// nonempty, leaving base unchanged. +func rcloneArgs(configPath string, base ...string) []string { + args := append([]string{}, base...) + if configPath != "" { + args = append(args, "--config", configPath) + } + return args +} + +// IsMissingPath reports whether err was caused by rclone exiting with +// status 3, which rclone uses when the remote path does not exist. +// Callers that treat a missing directory as an empty listing must check +// this so a genuine listing failure is never swallowed. +func IsMissingPath(err error) bool { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() == 3 } + return false } +// Upload streams in to remotePath relative to Remote, creating or replacing +// the object. It returns rclone execution errors, including input read failures. func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { fullDest := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "rcat", "--config", r.ConfigPath, fullDest) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "rcat", fullDest)...) cmd.Stdin = in @@ -43,10 +69,13 @@ func (r *RcloneBackend) Upload(remotePath string, in io.Reader) error { return nil } +// Download writes the object at remotePath relative to Remote to out. +// It returns rclone execution errors, including output write failures; out may +// have received partial data when an error is returned. func (r *RcloneBackend) Download(remotePath string, out io.Writer) error { fullSrc := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "cat", "--config", r.ConfigPath, fullSrc) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "cat", fullSrc)...) cmd.Stdout = out @@ -68,7 +97,7 @@ func (r *RcloneBackend) ListFiles(prefix []string) ([]byte, error) { remotePath = fmt.Sprintf("%s/%s", r.Remote, strings.Join(prefix, "/")) } - cmd := exec.Command("rclone", "lsjson", "--config", r.ConfigPath, remotePath) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "lsjson", remotePath)...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout @@ -80,16 +109,51 @@ func (r *RcloneBackend) ListFiles(prefix []string) ([]byte, error) { return stdout.Bytes(), nil } -// Delete removes a file or directory at the given remote path using rclone purge. +// ListRecursive lists every file under prefix, recursively. Paths are +// returned in the same remote-relative shape as Upload keys. +func (r *RcloneBackend) ListRecursive(prefix []string) ([]string, error) { + remotePath := r.Remote + if len(prefix) > 0 { + remotePath = fmt.Sprintf("%s/%s", r.Remote, strings.Join(prefix, "/")) + } + + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "lsjson", "-R", remotePath)...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("failed to list %s via rclone lsjson: %w\nRclone Error: %s", remotePath, err, stderr.String()) + } + + entries, err := ParseList(stdout.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to parse recursive listing: %w", err) + } + out := make([]string, 0, len(entries)) + for _, e := range entries { + if e.IsDir { + continue + } + if len(prefix) > 0 { + out = append(out, strings.Join(append(append([]string{}, prefix...), e.Path), "/")) + } else { + out = append(out, e.Path) + } + } + return out, nil +} + +// Delete removes a single file at the given remote path using rclone +// deletefile. Purge is for directories; deletefile is for files. func (r *RcloneBackend) Delete(remotePath string) error { fullPath := fmt.Sprintf("%s/%s", r.Remote, remotePath) - cmd := exec.Command("rclone", "purge", "--config", r.ConfigPath, fullPath) + cmd := exec.Command("rclone", rcloneArgs(r.ConfigPath, "deletefile", fullPath)...) var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to delete %s via rclone purge: %w\nRclone Error: %s", fullPath, err, stderr.String()) + return fmt.Errorf("failed to delete %s via rclone deletefile: %w\nRclone Error: %s", fullPath, err, stderr.String()) } return nil diff --git a/internal/storage/rclone_test.go b/internal/storage/rclone_test.go new file mode 100644 index 0000000..374e749 --- /dev/null +++ b/internal/storage/rclone_test.go @@ -0,0 +1,48 @@ +package storage + +import ( + "fmt" + "os/exec" + "reflect" + "testing" +) + +func TestRcloneArgsOmitsConfigWhenEmpty(t *testing.T) { + got := rcloneArgs("", "cat", "remote:file") + want := []string{"cat", "remote:file"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rcloneArgs() = %v, want %v", got, want) + } +} + +func TestRcloneArgsAppendsConfigWhenSet(t *testing.T) { + got := rcloneArgs("/etc/rclone.conf", "cat", "remote:file") + want := []string{"cat", "remote:file", "--config", "/etc/rclone.conf"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rcloneArgs() = %v, want %v", got, want) + } +} + +func TestIsMissingPathMatchesExitCodeThree(t *testing.T) { + // rclone exits 3 when the remote path is missing. + exitErr := exec.Command("sh", "-c", "exit 3").Run() + if exitErr == nil { + t.Fatal("sh exited 0, want non-zero") + } + if !IsMissingPath(fmt.Errorf("wrapped: %w", exitErr)) { + t.Fatal("IsMissingPath(wrapped exit 3) = false, want true") + } +} + +func TestIsMissingPathRejectsOtherErrors(t *testing.T) { + other := exec.Command("sh", "-c", "exit 1").Run() + if IsMissingPath(fmt.Errorf("wrapped: %w", other)) { + t.Fatal("IsMissingPath(exit 1) = true, want false") + } + if IsMissingPath(fmt.Errorf("plain failure")) { + t.Fatal("IsMissingPath(plain) = true, want false") + } + if IsMissingPath(nil) { + t.Fatal("IsMissingPath(nil) = true, want false") + } +} diff --git a/main.go b/main.go index 570390d..a1fc543 100644 --- a/main.go +++ b/main.go @@ -13,16 +13,29 @@ import ( var cli struct { Debug bool `help:"Enable debug mode." short:"d"` - Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` - Upload UploadCmd `cmd:"" help:"Encrypt, chunk, and upload a file to a storage destination."` - Resume ResumeCmd `cmd:"" help:"Validate saved state for an interrupted upload."` - Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` - Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` - Cleanup CleanupCmd `cmd:"" help:"Remove orphan backup directories missing manifest.age."` - Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` + Keygen KeygenCmd `cmd:"" help:"Generate an X25519 identity and recipient file."` + Init InitCmd `cmd:"" help:"Initialize a new gloak repository." group:"Repository"` + Backup BackupCmd `cmd:"" help:"Encrypt, chunk, and upload a file or directory to a repository."` + Upload UploadCmd `cmd:"" help:"Encrypt and upload a file as a standalone folder (no repository)."` + Restore RestoreCmd `cmd:"" help:"Restore and decrypt a backup from storage."` + Verify VerifyCmd `cmd:"" help:"Verify backup chunks and checksums."` + Snapshots SnapshotsCmd `cmd:"" help:"List snapshots in the repository."` + Prune PruneCmd `cmd:"" help:"Remove dead snapshots and unreferenced chunks (dry run by default)."` + Check CheckCmd `cmd:"" help:"Check repository consistency (structural)."` + Setup SetupCmd `cmd:"" help:"Download and install rclone to ~/.gloak/bin/."` } +// getDefaultDir returns the directory for identity, recipient, and helper +// files. GLOAK_DIR overrides the default (~/.gloak); ~ and a leading ~/ are +// expanded when the home directory is available. If home lookup fails, it +// returns GLOAK_DIR unchanged when set, or .gloak relative to the current directory. func getDefaultDir() string { + if dir := os.Getenv("GLOAK_DIR"); dir != "" { + if expanded, err := expandHome(dir); err == nil { + return expanded + } + return dir + } home, err := os.UserHomeDir() if err != nil { home = "." diff --git a/main_test.go b/main_test.go index e8798cc..d909073 100644 --- a/main_test.go +++ b/main_test.go @@ -48,15 +48,24 @@ func TestCommandOptionShortTags(t *testing.T) { }{ {name: "keygen output dir", cmd: KeygenCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "keygen force", cmd: KeygenCmd{}, fieldName: "Force", wantShort: "f"}, + {name: "init remote", cmd: InitCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "init recipient", cmd: InitCmd{}, fieldName: "Recipient", wantShort: "r"}, + {name: "init force", cmd: InitCmd{}, fieldName: "Force", wantShort: "f"}, {name: "upload recipient", cmd: UploadCmd{}, fieldName: "Recipient", wantShort: "r"}, {name: "upload remote", cmd: UploadCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "backup identity", cmd: BackupCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "backup remote", cmd: BackupCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore identity", cmd: RestoreCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "restore remote", cmd: RestoreCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "restore output dir", cmd: RestoreCmd{}, fieldName: "OutputDir", wantShort: "o"}, {name: "verify identity", cmd: VerifyCmd{}, fieldName: "Identity", wantShort: "i"}, {name: "verify remote", cmd: VerifyCmd{}, fieldName: "Remote", wantShort: "R"}, - {name: "cleanup remote", cmd: CleanupCmd{}, fieldName: "Remote", wantShort: "R"}, - {name: "cleanup yes", cmd: CleanupCmd{}, fieldName: "Yes", wantShort: "y"}, + {name: "snapshots identity", cmd: SnapshotsCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "snapshots remote", cmd: SnapshotsCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "prune identity", cmd: PruneCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "prune remote", cmd: PruneCmd{}, fieldName: "Remote", wantShort: "R"}, + {name: "check identity", cmd: CheckCmd{}, fieldName: "Identity", wantShort: "i"}, + {name: "check remote", cmd: CheckCmd{}, fieldName: "Remote", wantShort: "R"}, {name: "setup dir", cmd: SetupCmd{}, fieldName: "Dir", wantShort: "D"}, {name: "setup update", cmd: SetupCmd{}, fieldName: "Update", wantShort: "u"}, } @@ -73,3 +82,122 @@ func TestCommandOptionShortTags(t *testing.T) { }) } } + +func TestUploadCmdFileIsExistingFileArg(t *testing.T) { + field, ok := reflect.TypeOf(UploadCmd{}).FieldByName("File") + if !ok { + t.Fatal("UploadCmd.File missing") + } + if field.Tag.Get("type") != "existingfile" { + t.Fatalf("type tag = %q, want existingfile", field.Tag.Get("type")) + } + if _, ok := field.Tag.Lookup("arg"); !ok { + t.Fatal("File should be a positional arg") + } +} + +func TestBackupCmdPathArgTag(t *testing.T) { + field, ok := reflect.TypeOf(BackupCmd{}).FieldByName("ExistingFile") + if !ok { + t.Fatal("field missing") + } + if _, ok := field.Tag.Lookup("arg"); !ok { + t.Fatal("path should be a positional arg") + } + if got := field.Tag.Get("name"); got != "path" { + t.Fatalf("name tag = %q", got) + } + if strings.Contains(field.Tag.Get("type"), "existingfile") { + t.Fatal("existingfile type still set; directories must parse") + } +} + +func TestResolveInitRecipientsReadsFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "recipient.txt") + content := "# comment\n\nage1aaa\nage1bbb\n" + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("failed to write recipient file: %v", err) + } + + got, err := resolveInitRecipients(path) + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + want := []string{"age1aaa", "age1bbb"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveInitRecipients() = %v, want %v", got, want) + } +} + +func TestResolveInitRecipientsParsesInlineKeys(t *testing.T) { + got, err := resolveInitRecipients("age1aaa, age1bbb") + if err != nil { + t.Fatalf("resolveInitRecipients() error: %v", err) + } + want := []string{"age1aaa", "age1bbb"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("resolveInitRecipients() = %v, want %v", got, want) + } +} + +func TestResolveInitRecipientsEmptyFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "recipient.txt") + if err := os.WriteFile(path, []byte("# only a comment\n\n"), 0644); err != nil { + t.Fatalf("failed to write recipient file: %v", err) + } + + _, err := resolveInitRecipients(path) + if err == nil { + t.Fatal("resolveInitRecipients() succeeded with empty file") + } + if !strings.Contains(err.Error(), "no recipients") { + t.Fatalf("resolveInitRecipients() error = %q, want no recipients mention", err) + } +} + +func TestResolveInitRecipientsMissingFileErrors(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing.txt") + + _, err := resolveInitRecipients(path) + if err == nil { + t.Fatal("resolveInitRecipients() succeeded with missing file") + } + if !strings.Contains(err.Error(), "gloak keygen") { + t.Fatalf("resolveInitRecipients() error = %q, want gloak keygen mention", err) + } +} + +func TestGetDefaultDirUsesGLOAK_DIR(t *testing.T) { + dir := filepath.Join(t.TempDir(), "custom-gloak") + t.Setenv("GLOAK_DIR", dir) + + if got := getDefaultDir(); got != dir { + t.Fatalf("getDefaultDir() = %q, want %q", got, dir) + } +} + +func TestGetDefaultDirExpandsTildeInGLOAK_DIR(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("no home dir: %v", err) + } + t.Setenv("GLOAK_DIR", "~/custom-gloak") + + want := filepath.Join(home, "custom-gloak") + if got := getDefaultDir(); got != want { + t.Fatalf("getDefaultDir() = %q, want %q", got, want) + } +} + +func TestGetDefaultDirFallsBackToHomeWhenUnset(t *testing.T) { + t.Setenv("GLOAK_DIR", "") + + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("no home dir: %v", err) + } + want := filepath.Join(home, ".gloak") + if got := getDefaultDir(); got != want { + t.Fatalf("getDefaultDir() = %q, want %q", got, want) + } +} diff --git a/notify.go b/notify.go new file mode 100644 index 0000000..29c7da0 --- /dev/null +++ b/notify.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "os" + "strings" + "time" + + "github.com/pterm/pterm" + + "gloak/internal/config" + "gloak/internal/notify" +) + +// resolveTelegram merges Telegram credentials from the config file with +// nonempty environment overrides. The file at /config.toml provides +// the base values; GLOAK_TELEGRAM_TOKEN and GLOAK_TELEGRAM_CHAT_ID win when nonempty. +// Unreadable or malformed configuration is ignored, leaving environment values +// available for notifications. +func resolveTelegram() notify.TelegramConfig { + fileCfg, err := config.Load(config.Path(getDefaultDir())) + if err != nil { + pterm.Warning.Printf("ignoring config file: %v\n", err) + } + cfg := notify.TelegramConfig{ + Token: fileCfg.Telegram.Token, + ChatID: fileCfg.Telegram.ChatID, + } + if v := os.Getenv("GLOAK_TELEGRAM_TOKEN"); v != "" { + cfg.Token = v + } + if v := os.Getenv("GLOAK_TELEGRAM_CHAT_ID"); v != "" { + cfg.ChatID = v + } + return cfg +} + +// notifyCommand sends a short best-effort report after a successful command. +// Nonempty details are rendered as-is under the header. Missing credentials +// skip the send; send failures do not fail the host command. +func notifyCommand(command string, details ...string) { + cfg := resolveTelegram() + if !cfg.Enabled() { + return + } + var b strings.Builder + fmt.Fprintf(&b, "gloak %s completed", command) + for _, d := range details { + if d == "" { + continue + } + b.WriteByte('\n') + b.WriteString(d) + } + if err := cfg.Send(b.String()); err != nil { + pterm.Warning.Printf("telegram notify failed: %v\n", err) + } +} + +// formatDuration renders d in a short human form. +func formatDuration(d time.Duration) string { + if d < time.Second { + return d.Round(time.Millisecond).String() + } + return d.Round(time.Second).String() +} diff --git a/notify_test.go b/notify_test.go new file mode 100644 index 0000000..5be3458 --- /dev/null +++ b/notify_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func writeConfig(t *testing.T, content string) { + t.Helper() + dir := t.TempDir() + t.Setenv("GLOAK_DIR", dir) + if err := os.WriteFile(filepath.Join(dir, "config.toml"), []byte(content), 0600); err != nil { + t.Fatal(err) + } +} + +func TestResolveTelegramFromFile(t *testing.T) { + writeConfig(t, "[telegram]\ntoken = \"file-token\"\nchat_id = \"file-chat\"\n") + t.Setenv("GLOAK_TELEGRAM_TOKEN", "") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "") + + cfg := resolveTelegram() + if cfg.Token != "file-token" || cfg.ChatID != "file-chat" { + t.Fatalf("cfg = %+v", cfg) + } +} + +func TestResolveTelegramEnvOverridesFile(t *testing.T) { + writeConfig(t, "[telegram]\ntoken = \"file-token\"\nchat_id = \"file-chat\"\n") + t.Setenv("GLOAK_TELEGRAM_TOKEN", "env-token") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "env-chat") + + cfg := resolveTelegram() + if cfg.Token != "env-token" || cfg.ChatID != "env-chat" { + t.Fatalf("cfg = %+v, want env values", cfg) + } +} + +func TestResolveTelegramEmptyWithoutConfigOrEnv(t *testing.T) { + t.Setenv("GLOAK_DIR", t.TempDir()) + t.Setenv("GLOAK_TELEGRAM_TOKEN", "") + t.Setenv("GLOAK_TELEGRAM_CHAT_ID", "") + + cfg := resolveTelegram() + if cfg.Enabled() { + t.Fatalf("cfg = %+v, want disabled", cfg) + } +} diff --git a/prune.go b/prune.go new file mode 100644 index 0000000..4224276 --- /dev/null +++ b/prune.go @@ -0,0 +1,62 @@ +package main + +import ( + "fmt" + "path/filepath" + "time" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type PruneCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` + KeepLast int `help:"Keep the N newest snapshots." default:"0"` + KeepDaily int `help:"Keep the newest snapshot of each of the last N days." default:"0"` + KeepWeekly int `help:"Keep the newest snapshot of each of the last N weeks." default:"0"` + KeepMonthly int `help:"Keep the newest snapshot of each of the last N months." default:"0"` + Yes bool `help:"Actually delete. Without this flag prune is a dry run."` +} + +// Run prints a retention plan, applying deletions only when Yes is set. +// It returns prune errors and sends an optional notification after a successful apply. +func (c *PruneCmd) Run() error { + pterm.Info.Println("Starting PRUNE process...") + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + pterm.Info.Printf("Using identity from: %s\n", ident) + } + policy := flow.RetentionPolicy{ + KeepLast: c.KeepLast, + KeepDaily: c.KeepDaily, + KeepWeekly: c.KeepWeekly, + KeepMonthly: c.KeepMonthly, + } + start := time.Now() + res, err := flow.RunPrune(ident, c.Remote, policy, c.Yes) + if err != nil { + return err + } + pterm.Printf("Kept snapshots: %d\n", len(res.KeptSnapshots)) + pterm.Printf("Dead snapshots: %d\n", len(res.DeadSnapshots)) + pterm.Printf("Live chunks: %d\n", res.LiveChunks) + pterm.Printf("Garbage chunks: %d\n", res.GarbageChunks) + if !c.Yes { + pterm.Warning.Println("Dry run only. Pass --yes to delete.") + return nil + } + pterm.Success.Printf("Deleted %d snapshots and %d objects; compacted index: %v\n", + res.DeletedSnapshots, res.DeletedObjects, res.CompactedEpochs) + notifyCommand("prune", + fmt.Sprintf("Kept: %d snapshots", len(res.KeptSnapshots)), + fmt.Sprintf("Dead: %d snapshots", len(res.DeadSnapshots)), + fmt.Sprintf("Live chunks: %d", res.LiveChunks), + fmt.Sprintf("Garbage chunks: %d", res.GarbageChunks), + fmt.Sprintf("Deleted: %d snapshots, %d objects", res.DeletedSnapshots, res.DeletedObjects), + "Duration: "+formatDuration(time.Since(start)), + ) + return nil +} diff --git a/restore.go b/restore.go index c47baf2..7aca10d 100644 --- a/restore.go +++ b/restore.go @@ -2,6 +2,7 @@ package main import ( "path/filepath" + "time" "github.com/pterm/pterm" @@ -9,12 +10,16 @@ import ( ) type RestoreCmd struct { - UUID string `arg:"" name:"uuid" help:"Backup UUID to restore."` - Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` - Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` - OutputDir string `help:"Output directory for restored file." default:"./restored" short:"o"` + UUID string `arg:"" name:"uuid" help:"Backup UUID to restore."` + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` + OutputDir string `help:"Output directory for restored file." default:"./restored" short:"o"` + PathFilter string `help:"Restore only this relative path (and children) from a tree snapshot."` } +// Run restores the backup using the selected or default identity and sends an +// optional completion notification. Restore errors are returned, and failed +// restores may leave partial output. func (r *RestoreCmd) Run() error { pterm.Info.Println("Starting RESTORE process...") @@ -24,5 +29,18 @@ func (r *RestoreCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir) + start := time.Now() + if err := flow.RunRestore(r.UUID, ident, r.Remote, r.OutputDir, r.PathFilter); err != nil { + return err + } + details := []string{ + "Snapshot: " + r.UUID, + "Output: " + r.OutputDir, + } + if r.PathFilter != "" { + details = append(details, "Path filter: "+r.PathFilter) + } + details = append(details, "Duration: "+formatDuration(time.Since(start))) + notifyCommand("restore", details...) + return nil } diff --git a/resume.go b/resume.go deleted file mode 100644 index 3dfe6a9..0000000 --- a/resume.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/pterm/pterm" - - "gloak/internal/flow" -) - -type ResumeCmd struct { - UUID string `arg:"" name:"uuid" help:"Backup UUID to resume."` -} - -func (r *ResumeCmd) Run() error { - pterm.Info.Println("Starting RESUME process...") - return flow.RunResume(r.UUID) -} diff --git a/snapshots.go b/snapshots.go new file mode 100644 index 0000000..4c0b4ea --- /dev/null +++ b/snapshots.go @@ -0,0 +1,28 @@ +package main + +import ( + "path/filepath" + + "github.com/pterm/pterm" + + "gloak/internal/flow" +) + +type SnapshotsCmd struct { + Identity string `help:"Identity private key file (defaults to ~/.gloak/identity.txt)." short:"i"` + Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` +} + +// Run lists repository snapshots using the selected or default identity, +// returning repository access or snapshot loading errors. +func (c *SnapshotsCmd) Run() error { + pterm.Info.Println("Listing snapshots...") + ident := c.Identity + if ident == "" { + ident = filepath.Join(getDefaultDir(), "identity.txt") + } + if err := flow.RunSnapshots(ident, c.Remote); err != nil { + return err + } + return nil +} diff --git a/upload.go b/upload.go index fa901fb..760ef73 100644 --- a/upload.go +++ b/upload.go @@ -2,42 +2,32 @@ package main import ( "fmt" - "path/filepath" - "strings" "github.com/pterm/pterm" "gloak/internal/flow" + "gloak/internal/storage" ) +// UploadCmd writes a self-contained encrypted backup folder to a remote root. type UploadCmd struct { - File string `arg:"" help:"File to back up." type:"existingfile"` - Recipient string `help:"Recipient public key (age1...) or path to recipient.txt." short:"r"` - Remote string `help:"Destination Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` + File string `arg:"" name:"file" type:"existingfile" help:"File to back up"` + Recipient string `short:"r" help:"Recipient public key file or comma-separated keys (default ~/.gloak/recipient.txt)"` + Remote string `short:"R" required:"" help:"Remote root, e.g. keraaaki:backups or /mnt/backups"` } +// Run resolves recipients and uploads a standalone encrypted backup, printing +// its UUID on success. Recipient resolution and upload errors are returned. func (u *UploadCmd) Run() error { + recipients, err := resolveInitRecipients(u.Recipient) + if err != nil { + return err + } pterm.Info.Println("Starting UPLOAD process...") - - recip := u.Recipient - if recip == "" { - defaultRecipPath := filepath.Join(getDefaultDir(), "recipient.txt") - val, err := readKeyFile(defaultRecipPath, "age1") - if err != nil { - return fmt.Errorf("recipient not provided and %s could not be read: %w", defaultRecipPath, err) - } - recip = val - pterm.Info.Printf("Using recipient from: %s\n", defaultRecipPath) - } else if !strings.HasPrefix(recip, "age1") { - val, err := readKeyFile(recip, "age1") - if err != nil { - return fmt.Errorf("failed to read recipient file %s: %w", recip, err) - } - recip = val + backupID, err := flow.RunUpload(u.File, recipients, storage.NewRcloneBackend(u.Remote)) + if err != nil { + return fmt.Errorf("upload failed: %w", err) } - - pterm.DefaultBasicText.Printf("Target File : %s\n", u.File) - pterm.DefaultBasicText.Printf("Destination : %s\n", u.Remote) - - return flow.RunUpload(u.File, recip, u.Remote) + pterm.DefaultBasicText.Printf("Backup UUID: %s\n", backupID) + return nil } diff --git a/verify.go b/verify.go index a4980b4..6a088d8 100644 --- a/verify.go +++ b/verify.go @@ -2,6 +2,7 @@ package main import ( "path/filepath" + "time" "github.com/pterm/pterm" @@ -14,6 +15,9 @@ type VerifyCmd struct { Remote string `help:"Source Rclone URI (e.g., keraaaki:backup_folder or /mnt/backups)." required:"" short:"R"` } +// Run verifies the backup using the selected or default identity and sends an +// optional completion notification. Access, decryption, and integrity errors +// are returned. func (v *VerifyCmd) Run() error { pterm.Info.Println("Starting VERIFY process...") @@ -23,5 +27,13 @@ func (v *VerifyCmd) Run() error { pterm.Info.Printf("Using identity from: %s\n", ident) } - return flow.RunVerify(v.UUID, ident, v.Remote) + start := time.Now() + if err := flow.RunVerify(v.UUID, ident, v.Remote); err != nil { + return err + } + notifyCommand("verify", + "Snapshot: "+v.UUID, + "Duration: "+formatDuration(time.Since(start)), + ) + return nil }