From c87de4e6d380cbd5ccc3656bf2c0d7be31469cff Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:19:27 -0700 Subject: [PATCH 01/29] Install gdalcubes from GitHub now that CRAN archived it (#80) gdalcubes was archived on CRAN on 2026-09-16, which broke the pkgdown workflow's dependency resolve and user installs of the STAC functions. Upstream master (0.7.5) carries the filter_geom fix from our fork, so Remotes points at appelmar/gdalcubes with no suffix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- DESCRIPTION | 1 + README.md | 8 ++++++++ inst/notes/gdalcubes-pc-gotchas.md | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/DESCRIPTION b/DESCRIPTION index 47dfad2..ad7c9bf 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -53,6 +53,7 @@ Suggests: tidyterra, tmap Remotes: + appelmar/gdalcubes, bhaskarvk/leaflet.extras, NewGraphEnvironment/flooded, NewGraphEnvironment/gq diff --git a/README.md b/README.md index b0b053d..0e44e6f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,14 @@ Monitoring riparian vegetation loss matters for fish habitat, water quality, and pak::pak("NewGraphEnvironment/drift") ``` +The STAC fetch and cube functions (`dft_stac_fetch()`, `dft_stac_cube()`) also need +[gdalcubes](https://github.com/appelmar/gdalcubes), which was archived on CRAN on +2026-09-16. Install it from GitHub until it is back: + +```r +pak::pak("appelmar/gdalcubes") +``` + ## Quick start ```r diff --git a/inst/notes/gdalcubes-pc-gotchas.md b/inst/notes/gdalcubes-pc-gotchas.md index 4500666..9e4cdf3 100644 --- a/inst/notes/gdalcubes-pc-gotchas.md +++ b/inst/notes/gdalcubes-pc-gotchas.md @@ -10,6 +10,12 @@ Provenance, because it is now mixed: the #30-era bullets were verified on (`8bad203`), which is 0.7.4 plus the `filter_geom` segfault fix — with terra 1.9.34. Each bullet says which. +**Where gdalcubes comes from now (#80, 2026-09-28).** gdalcubes was archived on +CRAN on 2026-09-16. drift lists `appelmar/gdalcubes` in `Remotes:` with no suffix, +and upstream master (0.7.5, `ed68331`) has merged the `filter_geom` fix (their +PR #111), so the `NewGraphEnvironment/gdalcubes@newgraph` fork is no longer needed. +Do not install the fork over 0.7.5. + - **`gdalcubes::filter_geom()` is not worth using — now for measured reasons, not because it crashes (#47).** The original defect was a segfault in the compute worker (`gc_exec_worker`, `address 0x120`) or, intermittently, a silent all-NA From 98df81bf5a0d9a45049f81890c266eea13d4c01c Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:32:05 -0700 Subject: [PATCH 02/29] Point the network e2e cache assertions at the #48 scheme dir (#80) Since #48 the cache lives under /v2/, but five opt-in network assertions still listed / and so failed on every run. Found verifying gdalcubes 0.7.5; every data assertion passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- tests/testthat/test-dft_stac_cube.R | 6 +++--- tests/testthat/test-dft_stac_fetch.R | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index cb871fc..e535f7b 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -401,8 +401,8 @@ test_that("dft_stac_cube fetches an index stack end-to-end", { # and coverage is not one surviving pixel. Loose on purpose: cloud masking # legitimately removes a lot, so this guards the degenerate case, not quality. expect_lt(mean(is.na(vals[inpoly, , drop = FALSE])), 0.9) - # second call hits the cache (one cube_.tif under the source dir) - expect_length(list.files(file.path(cache, "sentinel-2-l2a"), + # the call cached exactly one cube_.tif under the source dir (#48 scheme) + expect_length(list.files(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), pattern = "^cube_.*\\.tif$"), 1) }) @@ -441,7 +441,7 @@ test_that("dft_stac_cube tiled read reproduces the untiled cube over the AOI", { expect_equal(terra::nlyr(tiled), terra::nlyr(untiled)) # same monthly axis expect_false(anyNA(terra::time(tiled))) # time set per layer # tiled and untiled each cache one cube_.tif, keyed apart (2 files total) - expect_length(list.files(file.path(cache, "sentinel-2-l2a"), + expect_length(list.files(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), pattern = "^cube_.*\\.tif$"), 2) # the efficiency claim: for this diagonal reach the tiled read streams fewer diff --git a/tests/testthat/test-dft_stac_fetch.R b/tests/testthat/test-dft_stac_fetch.R index 4f202a4..85e5d80 100644 --- a/tests/testthat/test-dft_stac_fetch.R +++ b/tests/testthat/test-dft_stac_fetch.R @@ -616,7 +616,7 @@ test_that("dft_stac_fetch tiled result matches untiled over the AOI", { cfg$stac_url, cfg$collection, cfg$asset, tile_size = NULL ) ) - expect_length(list.files(file.path(cache, "io-lulc"), + expect_length(list.files(drift:::cache_scheme_dir(cache, "io-lulc"), pattern = paste0("^2020_", attr(untiled_list, "cache_key"), "\\.nc$")), 1) # small tile_size relative to the AOI bbox → several tiles, most bbox-only @@ -629,9 +629,9 @@ test_that("dft_stac_fetch tiled result matches untiled over the AOI", { expect_s4_class(tiled, "SpatRaster") expect_equal(terra::nlyr(tiled), 1L) # extension routing: untiled caches a gdalcubes .nc, tiled a terra .tif - expect_length(list.files(file.path(cache, "io-lulc"), + expect_length(list.files(drift:::cache_scheme_dir(cache, "io-lulc"), pattern = "^2020_.*\\.nc$"), 1) - expect_length(list.files(file.path(cache, "io-lulc"), + expect_length(list.files(drift:::cache_scheme_dir(cache, "io-lulc"), pattern = "^2020_.*\\.tif$"), 1) # tiled == untiled over their common in-AOI cells: tiling changes only which # bbox pixels are streamed, not the classification. Put the tiled mosaic onto From 431577f011e26da3464eb479217cc5747cdc0754 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:50:23 -0700 Subject: [PATCH 03/29] Name the GitHub install when gdalcubes is missing (#80) rlang::check_installed() offered a bare-name install, which now means CRAN and fails. The three gdalcubes guards share one helper whose error gives pak::pak("appelmar/gdalcubes"). Found by code-check round 1. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- NEWS.md | 4 ++++ R/dft_index_expr.R | 2 +- R/dft_stac_cube.R | 2 +- R/dft_stac_fetch.R | 22 +++++++++++++++++++++- tests/testthat/test-dft_stac_fetch.R | 16 ++++++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index b3c09d7..87277b2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# drift 0.17.1 + +- **gdalcubes installs from GitHub (#80).** gdalcubes was archived on CRAN on 2026-09-16. That broke the pkgdown workflow's dependency resolve, which had failed on every push since 2026-09-26, and it left new users unable to install what `dft_stac_fetch()` and `dft_stac_cube()` need. `appelmar/gdalcubes` is now in `Remotes:` with no suffix, and the README says to install it with `pak::pak("appelmar/gdalcubes")`. So does the prompt `dft_stac_fetch()`, `dft_stac_cube()` and `dft_index_expr()` raise when it is missing. rlang's own install offer means CRAN and would fail. Upstream master (0.7.5) has merged the `filter_geom` segfault fix from `NewGraphEnvironment/gdalcubes@newgraph`, so the fork is no longer needed. On 0.7.5 the offline suite passes (1003 tests), and so does the opt-in network end-to-end for the cube and fetch paths (109 and 165). That run also surfaced five network assertions that had listed the pre-#48 cache path since that change, so they failed on every opt-in run. They now use `cache_scheme_dir()`. + # drift 0.17.0 - **The article gains patch shape and location, and the corridor question comes back negative (#73).** [What a Land-Cover Change Figure Is Made Of](https://newgraphenvironment.github.io/drift/articles/temporal-composition.html) reported *how much* of a two-epoch comparison is temporally unstable and never raised the spatial axis. It now reports both grains. Body prose 731 -> 1469 words against a cap raised 1000 -> 1500; the cap is a variable now, because the literal appeared twice on one line and changing one would have left the message lying. diff --git a/R/dft_index_expr.R b/R/dft_index_expr.R index f117fca..51ecd16 100644 --- a/R/dft_index_expr.R +++ b/R/dft_index_expr.R @@ -50,7 +50,7 @@ dft_index_expr <- function(cube, roles = NULL, scale = NULL, offset = NULL) { - rlang::check_installed("gdalcubes", reason = "to apply an index to a cube") + check_gdalcubes("to apply an index to a cube") cfg <- dft_stac_config(source) roles <- roles %||% cfg$roles scale <- scale %||% cfg$scale %||% 1 diff --git a/R/dft_stac_cube.R b/R/dft_stac_cube.R index 537a306..bed2f55 100644 --- a/R/dft_stac_cube.R +++ b/R/dft_stac_cube.R @@ -162,7 +162,7 @@ dft_stac_cube <- function(aoi, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()) { - rlang::check_installed("gdalcubes", reason = "to fetch STAC cubes") + check_gdalcubes("to fetch STAC cubes") # gdalcubes worker processes for the read. drift never set this before v0.9.0, # so every fetch ran single-threaded — not a considered choice, just the diff --git a/R/dft_stac_fetch.R b/R/dft_stac_fetch.R index 7039fbf..10584ee 100644 --- a/R/dft_stac_fetch.R +++ b/R/dft_stac_fetch.R @@ -98,7 +98,7 @@ dft_stac_fetch <- function(aoi, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()) { - rlang::check_installed("gdalcubes", reason = "to fetch STAC rasters") + check_gdalcubes("to fetch STAC rasters") # Normalize tile_size ONCE so the path gate (is.null) and the cache key derive # from the same snapped scalar. When tiling, tune GDAL for the many extra @@ -863,6 +863,26 @@ cache_hit_ok <- function(cache_file, label) { } +#' Require gdalcubes, naming where to get it +#' +#' gdalcubes was archived on CRAN on 2026-09-16 (#80). `rlang::check_installed()` +#' offers to install a missing package by bare name, which now means CRAN and +#' fails, and it never reads drift's `Remotes:`. So this aborts with the GitHub +#' install instead, for anyone who meets it before the README. +#' @noRd +check_gdalcubes <- function(reason) { + if (gdalcubes_available()) return(invisible(TRUE)) + cli::cli_abort(c( + "The {.pkg gdalcubes} package is required {reason}.", + "i" = "It was archived on CRAN on 2026-09-16. Install it from GitHub:", + " " = "{.code pak::pak(\"appelmar/gdalcubes\")}" + )) +} + +#' Whether gdalcubes can be loaded (a seam, so tests can simulate its absence) +#' @noRd +gdalcubes_available <- function() requireNamespace("gdalcubes", quietly = TRUE) + #' Auto-detect UTM EPSG code from sf geometry #' @noRd auto_utm_epsg <- function(x) { diff --git a/tests/testthat/test-dft_stac_fetch.R b/tests/testthat/test-dft_stac_fetch.R index 85e5d80..0454a6f 100644 --- a/tests/testthat/test-dft_stac_fetch.R +++ b/tests/testthat/test-dft_stac_fetch.R @@ -19,6 +19,22 @@ test_that("auto_utm_epsg handles prime meridian", { expect_equal(epsg, "EPSG:32631") }) +test_that("a missing gdalcubes names the GitHub install, not only CRAN", { + # gdalcubes is archived on CRAN (#80), so a bare install offer fails; the + # message has to say where it lives now. The tests below skip whenever + # gdalcubes is installed, so this simulates its absence through the seam. + testthat::local_mocked_bindings(gdalcubes_available = function() FALSE) + expect_error(drift:::check_gdalcubes("to fetch STAC rasters"), + "appelmar/gdalcubes", fixed = TRUE) + expect_error(drift:::check_gdalcubes("to fetch STAC rasters"), + "to fetch STAC rasters", fixed = TRUE) + aoi <- sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift"), + quiet = TRUE) + expect_error(dft_stac_fetch(aoi, source = "io-lulc", years = 2020), + "appelmar/gdalcubes", fixed = TRUE) + expect_error(dft_stac_cube(aoi), "appelmar/gdalcubes", fixed = TRUE) +}) + test_that("dft_stac_fetch requires gdalcubes", { skip_if(requireNamespace("gdalcubes", quietly = TRUE), "gdalcubes is installed, can't test missing-package path") From ed8c7e0886239db58e5950490400fe163f2450e9 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:55:35 -0700 Subject: [PATCH 04/29] Correct the test count in the 0.17.1 NEWS entry (#80) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 87277b2..27a4a45 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # drift 0.17.1 -- **gdalcubes installs from GitHub (#80).** gdalcubes was archived on CRAN on 2026-09-16. That broke the pkgdown workflow's dependency resolve, which had failed on every push since 2026-09-26, and it left new users unable to install what `dft_stac_fetch()` and `dft_stac_cube()` need. `appelmar/gdalcubes` is now in `Remotes:` with no suffix, and the README says to install it with `pak::pak("appelmar/gdalcubes")`. So does the prompt `dft_stac_fetch()`, `dft_stac_cube()` and `dft_index_expr()` raise when it is missing. rlang's own install offer means CRAN and would fail. Upstream master (0.7.5) has merged the `filter_geom` segfault fix from `NewGraphEnvironment/gdalcubes@newgraph`, so the fork is no longer needed. On 0.7.5 the offline suite passes (1003 tests), and so does the opt-in network end-to-end for the cube and fetch paths (109 and 165). That run also surfaced five network assertions that had listed the pre-#48 cache path since that change, so they failed on every opt-in run. They now use `cache_scheme_dir()`. +- **gdalcubes installs from GitHub (#80).** gdalcubes was archived on CRAN on 2026-09-16. That broke the pkgdown workflow's dependency resolve, which had failed on every push since 2026-09-26, and it left new users unable to install what `dft_stac_fetch()` and `dft_stac_cube()` need. `appelmar/gdalcubes` is now in `Remotes:` with no suffix, and the README says to install it with `pak::pak("appelmar/gdalcubes")`. So does the prompt `dft_stac_fetch()`, `dft_stac_cube()` and `dft_index_expr()` raise when it is missing. rlang's own install offer means CRAN and would fail. Upstream master (0.7.5) has merged the `filter_geom` segfault fix from `NewGraphEnvironment/gdalcubes@newgraph`, so the fork is no longer needed. On 0.7.5 the offline suite passes (1007 tests), and so does the opt-in network end-to-end for the cube and fetch paths (109 and 165). That run also surfaced five network assertions that had listed the pre-#48 cache path since that change, so they failed on every opt-in run. They now use `cache_scheme_dir()`. # drift 0.17.0 From d1ee76639ee16d681d29e79b4f10c3fd1cbb165f Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:55:35 -0700 Subject: [PATCH 05/29] Release v0.17.1 (#80) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ad7c9bf..35718c9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: drift Title: Detecting Riparian and Inland Floodplain Transitions -Version: 0.17.0 +Version: 0.17.1 Date: 2026-09-06 Authors@R: c( person("Allan", "Irvine", , "al@newgraphenvironment.com", role = c("aut", "cre"), From 0d6a43a4bc07b5596e08b54407bc49559f5cb5cb Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:21:06 -0700 Subject: [PATCH 06/29] Initialize PWF baseline for #79 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- planning/active/findings.md | 35 ++++++++++++++++ planning/active/progress.md | 8 ++++ planning/active/task_plan.md | 80 ++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 planning/active/findings.md create mode 100644 planning/active/progress.md create mode 100644 planning/active/task_plan.md diff --git a/planning/active/findings.md b/planning/active/findings.md new file mode 100644 index 0000000..f4733a0 --- /dev/null +++ b/planning/active/findings.md @@ -0,0 +1,35 @@ +# Findings — Dated reference imagery (#79) + +## Issue context + +## Problem + +Checking a land-cover label or a detected change needs **dated reference imagery for windows we choose**: the same season in different years, or early vs late season within a year. The basemaps in `dft_map_interactive()` (Esri, Google, and Bing in #78) are high resolution but have no date, so they can say what a place is and not when it changed. + +drift already fetches Sentinel-2 L2A cubes (`dft_stac_cube()`, with cloud masking, a `months` filter and median aggregation), but only as a **single-index stack** (`ndvi`, `kndvi`, `ndmi`). Three pieces are missing to use it as a reference-imagery source. floodplains#93 needs them for a stratified accuracy assessment of IO LULC in BC floodplains. + +## Scope + +1. **True-colour (and false-colour) composites.** Return a multi-band composite (e.g. B04/B03/B02, or B08/B04/B03 for vegetation) for a `datetime` window and `months`, through the same cloud mask, cache and tiling path as `dft_stac_cube()`. The new_graphiti post `2026-01-08-stac-ortho-mosaics` did this outside drift with rstac + gdalcubes (June–July, `eo:cloud_cover <= 20`, median). That code is the starting point, and should not become a second copy. +2. **Water and wetness indices.** Add NDWI (green/NIR) and MNDWI (green/SWIR16) to `dft_index_table()`. `ndmi` is there already. These are the direct indices for wetland drying and water extent. +3. **RGB layers in `dft_map_interactive()`.** Today `x` is classified rasters coloured through `class_table`. Accept RGB composites as switchable overlays too, with labels such as `"2017 Jun–Jul"` and `"2023 Aug–Sep"`, alongside the classified layers and transitions. +4. **HLS as a second cube source (can split off).** Harmonized Landsat Sentinel-2 through NASA Earthdata (`earthdatalogin`): 30 m, from 2013, 2–3 day revisit. It gives a pre-2017 baseline and the seasonal density Sentinel-2 alone lacks. It needs an auth path distinct from Planetary Computer's `sign_fn`. + +## Acceptance + +- A documented call returns a cloud-masked, median true-colour composite for an AOI, a year and a set of months. It is cached, and the cache key includes the bands and months. +- `dft_index_table()` includes `ndwi` and `mndwi`, with tests against known band values. +- `dft_map_interactive()` shows at least two dated RGB composites as switchable layers over the Esri/Google basemaps. +- HLS is either delivered or filed as its own issue, with the auth path worked out. + +## Related issues re-read at the plan gate (2026-09-28) + +- **drift#80:** gdalcubes was archived on CRAN on 2026-09-16. Upstream `appelmar/gdalcubes` master (0.7.5, `ed68331`) has merged our `filter_geom` fix (their PR #111). Handled in its own PR first. +- **drift#81:** the Olofsson 2014 sampler and estimators. It supplies the sample points that the reference imagery is reviewed at, and lists the review UI as out of scope (it points to #79). +- **floodplains#93:** "for each sample point: a map zoomed to the point, the dated composites as switchable layers". The review runs over whole floodplains, not a reach, which is why this issue has to hold at floodplain scale. +- The new_graphiti post `2026-01-08-stac-ortho-mosaics` is the starting point: rstac, bbox search, `eo:cloud_cover <= 20`, B04/B03/B02, `dt = P2M`, median, `leafem::addRasterRGB(quantiles = c(0.02, 0.98))`. It has no cloud mask and no offset handling, and stretches each layer on its own. + +## Errors Encountered + +| Error | Resolution | +|-------|------------| diff --git a/planning/active/progress.md b/planning/active/progress.md new file mode 100644 index 0000000..46639c5 --- /dev/null +++ b/planning/active/progress.md @@ -0,0 +1,8 @@ +# Progress — Dated reference imagery (#79) + +## Session 2026-09-28 + +- Plan-mode exploration. The phases were approved after re-reading #79-#81 and floodplains#93, which pivoted the plan to #80 first and to floodplain scale. +- Created branch `79-dated-reference-imagery-true-colour-comp` off main, in a worktree (`../drift-79`) while the #80 suites ran in the main tree. +- Scaffolded the PWF baseline with the approved phases. +- Next: Phase 1 (NDWI/MNDWI). diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md new file mode 100644 index 0000000..b5b2736 --- /dev/null +++ b/planning/active/task_plan.md @@ -0,0 +1,80 @@ +# Task: Dated reference imagery: true-colour composites, NDWI/MNDWI, and RGB layers in dft_map_interactive (+ HLS source) (#79) + +Checking a land-cover label or a detected change needs **dated reference imagery for windows we choose**: the same season in different years, or early vs late season within a year. The basemaps in `dft_map_interactive()` (Esri, Google, and Bing in #78) are high resolution but have no date, so they can say what a place is and not when it changed. + +drift already fetches Sentinel-2 L2A cubes (`dft_stac_cube()`, with cloud masking, a `months` filter and median aggregation), but only as a **single-index stack** (`ndvi`, `kndvi`, `ndmi`). Three pieces are missing to use it as a reference-imagery source. floodplains#93 needs them for a stratified accuracy assessment of IO LULC in BC floodplains. + +Plan-gate decisions (2026-09-28): + +- #80 (gdalcubes archived on CRAN) lands first as its own PR. This branch rebases onto main once it merges. +- The work has to hold at floodplain scale, because floodplains#93 reviews sample points across whole floodplains. There are two paths: per-point chips (buffered points plus `tile_size`), and COG composites served through titiler. +- Composites stay float reflectance, so they can later serve as classifier covariates. +- `dft_stac_composite()` is a new export that shares its internals with `dft_stac_cube()`. +- HLS gets a spike and its own issue. Re-read #79 before that phase. + +## Phase 1: NDWI and MNDWI (independent, lowest risk) + +- [ ] Add `green = "B03"` and `blue = "B02"` to the `sentinel-2-l2a` roles (`R/dft_stac_config.R`), and update its roxygen. +- [ ] Add `ndwi` `(green - nir) / (green + nir)` (McFeeters) and `mndwi` `(green - swir16) / (green + swir16)` (Xu) to `inst/indices/indices.csv`. +- [ ] Tests (`test-dft_index_expr.R`): both rows ship. Each resolved expression, evaluated in R over known band values, gives the hand value. Examples: green 0.1 / nir 0.3 → −0.5, including through S2 scale/offset tokens. `mndwi` resolves `swir16` → `B11`. Existing cube cache keys are unchanged, and the frozen-key test stays green. + +## Phase 2: extract shared cube internals (refactor, no behaviour change) + +- [ ] Move out of `R/dft_stac_cube.R`: + - GDAL/gdalcubes session setup and restore; + - STAC query, `months` filter and offset split (`stac_cube_items()`); + - extent → cube_view → pre/post `cover` → tiling/mosaic (`stac_cube_assemble()`), taking a pixel function. +- [ ] `dft_stac_cube()` calls these helpers and passes a `dft_index_expr()` closure as its pixel function. +- [ ] Guard: the frozen legacy key and every offline cube test pass unchanged. Run the network e2e (untiled + tiled) before and after, and record both. + +## Phase 3: `dft_stac_composite()` + +- [ ] `R/dft_stac_composite.R` with this signature: `aoi, years, months = 6:7, bands = c("red","green","blue"), source = "sentinel-2-l2a", res = 10, crs = NULL, aggregation = "median", resampling = "bilinear", clip = TRUE, cloud_cover_max = 20, mask_values = NULL, tile_size = NULL, parallel = NULL, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()`. `bands` takes any roles and any number of them. The map needs exactly 3. +- [ ] Each year gets one window: `t0` = 1st of `min(months)`, `t1` = end of `max(months)`, `dt = P{span}M`. Items are filtered to `months`. The result is one time step: a median, cloud-masked **reflectance** stack (scale/offset per band via `scale_token()`) with layers named by role. +- [ ] Return a named list labelled like `"2017 Jun–Jul"`. A single year returns a length-1 list. +- [ ] The cache is `/composite_.tif`, **written as a COG** (`filetype = "COG"`) so the cached file can go to S3 and titiler unchanged. It is written through `cache_write_atomic()`, and read back only after `cache_hit_ok()` and the non-empty check pass. `stac_composite_cache_key()` covers geometry, res, crs, bands (resolved assets, order-sensitive), months (sorted), year window, cloud_cover_max, mask_values, scale/offset, clip and tile_size. +- [ ] **Per-point chips.** A documented example and a test show the chip path: `aoi` = sample points buffered to polygons, `tile_size` set. Through `tile_grid()` only tiles that intersect the points stream. An offline test on the grid confirms scattered buffered points select a small share of tiles. +- [ ] Validation: an unknown band role errors with the available roles; empty `years`/`months` or months outside 1–12 abort; a non-cube source aborts. +- [ ] Tests (offline): key determinism and sensitivity; labels (range, single month, non-contiguous); window derivation, including February's end in a leap year; validation; the pixel expression. One network e2e on the packaged AOI: 3 bands, plausible reflectance, a valid COG (`gdalinfo` layout, not an exit code), cached on the second call. +- [ ] Roxygen with true-colour and false-colour (`c("nir","red","green")`) examples. Run `devtools::document()` and `pkgdown::check_pkgdown()`. + +## Phase 4: RGB layers in `dft_map_interactive()` + +- [ ] New `rgb = NULL` argument: a named list of 3-band SpatRasters **or** a named character vector of COG URLs. That mirrors `x`'s dual mode. + - Local: `leafem::addRasterRGB()`. leafem is already in Suggests; call `check_installed()` only when it is used. + - COG: a titiler URL with `bidx=1&bidx=2&bidx=3&rescale=…`, built by a sibling of `build_titiler_url()`. +- [ ] Every RGB layer gets **one shared stretch**, so a brightness difference between years is real: + - local mode uses a pooled 2–98% `domain` across all composites; + - COG mode uses an `rgb_rescale` argument, defaulting to a documented reflectance range. +- [ ] RGB groups are switchable overlays beneath the classified and transition layers. The first is visible when `x` is `NULL`, and they are hidden otherwise. `x` may be `NULL` when `rgb` is given. Centring and legends handle that case. +- [ ] Tests: synthetic 3-band rasters; groups in the layer control; `x = NULL`; no land-cover legend without `x`; a non-3-band layer errors by name; the shared domain is applied; the COG URL has 3 `bidx` and the shared rescale, is percent-encoded, and errors without `titiler_url`. +- [ ] Live check: 2017 Jun–Jul and 2023 Aug–Sep on the packaged AOI over Esri/Google. Save the widget, view it in the browser, and run the cartography self-review on the screenshot. + +## Phase 5: HLS spike → own issue; reconcile #79 + +- [ ] Re-read #79, since scope may have been edited again. +- [ ] Probe live: + - `earthdatalogin` netrc / GDAL config; + - a CMR-STAC `LPCLOUD` `HLSS30.v2.0` / `HLSL30.v2.0` search via rstac; + - one COG read through gdalcubes; + - S30 vs L30 band names (NIR `B8A` vs `B05`, and so on); + - the Fmask **bit** mask against SCL classes (`image_mask(bits=)`?). + Record the results in `findings.md`. +- [ ] File the HLS issue with the worked auth path and source-config shape. Edit the #79 body: point item 4 at it, and add the scale pivot (chips and COG) and the #80 dependency. + +## Phase 6: scale test, docs, release + +- [ ] BULK scale test with the RSS sampler. Record wall time and peak RSS for the PR: + - `dft_stac_composite()` on `bulk_co_ff04`, one year, `tile_size` set; + - a chip run over ~100 scattered buffered points. + Also note what `dft_map_interactive(rgb=)` does with a floodplain-size local raster (maxBytes), to confirm the doc points to the COG path. +- [ ] Update NEWS.md, the CLAUDE.md Core Pipeline (the composite call and the chip pattern), and the gotchas note if anything turns up. +- [ ] Bump the version (minor) as the final commit. + +## Validation + +- [ ] Tests pass (`devtools::test()`, plus `DRIFT_TEST_NETWORK=true` e2e for cube and composite) +- [ ] `lintr::lint_package()` clean; `pkgdown::check_pkgdown()` passes +- [ ] `/code-check` clean on each commit +- [ ] PWF checkboxes match landed work +- [ ] `/planning-archive` on completion From c29f4f3ee1173ad53a262b990f7a7dcddc5e3c1c Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:22:12 -0700 Subject: [PATCH 07/29] Add NDWI and MNDWI to the index registry (#79) Adds the green and blue Sentinel-2 roles (B03, B02) that the water indices and true-colour composites need. Tests evaluate the resolved expressions over known reflectance and over post-2022 DN through the offset, so they check the value and not only the string. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_config.R | 7 +++-- inst/indices/indices.csv | 2 ++ planning/active/progress.md | 1 + planning/active/task_plan.md | 6 ++-- tests/testthat/test-dft_index_expr.R | 43 ++++++++++++++++++++++++++- tests/testthat/test-dft_stac_config.R | 4 ++- 6 files changed, 56 insertions(+), 7 deletions(-) diff --git a/R/dft_stac_config.R b/R/dft_stac_config.R index dbc110d..32ab187 100644 --- a/R/dft_stac_config.R +++ b/R/dft_stac_config.R @@ -9,7 +9,7 @@ #' `"esa-worldcover"`) host single-band classified rasters and carry a flat #' `asset` name for [dft_stac_fetch()]. **Cube** sources (`"sentinel-2-l2a"`) #' host multi-band reflectance imagery and instead carry a role-based band map -#' (`red`/`nir`/`swir16`/`mask`), mask values, and reflectance scale/offset for +#' (`red`/`green`/`blue`/`nir`/`swir16`/`mask`), mask values, and reflectance scale/offset for #' [dft_stac_cube()]; they are marked with `cube = TRUE`. The role-based schema #' means a new reflectance source (e.g. Landsat C2 L2) drops in with no API #' change — only the role→asset map and scale/offset differ. @@ -20,7 +20,8 @@ #' #' @return A list. Categorical sources have elements `stac_url`, `collection`, #' `asset`, `available_years`. Cube sources have `stac_url`, `collection`, -#' `cube = TRUE`, `roles` (a named list mapping `red`/`nir`/`swir16`/`mask` +#' `cube = TRUE`, `roles` (a named list mapping +#' `red`/`green`/`blue`/`nir`/`swir16`/`mask` #' to asset names), `mask_values` (integer mask classes to exclude), #' `scale`/`offset` (DN → reflectance affine transform), and #' `available_datetime` (an ISO 8601 interval string). The `cube` field is @@ -57,6 +58,8 @@ dft_stac_config <- function(source = c("io-lulc", "esa-worldcover", # Planetary Computer asset names (NOT the Element84/AWS red/nir/scl names) roles = list( red = "B04", + green = "B03", + blue = "B02", nir = "B08", swir16 = "B11", mask = "SCL" diff --git a/inst/indices/indices.csv b/inst/indices/indices.csv index 26f0c21..e0a5e62 100644 --- a/inst/indices/indices.csv +++ b/inst/indices/indices.csv @@ -2,3 +2,5 @@ index,formula,roles,description ndvi,"(nir - red) / (nir + red)","nir,red",Normalized Difference Vegetation Index kndvi,"tanh(pow((nir - red) / (nir + red), 2))","nir,red",Kernel NDVI (tanh of NDVI squared) ndmi,"(nir - swir16) / (nir + swir16)","nir,swir16",Normalized Difference Moisture Index +ndwi,"(green - nir) / (green + nir)","green,nir",Normalized Difference Water Index (McFeeters 1996) +mndwi,"(green - swir16) / (green + swir16)","green,swir16",Modified NDWI (Xu 2006) diff --git a/planning/active/progress.md b/planning/active/progress.md index 46639c5..37033c2 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -6,3 +6,4 @@ - Created branch `79-dated-reference-imagery-true-colour-comp` off main, in a worktree (`../drift-79`) while the #80 suites ran in the main tree. - Scaffolded the PWF baseline with the approved phases. - Next: Phase 1 (NDWI/MNDWI). +- Phase 1 done. Added the `green`/`blue` S2 roles and the `ndwi`/`mndwi` registry rows. The tests evaluate the resolved expressions over known reflectance, and over post-2022 DN through the offset. A mutation check swapping NDWI's sign failed 4 tests. The frozen cube key is unchanged. diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md index b5b2736..74381c4 100644 --- a/planning/active/task_plan.md +++ b/planning/active/task_plan.md @@ -14,9 +14,9 @@ Plan-gate decisions (2026-09-28): ## Phase 1: NDWI and MNDWI (independent, lowest risk) -- [ ] Add `green = "B03"` and `blue = "B02"` to the `sentinel-2-l2a` roles (`R/dft_stac_config.R`), and update its roxygen. -- [ ] Add `ndwi` `(green - nir) / (green + nir)` (McFeeters) and `mndwi` `(green - swir16) / (green + swir16)` (Xu) to `inst/indices/indices.csv`. -- [ ] Tests (`test-dft_index_expr.R`): both rows ship. Each resolved expression, evaluated in R over known band values, gives the hand value. Examples: green 0.1 / nir 0.3 → −0.5, including through S2 scale/offset tokens. `mndwi` resolves `swir16` → `B11`. Existing cube cache keys are unchanged, and the frozen-key test stays green. +- [x] Add `green = "B03"` and `blue = "B02"` to the `sentinel-2-l2a` roles (`R/dft_stac_config.R`), and update its roxygen. +- [x] Add `ndwi` `(green - nir) / (green + nir)` (McFeeters) and `mndwi` `(green - swir16) / (green + swir16)` (Xu) to `inst/indices/indices.csv`. +- [x] Tests (`test-dft_index_expr.R`): both rows ship. Each resolved expression, evaluated in R over known band values, gives the hand value. Examples: green 0.1 / nir 0.3 → −0.5, including through S2 scale/offset tokens. `mndwi` resolves `swir16` → `B11`. Existing cube cache keys are unchanged, and the frozen-key test stays green. ## Phase 2: extract shared cube internals (refactor, no behaviour change) diff --git a/tests/testthat/test-dft_index_expr.R b/tests/testthat/test-dft_index_expr.R index 26f3cd5..4778987 100644 --- a/tests/testthat/test-dft_index_expr.R +++ b/tests/testthat/test-dft_index_expr.R @@ -1,5 +1,11 @@ # S2 role map (Planetary Computer band names) used across the resolver tests -s2_roles <- list(red = "B04", nir = "B08", swir16 = "B11", mask = "SCL") +s2_roles <- list(red = "B04", green = "B03", blue = "B02", nir = "B08", + swir16 = "B11", mask = "SCL") + +# Evaluate a resolved expression in R over named band values. The water indices +# are plain arithmetic, so R and tinyexpr agree on them; this checks the NUMBER +# the formula produces, not just its string. +eval_expr <- function(expr, bands) eval(parse(text = expr), as.list(bands)) test_that("dft_index_table ships ndvi, kndvi, ndmi with formulas over roles", { tbl <- dft_index_table() @@ -59,3 +65,38 @@ test_that("an index needing an absent role errors", { "role" ) }) + + +test_that("dft_index_table ships ndwi and mndwi over green", { + tbl <- dft_index_table() + expect_true(all(c("ndwi", "mndwi") %in% tbl$index)) + expect_setequal(drift:::index_roles("ndwi"), c("green", "nir")) + expect_setequal(drift:::index_roles("mndwi"), c("green", "swir16")) +}) + +test_that("ndwi and mndwi give the hand-computed value on known reflectance", { + # Open water: green 0.10, nir 0.03, swir16 0.01 -> both strongly positive. + # Vegetation: green 0.10, nir 0.30, swir16 0.20 -> both negative. + water <- c(B03 = 0.10, B08 = 0.03, B11 = 0.01) + veg <- c(B03 = 0.10, B08 = 0.30, B11 = 0.20) + ndwi <- drift:::index_resolve_expr("ndwi", s2_roles, scale = 1, offset = 0) + mndwi <- drift:::index_resolve_expr("mndwi", s2_roles, scale = 1, offset = 0) + expect_equal(ndwi, "(B03 - B08) / (B03 + B08)") + expect_equal(mndwi, "(B03 - B11) / (B03 + B11)") + expect_equal(eval_expr(ndwi, water), (0.10 - 0.03) / (0.10 + 0.03)) + expect_equal(eval_expr(ndwi, veg), -0.5) + expect_equal(eval_expr(mndwi, water), (0.10 - 0.01) / (0.10 + 0.01)) + expect_equal(eval_expr(mndwi, veg), -1 / 3) +}) + +test_that("ndwi applies the S2 offset per band, so DN input gives reflectance output", { + # Post-2022 S2 DN = reflectance * 1e4 + 1000. Feeding those DNs through the + # scale/offset-folded expression must return the same value as the + # reflectance above; computing it on raw DN would not (the offset does not + # cancel in a ratio), which is the defect the affine folding exists to stop. + dn <- c(B03 = 0.10 * 1e4 + 1000, B08 = 0.30 * 1e4 + 1000) + expr <- drift:::index_resolve_expr("ndwi", s2_roles, scale = 1e-4, offset = -0.1) + expect_equal(eval_expr(expr, dn), -0.5) + raw <- drift:::index_resolve_expr("ndwi", s2_roles, scale = 1, offset = 0) + expect_false(isTRUE(all.equal(eval_expr(raw, dn), -0.5))) +}) diff --git a/tests/testthat/test-dft_stac_config.R b/tests/testthat/test-dft_stac_config.R index 9904a24..f8fd472 100644 --- a/tests/testthat/test-dft_stac_config.R +++ b/tests/testthat/test-dft_stac_config.R @@ -36,8 +36,10 @@ test_that("dft_stac_config returns role-based cube config for sentinel-2-l2a", { expect_match(cfg$stac_url, "planetarycomputer") # role -> asset map uses Planetary Computer band names (B04/B08/B11/SCL) - expect_named(cfg$roles, c("red", "nir", "swir16", "mask")) + expect_named(cfg$roles, c("red", "green", "blue", "nir", "swir16", "mask")) expect_equal(cfg$roles$red, "B04") + expect_equal(cfg$roles$green, "B03") + expect_equal(cfg$roles$blue, "B02") expect_equal(cfg$roles$nir, "B08") expect_equal(cfg$roles$swir16, "B11") expect_equal(cfg$roles$mask, "SCL") From ec70c9b5614d7928663eb5001371d636ef37c564 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:27:10 -0700 Subject: [PATCH 08/29] Extract shared cube internals from dft_stac_cube (#79) Session setup, the cache read gate, the STAC item query with its offset split, and the tiled or untiled assembly move into internal helpers. The assembly takes a pixel function, so the composite coming next can share every line of the read path. Behaviour is unchanged: offline cube tests pass and a live untiled cube is byte-identical to main's. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_cube.R | 370 +++++++++++++++++++++-------------- planning/active/progress.md | 1 + planning/active/task_plan.md | 4 +- 3 files changed, 223 insertions(+), 152 deletions(-) diff --git a/R/dft_stac_cube.R b/R/dft_stac_cube.R index bed2f55..c011da2 100644 --- a/R/dft_stac_cube.R +++ b/R/dft_stac_cube.R @@ -164,38 +164,10 @@ dft_stac_cube <- function(aoi, sign_fn = rstac::sign_planetary_computer()) { check_gdalcubes("to fetch STAC cubes") - # gdalcubes worker processes for the read. drift never set this before v0.9.0, - # so every fetch ran single-threaded — not a considered choice, just the - # consequence of never calling gdalcubes_options(). Measured 2.05x at 4 and - # 2.47x at 8 on the packaged AOI, with byte-identical output at every setting - # (drift#47), so it is a pure cost knob and stays out of the cache key. - # The gain is CONCURRENCY, not the finer chunking it also causes: arms that - # isolate chunk size at parallel = 1 measured finer chunks 45% SLOWER. - # Restored on exit so the caller's gdalcubes session is untouched, matching - # the GDAL config handling below. - old_parallel <- gdalcubes::gdalcubes_options()$parallel - parallel <- cube_parallel_check(parallel, session = old_parallel) - gdalcubes::gdalcubes_options(parallel = parallel) - on.exit(gdalcubes::gdalcubes_options(parallel = old_parallel), add = TRUE) - - # GDAL cloud-read tuning for /vsicurl COG streaming (biggest win: - # DISABLE_READDIR_ON_OPEN avoids a remote directory listing on every open). - # Restored on exit so we don't mutate the caller's session. - gdal_cfg <- c( - GDAL_DISABLE_READDIR_ON_OPEN = "EMPTY_DIR", - GDAL_HTTP_MULTIPLEX = "YES", - GDAL_HTTP_VERSION = "2", - VSI_CACHE = "TRUE", - CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif" - ) - old_cfg <- Sys.getenv(names(gdal_cfg), unset = NA) - do.call(Sys.setenv, as.list(gdal_cfg)) - on.exit({ - set_again <- old_cfg[!is.na(old_cfg)] - if (length(set_again)) do.call(Sys.setenv, as.list(set_again)) - unset <- names(old_cfg)[is.na(old_cfg)] - if (length(unset)) Sys.unsetenv(unset) - }, add = TRUE) + # gdalcubes worker count and GDAL /vsicurl tuning, both restored on exit so + # the caller's session is untouched (see stac_cube_session()). + restore_session <- stac_cube_session(parallel) + on.exit(restore_session(), add = TRUE) cfg <- dft_stac_config(source) if (!isTRUE(cfg$cube)) { @@ -212,7 +184,6 @@ dft_stac_cube <- function(aoi, # (e.g. Sentinel-2 +1000 DN from 2022-01-25) carry the boundary date and the # pre-boundary offset; the fetch splits items at the boundary and corrects # each side, so a series crossing it has no artificial index step. - offset_boundary <- cfg$offset_boundary offset_before <- cfg$offset_before %||% 0 # normalize clip to a single scalar so the mask gate and the cache key agree: a # truthy-but-non-TRUE clip (e.g. 1 or "TRUE") must not skip the mask yet key as @@ -230,21 +201,17 @@ dft_stac_cube <- function(aoi, target_crs <- if (is.null(crs)) auto_utm_epsg(aoi) else crs aoi_wgs84 <- sf::st_transform(aoi, 4326) aoi_target <- sf::st_transform(aoi, as.integer(gsub("EPSG:", "", target_crs))) - bbox_target <- sf::st_bbox(aoi_target) # Assets: the index's required roles + the mask role (index_roles errors on # an unknown index before any network call) roles_needed <- index_roles(index) band_assets <- unlist(cfg$roles[roles_needed], use.names = FALSE) - mask_asset <- cfg$roles$mask # datetime interval -> cube_view time bounds dr <- strsplit(datetime, "/", fixed = TRUE)[[1]] if (length(dr) != 2) { cli::cli_abort("`datetime` must be an ISO 8601 interval {.val start/end}.") } - t0 <- dr[1] - t1 <- dr[2] # Cache # // (#48); recursive = TRUE required for the extra level @@ -270,40 +237,160 @@ dft_stac_cube <- function(aoi, # aborts (re-fetching will not fix a genuine AOI/datetime mismatch), a corrupt # one re-fetches. if (!force && file.exists(cache_file) && cache_hit_ok(cache_file, "cube")) { - r <- terra::rast(cache_file) - # Guard the READ path too, not only the write. A cube written by an earlier - # version could be all-NA — drift's end-to-end test passed on exactly that - # (#47) — and the cache key is unchanged, so upgrading does not invalidate - # it. Without this the empty cube is served silently and, under - # `force = FALSE`, permanently. - # - # The warning handler makes this a WHOLE-FILE read probe at zero added cost: - # cube_check_nonempty() already scans every pixel via terra::global(), and a - # GDAL read failure surfaces as a warning while global() happily returns a - # number computed from whatever it got. That is strictly stronger than the - # last-row probe cache_hit_ok() just ran, and the cube is the path where it - # matters most — its cache is a multi-hour stream, not an annual raster. - read_dirty <- FALSE - withCallingHandlers( - cube_check_nonempty(r, cfg$collection, datetime, cached = TRUE), - warning = function(w) { - read_dirty <<- TRUE - invokeRestart("muffleWarning") - } - ) - if (!read_dirty) { + r <- stac_cube_cache_read(cache_file, cfg$collection, datetime) + if (!is.null(r)) { message(" cube: cached") terra::time(r) <- month_times(terra::nlyr(r)) return(r) } - cli::cli_warn(c( - "The cached cube's pixel data could not be read cleanly; re-fetching.", - "i" = "{.file {cache_file}}" - )) - rm(r) } - # STAC query: intersects (not bbox) + scene cloud pre-filter + pagination + fetched <- stac_cube_items(cfg, aoi_wgs84, datetime, cloud_cover_max, + months, sign_fn) + + # The index is applied per offset side: the pixel function receives the + # offset for the item subset it is building (see stac_cube_assemble()). + pixel_fn <- function(cube, offset_use) { + dft_index_expr(cube, index = index, source = source, + roles = cfg$roles, scale = scale, offset = offset_use) + } + stk <- stac_cube_assemble( + fetched, cfg, aoi_target, target_crs, t0 = dr[1], t1 = dr[2], res = res, + dt = dt, aggregation = aggregation, resampling = resampling, + band_assets = band_assets, mask_values = mask_values, + offset = offset, offset_before = offset_before, pixel_fn = pixel_fn, + tile_size = tile_size + ) + + # Restore the AOI-polygon clip removed in #30: mask the assembled stack. Cells + # the polygon does not touch become NA on every layer (terra::mask() defaults + # to touches = TRUE, so the boundary is inclusive by up to one cell), so + # dft_rast_break()/dft_rast_trend() skip them via their + # `rowSums(!is.na) >= min_obs` gate. `mask` preserves nlyr and time is set + # below, so the cached tif — and the cache-read path — need no other change. + if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) + + # Post-condition before anything is cached: a cube with no data at all is the + # gdalcubes all-NA failure mode (#32/#47), and it is SILENT — nothing upstream + # raises, and a cached empty cube is then served for every later call with the + # same key. Cheap to check, and the only place it can be caught on the real + # network path rather than in a fixture. + # + # The whole stack, NOT layer 1: an individual layer being empty is DOCUMENTED + # behaviour, not a fault. `months` keeps the series regular by leaving months + # with no retained scenes as NA, so the packaged `months = 6:9` example has + # eight all-NA layers and January first. Checking layer 1 would abort the + # vignette's own call after the full COG stream. + cube_check_nonempty(stk, cfg$collection, datetime, cached = FALSE) + + terra::time(stk) <- month_times(terra::nlyr(stk)) + names(stk) <- rep(index, terra::nlyr(stk)) + # Atomic publish (#41). Losing this cache to an interrupted write is far more + # expensive here than in dft_stac_fetch() — it is a multi-hour Sentinel-2 + # stream rather than a small annual raster. + cache_write_atomic(cache_file, function(out) { + terra::writeRaster(stk, out, overwrite = TRUE) + }) + stk +} + + +#' Set the gdalcubes worker count and GDAL cloud-read tuning for a cube read +#' +#' Shared by [dft_stac_cube()] and [dft_stac_composite()]. Returns a function +#' that restores the caller's session, which the caller registers with +#' `on.exit(restore(), add = TRUE)`: an `on.exit()` here would fire when this +#' helper returns, not when the read finishes. +#' +#' gdalcubes worker processes: drift never set this before v0.9.0, so every +#' fetch ran single-threaded — not a considered choice, just the consequence of +#' never calling gdalcubes_options(). Measured 2.05x at 4 and 2.47x at 8 on the +#' packaged AOI, with byte-identical output at every setting (drift#47), so it +#' is a pure cost knob and stays out of the cache key. The gain is CONCURRENCY, +#' not the finer chunking it also causes: arms that isolate chunk size at +#' parallel = 1 measured finer chunks 45% SLOWER. +#' +#' GDAL cloud-read tuning for /vsicurl COG streaming (biggest win: +#' DISABLE_READDIR_ON_OPEN avoids a remote directory listing on every open). +#' @noRd +stac_cube_session <- function(parallel) { + old_parallel <- gdalcubes::gdalcubes_options()$parallel + parallel <- cube_parallel_check(parallel, session = old_parallel) + gdalcubes::gdalcubes_options(parallel = parallel) + + gdal_cfg <- c( + GDAL_DISABLE_READDIR_ON_OPEN = "EMPTY_DIR", + GDAL_HTTP_MULTIPLEX = "YES", + GDAL_HTTP_VERSION = "2", + VSI_CACHE = "TRUE", + CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif" + ) + old_cfg <- Sys.getenv(names(gdal_cfg), unset = NA) + do.call(Sys.setenv, as.list(gdal_cfg)) + + function() { + gdalcubes::gdalcubes_options(parallel = old_parallel) + set_again <- old_cfg[!is.na(old_cfg)] + if (length(set_again)) do.call(Sys.setenv, as.list(set_again)) + unset <- names(old_cfg)[is.na(old_cfg)] + if (length(unset)) Sys.unsetenv(unset) + invisible() + } +} + + +#' Read a cached cube-family raster, or return NULL if its pixels are unreadable +#' +#' The cache-read half of [dft_stac_cube()] and [dft_stac_composite()], called +#' after `cache_hit_ok()` has passed. Guards the READ path too, not only the +#' write. A cube written by an earlier version could be all-NA — drift's +#' end-to-end test passed on exactly that (#47) — and the cache key is +#' unchanged, so upgrading does not invalidate it. Without this the empty cube +#' is served silently and, under `force = FALSE`, permanently. +#' +#' The warning handler makes this a WHOLE-FILE read probe at zero added cost: +#' cube_check_nonempty() already scans every pixel via terra::global(), and a +#' GDAL read failure surfaces as a warning while global() happily returns a +#' number computed from whatever it got. That is strictly stronger than the +#' last-row probe cache_hit_ok() just ran, and the cube is the path where it +#' matters most — its cache is a multi-hour stream, not an annual raster. +#' +#' An all-`NA` cache still ABORTS (via cube_check_nonempty()); only an unclean +#' read returns `NULL`, which the caller treats as a miss and re-fetches. +#' @noRd +stac_cube_cache_read <- function(cache_file, collection, datetime) { + r <- terra::rast(cache_file) + read_dirty <- FALSE + withCallingHandlers( + cube_check_nonempty(r, collection, datetime, cached = TRUE), + warning = function(w) { + read_dirty <<- TRUE + invokeRestart("muffleWarning") + } + ) + if (!read_dirty) return(r) + cli::cli_warn(c( + "The cached cube's pixel data could not be read cleanly; re-fetching.", + "i" = "{.file {cache_file}}" + )) + NULL +} + + +#' Query the STAC items for a cube read and split them at the offset boundary +#' +#' Shared by [dft_stac_cube()] and [dft_stac_composite()]. Three STAC-query +#' specifics: pagination via [rstac::items_fetch()] is mandatory (a monthly +#' multi-year query returns hundreds of items; a single page silently +#' truncates); the query uses `intersects` with the AOI geometry, not a bounding +#' box; and a scene-level `eo:cloud_cover` pre-filter shrinks the collection +#' before any pixel is read. +#' +#' @return A list with `features` (the signed, month-filtered STAC features) and +#' `is_pre` (logical, one per feature: acquired before `cfg$offset_boundary`). +#' @noRd +stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, + sign_fn) { message("Querying STAC: ", cfg$collection, " (", datetime, ")...") items <- rstac::stac(cfg$stac_url) |> rstac::stac_search( @@ -335,58 +422,81 @@ dft_stac_cube <- function(aoi, if (n_items == 0) stop("No STAC items found for ", cfg$collection) # Baseline-conditional offset: split items at the boundary so each side is - # corrected with its own offset (below). The split is by item date, so it is - # the same for every read extent — compute it (and announce it) once, before + # corrected with its own offset. The split is by item date, so it is the same + # for every read extent — compute it (and announce it) once, before # assembling any cube. is_pre <- rep(FALSE, length(items$features)) - if (!is.null(offset_boundary)) { + if (!is.null(cfg$offset_boundary)) { item_date <- as.Date(substr( vapply(items$features, function(f) f$properties$datetime %||% NA_character_, ""), 1, 10 )) - is_pre <- !is.na(item_date) & item_date < as.Date(offset_boundary) + is_pre <- !is.na(item_date) & item_date < as.Date(cfg$offset_boundary) } if (any(is_pre) && !all(is_pre)) { - message(" offset split at ", offset_boundary, ": ", + message(" offset split at ", cfg$offset_boundary, ": ", sum(is_pre), " pre / ", sum(!is_pre), " post") } + list(features = items$features, is_pre = is_pre) +} - # Build the masked index cube for one item subset with one offset over a given - # cube_view, materialize it, and read it back as a terra stack. The cube spans - # the view's extent; the AOI clip happens on the output via terra::mask() - # (stac_cube_clip, #32), as dft_stac_fetch() does — not gdalcubes::filter_geom(). - # - # That is now a MEASURED choice rather than a workaround (#47). The segfault - # was fixed in NewGraphEnvironment/gdalcubes@newgraph, so filter_geom is - # available; it is simply not worth using. It skips whole CHUNKS, and gdalcubes - # clamps a chunk edge to [64, 1024] px — coarse enough to skip nothing at the - # default, and fine enough to break alignment with the COGs' 512x512 blocks - # when forced down. Benchmarked on the packaged AOI: default chunking 462 - # requests / 236.9 s against the bbox baseline's 462 / 236.8 s, and 64 px - # chunking 693 / 348.2 s — 1.5x the requests to save 26.7% of the ground. - # It also clips at CELL CENTRE where terra::mask() is touches = TRUE, so - # swapping them would silently shrink the analysed footprint by 15.5%. - # See data-raw/benchmark_filter_geom.R and inst/notes/gdalcubes-pc-gotchas.md. - build_index_stack <- function(features, offset_use, v) { + +#' Assemble a masked cube stack over the AOI, tiled or not +#' +#' Shared by [dft_stac_cube()] and [dft_stac_composite()]; the two differ only in +#' `pixel_fn`, which turns the masked reflectance cube for one item subset into +#' the bands to keep (an index for the cube, scaled reflectance for the +#' composite). `pixel_fn(cube, offset_use)` receives the offset for the subset +#' it is building, so each side of the offset boundary is corrected with its own. +#' +#' The cube spans the view's extent; the AOI clip happens on the output via +#' terra::mask() (stac_cube_clip, #32), as dft_stac_fetch() does — not +#' gdalcubes::filter_geom(). That is now a MEASURED choice rather than a +#' workaround (#47). The segfault was fixed in NewGraphEnvironment/gdalcubes@newgraph, +#' so filter_geom is available; it is simply not worth using. It skips whole +#' CHUNKS, and gdalcubes clamps a chunk edge to `[64, 1024]` px — coarse enough to +#' skip nothing at the default, and fine enough to break alignment with the +#' COGs' 512x512 blocks when forced down. Benchmarked on the packaged AOI: +#' default chunking 462 requests / 236.9 s against the bbox baseline's 462 / +#' 236.8 s, and 64 px chunking 693 / 348.2 s — 1.5x the requests to save 26.7% +#' of the ground. It also clips at CELL CENTRE where terra::mask() is touches = +#' TRUE, so swapping them would silently shrink the analysed footprint by 15.5%. +#' See data-raw/benchmark_filter_geom.R and inst/notes/gdalcubes-pc-gotchas.md. +#' +#' Untiled (tile_size = NULL): one cube over the AOI bounding box. Tiled (#38): +#' stream only the res-aligned tiles that intersect the AOI polygon and mosaic +#' them, so a sparse corridor reads near its footprint instead of the full bbox. +#' tile_grid()/tile_size_check() are the shared download-tiling helpers from #36 +#' (defined in dft_stac_fetch.R). +#' @noRd +stac_cube_assemble <- function(fetched, cfg, aoi_target, target_crs, t0, t1, + res, dt, aggregation, resampling, band_assets, + mask_values, offset, offset_before, pixel_fn, + tile_size = NULL) { + mask_asset <- cfg$roles$mask + features <- fetched$features + is_pre <- fetched$is_pre + + # Build the masked cube for one item subset with one offset over a given + # cube_view, materialize it, and read it back as a terra stack. + build_stack <- function(feats, offset_use, v) { img_col <- gdalcubes::stac_image_collection( - features, asset_names = c(band_assets, mask_asset) + feats, asset_names = c(band_assets, mask_asset) ) cube <- gdalcubes::raster_cube( img_col, v, mask = gdalcubes::image_mask(mask_asset, values = mask_values) ) - idx <- dft_index_expr(cube, index = index, source = source, - roles = cfg$roles, scale = scale, offset = offset_use) + out <- pixel_fn(cube, offset_use) tmp <- tempfile(fileext = ".nc") - gdalcubes::write_ncdf(idx, tmp, overwrite = TRUE) + gdalcubes::write_ncdf(out, tmp, overwrite = TRUE) terra::rast(tmp) } - # Assemble the full masked index stack for one space extent: build the cube_view - # over that extent (same t0/t1/dt for every extent, so every tile yields the same - # nlyr), run the offset split, and coalesce the pre/post subcubes with - # terra::cover (both built over the same view so their layers align). A local - # closure (not @noRd) because it reads the call's items/offset/index/etc. - assemble_index_stack <- function(extent) { + # Assemble the full masked stack for one space extent: build the cube_view + # over that extent (same t0/t1/dt for every extent, so every tile yields the + # same nlyr), run the offset split, and coalesce the pre/post subcubes with + # terra::cover (both built over the same view so their layers align). + assemble_extent <- function(extent) { v <- gdalcubes::cube_view( srs = target_crs, extent = list( @@ -399,69 +509,29 @@ dft_stac_cube <- function(aoi, ) if (any(is_pre) && !all(is_pre)) { terra::cover( - build_index_stack(items$features[is_pre], offset_before, v), - build_index_stack(items$features[!is_pre], offset, v) + build_stack(features[is_pre], offset_before, v), + build_stack(features[!is_pre], offset, v) ) } else { - build_index_stack(items$features, - if (all(is_pre)) offset_before else offset, v) + build_stack(features, if (all(is_pre)) offset_before else offset, v) } } - # Untiled (tile_size = NULL): one cube over the AOI bounding box — unchanged - # behavior. Tiled (#38): stream only the res-aligned tiles that intersect the - # AOI polygon and mosaic them, so a sparse corridor reads near its footprint - # instead of the full bbox. tile_grid()/tile_size_check() are the shared - # download-tiling helpers from #36 (defined in dft_stac_fetch.R); the GDAL - # /vsicurl tuning set at the top of this function already covers the extra - # per-tile COG opens. + bbox_target <- sf::st_bbox(aoi_target) bbox_ext <- list( left = bbox_target[["xmin"]], right = bbox_target[["xmax"]], bottom = bbox_target[["ymin"]], top = bbox_target[["ymax"]] ) - if (is.null(tile_size)) { - stk <- assemble_index_stack(bbox_ext) - } else { - tiles <- tile_grid(aoi_target, tile_size, res) - message(" tiling read into ", length(tiles), " tile(s) intersecting the AOI") - tile_stacks <- lapply(tiles, assemble_index_stack) - # every tile shares t0/t1/dt so nlyr is uniform; guard so a future per-tile - # time bound fails legibly here rather than deep inside terra::merge. - # terra::nlyr() returns a double, so the vapply template is numeric(1). - stopifnot(length(unique(vapply(tile_stacks, terra::nlyr, numeric(1)))) == 1L) - stk <- mosaic_stacks(tile_stacks) - } - - # Restore the AOI-polygon clip removed in #30: mask the assembled stack. Cells - # the polygon does not touch become NA on every layer (terra::mask() defaults - # to touches = TRUE, so the boundary is inclusive by up to one cell), so - # dft_rast_break()/dft_rast_trend() skip them via their - # `rowSums(!is.na) >= min_obs` gate. `mask` preserves nlyr and time is set - # below, so the cached tif — and the cache-read path — need no other change. - if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) - - # Post-condition before anything is cached: a cube with no data at all is the - # gdalcubes all-NA failure mode (#32/#47), and it is SILENT — nothing upstream - # raises, and a cached empty cube is then served for every later call with the - # same key. Cheap to check, and the only place it can be caught on the real - # network path rather than in a fixture. - # - # The whole stack, NOT layer 1: an individual layer being empty is DOCUMENTED - # behaviour, not a fault. `months` keeps the series regular by leaving months - # with no retained scenes as NA, so the packaged `months = 6:9` example has - # eight all-NA layers and January first. Checking layer 1 would abort the - # vignette's own call after the full COG stream. - cube_check_nonempty(stk, cfg$collection, datetime, cached = FALSE) + if (is.null(tile_size)) return(assemble_extent(bbox_ext)) - terra::time(stk) <- month_times(terra::nlyr(stk)) - names(stk) <- rep(index, terra::nlyr(stk)) - # Atomic publish (#41). Losing this cache to an interrupted write is far more - # expensive here than in dft_stac_fetch() — it is a multi-hour Sentinel-2 - # stream rather than a small annual raster. - cache_write_atomic(cache_file, function(out) { - terra::writeRaster(stk, out, overwrite = TRUE) - }) - stk + tiles <- tile_grid(aoi_target, tile_size, res) + message(" tiling read into ", length(tiles), " tile(s) intersecting the AOI") + tile_stacks <- lapply(tiles, assemble_extent) + # every tile shares t0/t1/dt so nlyr is uniform; guard so a future per-tile + # time bound fails legibly here rather than deep inside terra::merge. + # terra::nlyr() returns a double, so the vapply template is numeric(1). + stopifnot(length(unique(vapply(tile_stacks, terra::nlyr, numeric(1)))) == 1L) + mosaic_stacks(tile_stacks) } diff --git a/planning/active/progress.md b/planning/active/progress.md index 37033c2..87eb0b8 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -7,3 +7,4 @@ - Scaffolded the PWF baseline with the approved phases. - Next: Phase 1 (NDWI/MNDWI). - Phase 1 done. Added the `green`/`blue` S2 roles and the `ndwi`/`mndwi` registry rows. The tests evaluate the resolved expressions over known reflectance, and over post-2022 DN through the offset. A mutation check swapping NDWI's sign failed 4 tests. The frozen cube key is unchanged. +- Phase 2 refactor landed as `stac_cube_session()`, `stac_cube_cache_read()`, `stac_cube_items()` and `stac_cube_assemble(pixel_fn=)`. The offline cube tests pass 90/90, including the frozen key and the mocked cache gates. **Live byte identity:** an untiled NDVI cube (packaged AOI, 2021-07/08, parallel 4) from main against the refactor gives identical values and NA pattern, max diff 0, 77.1 s vs 77.7 s. Recorded in the scratchpad `cube_identity.R`. The tiled check is pending, in the worktree network run. diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md index 74381c4..413b3cb 100644 --- a/planning/active/task_plan.md +++ b/planning/active/task_plan.md @@ -20,11 +20,11 @@ Plan-gate decisions (2026-09-28): ## Phase 2: extract shared cube internals (refactor, no behaviour change) -- [ ] Move out of `R/dft_stac_cube.R`: +- [x] Move out of `R/dft_stac_cube.R`: - GDAL/gdalcubes session setup and restore; - STAC query, `months` filter and offset split (`stac_cube_items()`); - extent → cube_view → pre/post `cover` → tiling/mosaic (`stac_cube_assemble()`), taking a pixel function. -- [ ] `dft_stac_cube()` calls these helpers and passes a `dft_index_expr()` closure as its pixel function. +- [x] `dft_stac_cube()` calls these helpers and passes a `dft_index_expr()` closure as its pixel function. - [ ] Guard: the frozen legacy key and every offline cube test pass unchanged. Run the network e2e (untiled + tiled) before and after, and record both. ## Phase 3: `dft_stac_composite()` From efd84211501f3fb91f333906ca4f398475c38a6d Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:36:49 -0700 Subject: [PATCH 09/29] Add dft_stac_composite for dated reflectance composites (#79) Each year's window (a run of calendar months) is reduced to a cloud-masked median of the requested band roles, as surface reflectance, through the read path shared with dft_stac_cube. Cached as a COG so it can be served by titiler unchanged. From the plan review: - chips around sample points are one call per point, not one tiled call over all points, which would span the floodplain under one key; - a window straddling the 2022-01-25 offset change is refused rather than returned as a pre-side cover; - an empty year is dropped with a warning; - the STAC query uses explicit times, since a date-only end bound drops the last day's scenes; - the network e2e cache paths now follow the #48 scheme dir; - a test pins how dft_stac_cube calls its cache key. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- NAMESPACE | 1 + R/dft_stac_composite.R | 431 +++++++++++++++++++++++ R/dft_stac_cube.R | 14 +- man/dft_stac_composite.Rd | 176 +++++++++ planning/active/review-plan.md | 38 ++ tests/testthat/test-dft_stac_composite.R | 248 +++++++++++++ tests/testthat/test-dft_stac_cube.R | 31 ++ 7 files changed, 935 insertions(+), 4 deletions(-) create mode 100644 R/dft_stac_composite.R create mode 100644 man/dft_stac_composite.Rd create mode 100644 planning/active/review-plan.md create mode 100644 tests/testthat/test-dft_stac_composite.R diff --git a/NAMESPACE b/NAMESPACE index 3f20d69..c78362a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -18,6 +18,7 @@ export(dft_rast_summarize) export(dft_rast_transition) export(dft_rast_trend) export(dft_stac_classes) +export(dft_stac_composite) export(dft_stac_config) export(dft_stac_cube) export(dft_stac_fetch) diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R new file mode 100644 index 0000000..6043ef4 --- /dev/null +++ b/R/dft_stac_composite.R @@ -0,0 +1,431 @@ +#' Fetch dated, cloud-masked reflectance composites from a STAC catalog +#' +#' Reference imagery for windows you choose. For each year, streams every scene +#' in the chosen calendar months, masks clouds per pixel, and reduces them to +#' one median composite of the requested bands. A true-colour composite of the +#' same season in different years, or of early against late season within one +#' year, says **when** a place changed. The Esri and Google basemaps in +#' [dft_map_interactive()] carry no capture date, so they cannot. +#' +#' This is the multi-band sibling of [dft_stac_cube()] and shares its read path: +#' the same STAC query (paginated, `intersects` the AOI, scene-level +#' `eo:cloud_cover` pre-filter), the same per-pixel mask, the same Sentinel-2 +#' offset correction at the 2022-01-25 processing-baseline boundary, the same +#' tiling and the same AOI clip. Where the cube returns one index per month, this +#' returns the bands themselves, one time step per window. +#' +#' Values are **surface reflectance** (the source's `scale` and `offset` +#' applied), stored as floating point rather than display bytes. That keeps a +#' composite usable as data, for an index or as classifier input, and leaves +#' contrast stretching to display time. [dft_map_interactive()] applies one +#' shared stretch to every composite it is given, so a brightness difference +#' between years is a real one. +#' +#' @section Caching: +#' Each year's composite is written once under [dft_cache_path()] as +#' `/composite_.tif`, a Cloud Optimized GeoTIFF. The key hashes the +#' AOI geometry and every parameter that changes the pixels: bands (in order), +#' months, the year's window, resolution, CRS, aggregation, resampling, cloud +#' cover, mask values, reflectance scale and offset, `clip` and `tile_size`. +#' Because the file is a COG, it can be copied to object storage and served +#' through titiler unchanged, which is how a floodplain-wide composite reaches +#' [dft_map_interactive()]: `rgb` accepts COG URLs as well as rasters. +#' +#' @section Floodplain scale and per-point chips: +#' A composite over a whole floodplain streams the floodplain's bounding box +#' and produces a raster the size of it. When the purpose is reviewing sample +#' points, call this **once per buffered point** instead (see the examples). +#' Each chip is small, streams only the COG blocks under it, and is cached on +#' its own, so adding a point to the sample fetches one chip rather than +#' invalidating the rest. Passing all the points as one multi-feature `aoi` +#' does not do this: the output spans the points' combined bounding box, and +#' one cache entry covers every point. +#' +#' @section Windows and the Sentinel-2 offset boundary: +#' A window is one calendar year's run of `months`, so a winter window that +#' crosses the new year (December to February) cannot be expressed; `c(12, 1, +#' 2)` composites January, February and December of the **same** year, and the +#' label says so. Sentinel-2 changed its reflectance offset on 2022-01-25. A +#' window with scenes on both sides of that date cannot be reduced to one true +#' median, because gdalcubes aggregates before the per-side offset is applied, +#' so such a window is refused rather than returned as something that is not a +#' median. Only a January 2022 window can straddle it. +#' +#' @param aoi An `sf` polygon defining the area of interest. A multi-feature +#' `aoi` is read as its union; for sample points, see the chips section. +#' @param years Integer vector of years. One composite is returned per year. +#' @param months Integer vector of calendar months (1-12) to composite (default +#' `6:7`, June and July). Each year's window runs from the first day of the +#' earliest month to the last day of the latest; scenes from months not listed +#' are dropped, so `c(6, 8)` composites June and August and skips July. +#' @param bands Character vector of band **roles** from [dft_stac_config()] +#' (default `c("red", "green", "blue")`, true colour). Any roles and any +#' number of them; `c("nir", "red", "green")` is the standard vegetation false +#' colour. Layers are named by role, in the order given. +#' @param source Character. A cube source name for [dft_stac_config()] (default +#' `"sentinel-2-l2a"`). +#' @param res Numeric. Output pixel size in CRS units (default 10). +#' @param crs Character. Target CRS as an EPSG string. When `NULL`, +#' auto-detected from the AOI centroid's UTM zone. +#' @param aggregation Character. How scenes within a window are reduced +#' (default `"median"`). +#' @param resampling Character. Spatial resampling (default `"bilinear"`). +#' @param clip Logical. Clip the output to the AOI polygon (default `FALSE`). +#' Reference imagery is read around a place, not only inside it, so the +#' default keeps the whole AOI bounding box. `TRUE` uses the same rule as +#' [dft_stac_cube()]: every cell the polygon touches is kept. +#' @param cloud_cover_max Numeric. Scene-level `eo:cloud_cover` maximum percent +#' (default 20, stricter than the cube's 60 because a composite of few clear +#' scenes looks better than one of many cloudy ones). +#' @param mask_values Integer vector of mask-band classes to exclude. When +#' `NULL`, taken from [dft_stac_config()]. +#' @param tile_size Numeric or `NULL` (default). Read-tiling edge length in CRS +#' units; only tiles intersecting the AOI are streamed. A memory knob, not a +#' speed one; see [dft_stac_cube()]. +#' @param parallel Integer or `NULL`. gdalcubes worker processes; see +#' [dft_stac_cube()]. +#' @param cache_dir Character. Cache directory. When `NULL`, uses +#' [dft_cache_path()]. +#' @param force Logical. Re-fetch even if cached (default `FALSE`). The +#' replacement is atomic. +#' @param sign_fn A signing function for STAC assets. Default is +#' [rstac::sign_planetary_computer()]. +#' +#' @return A named list of [terra::SpatRaster]s, one per year, each with one +#' layer per band. Names label the window, e.g. `"2017 Jun–Jul"`, and +#' become layer labels in [dft_map_interactive()]. Each raster's +#' [terra::time()] is the window start. A year with no usable scenes is +#' dropped with a warning; if every year is empty, the call aborts. +#' +#' @seealso [dft_map_interactive()] (`rgb =`) to display them, +#' [dft_stac_cube()] for index time series. +#' +#' @examples +#' \dontrun{ +#' aoi <- sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift")) +#' +#' # True colour, same season in two years +#' tc <- dft_stac_composite(aoi, years = c(2017, 2023), months = 6:7) +#' names(tc) +#' terra::plotRGB(tc[[1]], stretch = "lin") +#' +#' # Vegetation false colour, late season +#' fc <- dft_stac_composite(aoi, years = 2023, months = 8:9, +#' bands = c("nir", "red", "green")) +#' +#' # Chips around sample points: one small, separately cached composite each +#' pts <- sf::st_as_sf(sf::st_sample(aoi, 20)) +#' buf <- sf::st_buffer(pts, 300) +#' chips <- lapply(seq_len(nrow(buf)), function(i) { +#' dft_stac_composite(buf[i, ], years = c(2017, 2023)) +#' }) +#' +#' dft_map_interactive(rgb = c(tc, fc[1]), aoi = aoi) +#' } +#' +#' @export +dft_stac_composite <- function(aoi, + years, + months = 6:7, + bands = c("red", "green", "blue"), + source = "sentinel-2-l2a", + res = 10, + crs = NULL, + aggregation = "median", + resampling = "bilinear", + clip = FALSE, + cloud_cover_max = 20, + mask_values = NULL, + tile_size = NULL, + parallel = NULL, + cache_dir = NULL, + force = FALSE, + sign_fn = rstac::sign_planetary_computer()) { + rlang::check_installed("gdalcubes", reason = "to fetch STAC composites") + + cfg <- dft_stac_config(source) + if (!isTRUE(cfg$cube)) { + cli::cli_abort(c( + "Source {.val {source}} is not a cube source.", + "i" = "Use {.fn dft_stac_fetch} for categorical rasters." + )) + } + years <- composite_years_check(years) + months <- composite_months_check(months) + band_assets <- composite_band_assets(bands, cfg$roles) + + mask_values <- mask_values %||% cfg$mask_values + scale <- cfg$scale %||% 1 + offset <- cfg$offset %||% 0 + offset_before <- cfg$offset_before %||% 0 + clip <- isTRUE(as.logical(clip)) + if (!is.null(tile_size)) tile_size <- tile_size_check(tile_size, res) + + if (inherits(aoi, "SpatVector")) aoi <- sf::st_as_sf(aoi) + stopifnot(inherits(aoi, c("sf", "sfc"))) + target_crs <- if (is.null(crs)) auto_utm_epsg(aoi) else crs + aoi_wgs84 <- sf::st_transform(aoi, 4326) + aoi_target <- sf::st_transform(aoi, as.integer(gsub("EPSG:", "", target_crs))) + + restore_session <- stac_cube_session(parallel) + on.exit(restore_session(), add = TRUE) + + cache_source_dir <- cache_scheme_dir(cache_dir, source) + dir.create(cache_source_dir, recursive = TRUE, showWarnings = FALSE) + + # Every band scaled to reflectance with the offset for the side of the + # 2022-01-25 boundary the window's scenes fall on (a window straddling it is + # refused by composite_offset_check()). + pixel_fn <- function(cube, offset_use) { + exprs <- vapply(band_assets, scale_token, character(1), + scale = scale, offset = offset_use, USE.NAMES = FALSE) + gdalcubes::apply_pixel(cube, exprs, names = bands) + } + + out <- lapply(years, function(year) { + w <- composite_window(year, months) + label <- composite_label(year, months) + cache_key <- stac_composite_cache_key( + aoi_target, res, target_crs, w$datetime, w$dt, aggregation, resampling, + cfg$stac_url, cfg$collection, band_assets, bands, cloud_cover_max, + mask_values, scale, offset, offset_before, months, clip, tile_size + ) + cache_file <- file.path(cache_source_dir, + paste0("composite_", cache_key, ".tif")) + + if (!force && file.exists(cache_file) && + cache_hit_ok(cache_file, "composite")) { + r <- stac_cube_cache_read(cache_file, cfg$collection, w$datetime, + label = "composite") + if (!is.null(r)) { + message(" composite ", label, ": cached") + return(composite_finish(r, bands, w$t0)) + } + } + + build <- function() { + fetched <- stac_cube_items(cfg, aoi_wgs84, w$query, cloud_cover_max, + months, sign_fn) + composite_offset_check(fetched$is_pre, label, cfg$offset_boundary) + stk <- stac_cube_assemble( + fetched, cfg, aoi_target, target_crs, t0 = w$t0, t1 = w$t1, res = res, + dt = w$dt, aggregation = aggregation, resampling = resampling, + band_assets = band_assets, mask_values = mask_values, + offset = offset, offset_before = offset_before, pixel_fn = pixel_fn, + tile_size = tile_size + ) + composite_layers_check(stk, bands, label, w) + if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) + cube_check_nonempty(stk, cfg$collection, w$datetime, cached = FALSE) + stk <- composite_finish(stk, bands, w$t0) + cache_write_atomic(cache_file, function(path) { + terra::writeRaster( + stk, path, filetype = "COG", datatype = "FLT4S", overwrite = TRUE, + gdal = c("COMPRESS=DEFLATE", "PREDICTOR=YES", "BLOCKSIZE=512", + "OVERVIEW_RESAMPLING=AVERAGE") + ) + }) + stk + } + # An empty or fully clouded year is a result, not a failure: warn and drop + # it, keeping the years already built (and cached) before it. + tryCatch( + build(), + drift_no_items = function(e) composite_skip(label, "no scenes"), + drift_empty_cube = function(e) composite_skip(label, "no clear pixels") + ) + }) + names(out) <- vapply(years, composite_label, character(1), months = months) + out <- out[!vapply(out, is.null, logical(1))] + if (!length(out)) { + cli::cli_abort(c( + "No year produced a composite.", + "i" = "Every window had no scenes or no clear pixels; try more \\ + {.arg months} or a higher {.arg cloud_cover_max}." + )) + } + out +} + + +#' Warn that a year was dropped, and return NULL for it +#' @noRd +composite_skip <- function(label, why) { + cli::cli_warn("Skipping the {label} composite: {why}.") + NULL +} + + +#' Refuse a window whose scenes straddle the reflectance-offset boundary +#' +#' gdalcubes reduces scenes to the median BEFORE the pixel function applies the +#' offset, so the two sides cannot be pooled into one median. The cube handles +#' this with terra::cover(pre, post), which for a composite window would return +#' the pre-boundary median wherever it has data, not a median of the window. +#' Refusing is honest; only a January 2022 window can reach it. +#' @noRd +composite_offset_check <- function(is_pre, label, boundary) { + if (any(is_pre) && !all(is_pre)) { + cli::cli_abort(c( + "The {label} window has scenes on both sides of the {boundary} \\ + reflectance-offset change, so it has no single median.", + "i" = "Choose months that do not include January of that year." + ), class = "drift_composite_offset_split") + } + invisible(TRUE) +} + + +#' Check a composite has one layer per band, in band order +#' +#' Two ways it can go wrong, both silent downstream: a window and `dt` that +#' disagree give more than one time step, and a multi-variable NetCDF read back +#' by terra in an order other than the bands' would put the wrong band in a +#' channel when the layers are renamed by position. +#' @noRd +composite_layers_check <- function(stk, bands, label, w) { + if (terra::nlyr(stk) != length(bands)) { + cli::cli_abort(c( + "The {label} composite has {terra::nlyr(stk)} layer{?s}; expected \\ + {length(bands)} (one per band).", + "i" = "Window {.val {w$datetime}} with {.code dt = {w$dt}} should be \\ + a single time step." + )) + } + if (!all(startsWith(names(stk), bands))) { + cli::cli_abort(c( + "The {label} composite's layers are not in band order.", + "x" = "Layers: {.val {names(stk)}}; bands: {.val {bands}}." + )) + } + invisible(TRUE) +} + +#' Name a composite's layers by band role and stamp the window start +#' @noRd +composite_finish <- function(r, bands, t0) { + names(r) <- bands + terra::time(r) <- rep(as.Date(t0), terra::nlyr(r)) + r +} + + +#' Validate `years`: non-empty whole numbers, no NA +#' @noRd +composite_years_check <- function(years) { + if (!is.numeric(years) || length(years) == 0L || anyNA(years) || + any(years != trunc(years))) { + cli::cli_abort(c( + "{.arg years} must be one or more whole-number years.", + "x" = "Got {.obj_type_friendly {years}}." + )) + } + as.integer(years) +} + + +#' Validate `months`: non-empty whole numbers in 1-12; returned sorted, unique +#' @noRd +composite_months_check <- function(months) { + if (!is.numeric(months) || length(months) == 0L || anyNA(months) || + any(months != trunc(months)) || any(months < 1 | months > 12)) { + cli::cli_abort(c( + "{.arg months} must be one or more calendar months, whole numbers 1-12.", + "x" = "Got {.obj_type_friendly {months}}." + )) + } + sort(unique(as.integer(months))) +} + + +#' Resolve band roles to asset names, erroring on a role the source lacks +#' +#' The mask role is not a band and is refused, as is a repeated role (two layers +#' with one name). +#' @noRd +composite_band_assets <- function(bands, roles) { + available <- setdiff(names(roles), "mask") + if (!is.character(bands) || length(bands) == 0L || anyNA(bands) || + anyDuplicated(bands)) { + cli::cli_abort(c( + "{.arg bands} must be one or more distinct band roles.", + "i" = "Available roles: {.val {available}}." + )) + } + unknown <- setdiff(bands, available) + if (length(unknown)) { + cli::cli_abort(c( + "Unknown band role{?s} {.val {unknown}}.", + "i" = "Available roles: {.val {available}}." + )) + } + unlist(roles[bands], use.names = FALSE) +} + + +#' One year's composite window: first day of the first month to the last day of +#' the last, as one `dt` step +#' @noRd +composite_window <- function(year, months) { + first <- min(months) + last <- max(months) + t0 <- as.Date(sprintf("%04d-%02d-01", year, first)) + # last day of the last month: the day before the 1st of the month after it + t1 <- seq(as.Date(sprintf("%04d-%02d-01", year, last)), + by = "month", length.out = 2L)[2L] - 1L + list( + t0 = format(t0), t1 = format(t1), + datetime = paste0(format(t0), "/", format(t1)), + # The STAC query needs explicit times: a date-only end bound is read as + # 00:00Z and drops the last day's scenes (measured: a window ending on a + # scene day returned 22 of 23 items), which in BC are acquired ~19:00Z. + query = paste0(format(t0), "T00:00:00Z/", format(t1), "T23:59:59Z"), + dt = sprintf("P%dM", last - first + 1L) + ) +} + + +#' Label a composite window, e.g. "2017 Jun-Jul" with an en dash +#' +#' A contiguous run of months is a range, a single month is itself, and a gap +#' lists the months, so the label never claims a month that was not composited. +#' @noRd +composite_label <- function(year, months) { + months <- sort(unique(months)) + abb <- month.abb[months] + span <- if (length(months) == 1L) { + abb + } else if (all(diff(months) == 1L)) { + paste0(abb[1L], "\u2013", abb[length(abb)]) + } else { + paste(abb, collapse = ", ") + } + paste(year, span) +} + + +#' Cache key for one composite +#' +#' Its own function rather than a reuse of stac_cube_cache_key(), whose legacy +#' shape is frozen. The leading `"composite"` tag keeps the two families apart +#' even if their parameter lists ever coincide; the filename prefix already +#' does, and this makes it true of the hash as well. Bands and their assets are +#' order-sensitive (they are the layer order); months and mask values are not. +#' @noRd +stac_composite_cache_key <- function(aoi_target, res, target_crs, datetime, dt, + aggregation, resampling, stac_url, + collection, band_assets, bands, + cloud_cover_max, mask_values, scale, + offset, offset_before, months, clip, + tile_size = NULL) { + geom_wkb <- sf::st_as_binary(sf::st_geometry(aoi_target), endian = "little") + parts <- list( + "composite", geom_wkb, as.numeric(res), target_crs, datetime, dt, + aggregation, resampling, stac_url, collection, band_assets, bands, + as.numeric(cloud_cover_max), sort(as.numeric(mask_values)), + as.numeric(scale), as.numeric(offset), as.numeric(offset_before), + sort(as.numeric(months)), as.logical(clip), + if (is.null(tile_size)) NA_real_ else as.numeric(tile_size) + ) + cache_key_hash(parts) +} diff --git a/R/dft_stac_cube.R b/R/dft_stac_cube.R index c011da2..267bce1 100644 --- a/R/dft_stac_cube.R +++ b/R/dft_stac_cube.R @@ -358,7 +358,8 @@ stac_cube_session <- function(parallel) { #' An all-`NA` cache still ABORTS (via cube_check_nonempty()); only an unclean #' read returns `NULL`, which the caller treats as a miss and re-fetches. #' @noRd -stac_cube_cache_read <- function(cache_file, collection, datetime) { +stac_cube_cache_read <- function(cache_file, collection, datetime, + label = "cube") { r <- terra::rast(cache_file) read_dirty <- FALSE withCallingHandlers( @@ -370,7 +371,7 @@ stac_cube_cache_read <- function(cache_file, collection, datetime) { ) if (!read_dirty) return(r) cli::cli_warn(c( - "The cached cube's pixel data could not be read cleanly; re-fetching.", + "The cached {label}'s pixel data could not be read cleanly; re-fetching.", "i" = "{.file {cache_file}}" )) NULL @@ -419,7 +420,12 @@ stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, n_items <- length(items$features) message(" ", n_items, " items returned") - if (n_items == 0) stop("No STAC items found for ", cfg$collection) + if (n_items == 0) { + # Classed so a multi-year caller can skip an empty year; the message is the + # one this has always raised. + rlang::abort(paste0("No STAC items found for ", cfg$collection), + class = "drift_no_items") + } # Baseline-conditional offset: split items at the boundary so each side is # corrected with its own offset. The split is by item date, so it is the same @@ -552,7 +558,7 @@ stac_cube_assemble <- function(fetched, cfg, aoi_target, target_crs, t0, t1, #' @noRd cube_check_nonempty <- function(stk, collection, datetime, cached) { if (sum(terra::global(stk, "notNA")$notNA) > 0) return(invisible(stk)) - cli::cli_abort(c( + cli::cli_abort(class = "drift_empty_cube", c( "The {if (cached) 'cached' else 'assembled'} cube has no data on any layer.", "i" = "Every cell is {.val NA} on all {terra::nlyr(stk)} layers. This is \\ either the gdalcubes all-{.val NA} failure mode, or the AOI does \\ diff --git a/man/dft_stac_composite.Rd b/man/dft_stac_composite.Rd new file mode 100644 index 0000000..5892198 --- /dev/null +++ b/man/dft_stac_composite.Rd @@ -0,0 +1,176 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dft_stac_composite.R +\name{dft_stac_composite} +\alias{dft_stac_composite} +\title{Fetch dated, cloud-masked reflectance composites from a STAC catalog} +\usage{ +dft_stac_composite( + aoi, + years, + months = 6:7, + bands = c("red", "green", "blue"), + source = "sentinel-2-l2a", + res = 10, + crs = NULL, + aggregation = "median", + resampling = "bilinear", + clip = FALSE, + cloud_cover_max = 20, + mask_values = NULL, + tile_size = NULL, + parallel = NULL, + cache_dir = NULL, + force = FALSE, + sign_fn = rstac::sign_planetary_computer() +) +} +\arguments{ +\item{aoi}{An \code{sf} polygon defining the area of interest. A multi-feature +\code{aoi} is read as its union; for sample points, see the chips section.} + +\item{years}{Integer vector of years. One composite is returned per year.} + +\item{months}{Integer vector of calendar months (1-12) to composite (default +\code{6:7}, June and July). Each year's window runs from the first day of the +earliest month to the last day of the latest; scenes from months not listed +are dropped, so \code{c(6, 8)} composites June and August and skips July.} + +\item{bands}{Character vector of band \strong{roles} from \code{\link[=dft_stac_config]{dft_stac_config()}} +(default \code{c("red", "green", "blue")}, true colour). Any roles and any +number of them; \code{c("nir", "red", "green")} is the standard vegetation false +colour. Layers are named by role, in the order given.} + +\item{source}{Character. A cube source name for \code{\link[=dft_stac_config]{dft_stac_config()}} (default +\code{"sentinel-2-l2a"}).} + +\item{res}{Numeric. Output pixel size in CRS units (default 10).} + +\item{crs}{Character. Target CRS as an EPSG string. When \code{NULL}, +auto-detected from the AOI centroid's UTM zone.} + +\item{aggregation}{Character. How scenes within a window are reduced +(default \code{"median"}).} + +\item{resampling}{Character. Spatial resampling (default \code{"bilinear"}).} + +\item{clip}{Logical. Clip the output to the AOI polygon (default \code{FALSE}). +Reference imagery is read around a place, not only inside it, so the +default keeps the whole AOI bounding box. \code{TRUE} uses the same rule as +\code{\link[=dft_stac_cube]{dft_stac_cube()}}: every cell the polygon touches is kept.} + +\item{cloud_cover_max}{Numeric. Scene-level \code{eo:cloud_cover} maximum percent +(default 20, stricter than the cube's 60 because a composite of few clear +scenes looks better than one of many cloudy ones).} + +\item{mask_values}{Integer vector of mask-band classes to exclude. When +\code{NULL}, taken from \code{\link[=dft_stac_config]{dft_stac_config()}}.} + +\item{tile_size}{Numeric or \code{NULL} (default). Read-tiling edge length in CRS +units; only tiles intersecting the AOI are streamed. A memory knob, not a +speed one; see \code{\link[=dft_stac_cube]{dft_stac_cube()}}.} + +\item{parallel}{Integer or \code{NULL}. gdalcubes worker processes; see +\code{\link[=dft_stac_cube]{dft_stac_cube()}}.} + +\item{cache_dir}{Character. Cache directory. When \code{NULL}, uses +\code{\link[=dft_cache_path]{dft_cache_path()}}.} + +\item{force}{Logical. Re-fetch even if cached (default \code{FALSE}). The +replacement is atomic.} + +\item{sign_fn}{A signing function for STAC assets. Default is +\code{\link[rstac:items_sign_planetary_computer]{rstac::sign_planetary_computer()}}.} +} +\value{ +A named list of \link[terra:SpatRaster-class]{terra::SpatRaster}s, one per year, each with one +layer per band. Names label the window, e.g. \code{"2017 Jun–Jul"}, and +become layer labels in \code{\link[=dft_map_interactive]{dft_map_interactive()}}. Each raster's +\code{\link[terra:time]{terra::time()}} is the window start. A year with no usable scenes is +dropped with a warning; if every year is empty, the call aborts. +} +\description{ +Reference imagery for windows you choose. For each year, streams every scene +in the chosen calendar months, masks clouds per pixel, and reduces them to +one median composite of the requested bands. A true-colour composite of the +same season in different years, or of early against late season within one +year, says \strong{when} a place changed. The Esri and Google basemaps in +\code{\link[=dft_map_interactive]{dft_map_interactive()}} carry no capture date, so they cannot. +} +\details{ +This is the multi-band sibling of \code{\link[=dft_stac_cube]{dft_stac_cube()}} and shares its read path: +the same STAC query (paginated, \code{intersects} the AOI, scene-level +\code{eo:cloud_cover} pre-filter), the same per-pixel mask, the same Sentinel-2 +offset correction at the 2022-01-25 processing-baseline boundary, the same +tiling and the same AOI clip. Where the cube returns one index per month, this +returns the bands themselves, one time step per window. + +Values are \strong{surface reflectance} (the source's \code{scale} and \code{offset} +applied), stored as floating point rather than display bytes. That keeps a +composite usable as data, for an index or as classifier input, and leaves +contrast stretching to display time. \code{\link[=dft_map_interactive]{dft_map_interactive()}} applies one +shared stretch to every composite it is given, so a brightness difference +between years is a real one. +} +\section{Caching}{ + +Each year's composite is written once under \code{\link[=dft_cache_path]{dft_cache_path()}} as +\verb{/composite_.tif}, a Cloud Optimized GeoTIFF. The key hashes the +AOI geometry and every parameter that changes the pixels: bands (in order), +months, the year's window, resolution, CRS, aggregation, resampling, cloud +cover, mask values, reflectance scale and offset, \code{clip} and \code{tile_size}. +Because the file is a COG, it can be copied to object storage and served +through titiler unchanged, which is how a floodplain-wide composite reaches +\code{\link[=dft_map_interactive]{dft_map_interactive()}}: \code{rgb} accepts COG URLs as well as rasters. +} + +\section{Floodplain scale and per-point chips}{ + +A composite over a whole floodplain streams the floodplain's bounding box +and produces a raster the size of it. When the purpose is reviewing sample +points, call this \strong{once per buffered point} instead (see the examples). +Each chip is small, streams only the COG blocks under it, and is cached on +its own, so adding a point to the sample fetches one chip rather than +invalidating the rest. Passing all the points as one multi-feature \code{aoi} +does not do this: the output spans the points' combined bounding box, and +one cache entry covers every point. +} + +\section{Windows and the Sentinel-2 offset boundary}{ + +A window is one calendar year's run of \code{months}, so a winter window that +crosses the new year (December to February) cannot be expressed; \code{c(12, 1, 2)} composites January, February and December of the \strong{same} year, and the +label says so. Sentinel-2 changed its reflectance offset on 2022-01-25. A +window with scenes on both sides of that date cannot be reduced to one true +median, because gdalcubes aggregates before the per-side offset is applied, +so such a window is refused rather than returned as something that is not a +median. Only a January 2022 window can straddle it. +} + +\examples{ +\dontrun{ +aoi <- sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift")) + +# True colour, same season in two years +tc <- dft_stac_composite(aoi, years = c(2017, 2023), months = 6:7) +names(tc) +terra::plotRGB(tc[[1]], stretch = "lin") + +# Vegetation false colour, late season +fc <- dft_stac_composite(aoi, years = 2023, months = 8:9, + bands = c("nir", "red", "green")) + +# Chips around sample points: one small, separately cached composite each +pts <- sf::st_as_sf(sf::st_sample(aoi, 20)) +buf <- sf::st_buffer(pts, 300) +chips <- lapply(seq_len(nrow(buf)), function(i) { + dft_stac_composite(buf[i, ], years = c(2017, 2023)) +}) + +dft_map_interactive(rgb = c(tc, fc[1]), aoi = aoi) +} + +} +\seealso{ +\code{\link[=dft_map_interactive]{dft_map_interactive()}} (\verb{rgb =}) to display them, +\code{\link[=dft_stac_cube]{dft_stac_cube()}} for index time series. +} diff --git a/planning/active/review-plan.md b/planning/active/review-plan.md new file mode 100644 index 0000000..6a7eedd --- /dev/null +++ b/planning/active/review-plan.md @@ -0,0 +1,38 @@ +# Plan review — #79 (Plan agent, 2026-09-28) + +This is the reviewer's report, condensed. Each finding carries the verdict reached after probing it. The reviewer ran read-only while Phase 2 was still uncommitted. + +## Blockers + +| # | Finding | Verdict | +|---|---|---| +| B1 | leafem ignores `domain` unless `quantiles = NULL`, and its defaults are `r=3,g=2,b=1`. `domain` is one range for all bands, so false-colour NIR saturates. | The draft already passes `r=1,g=2,b=3, quantiles=NULL`. **Real on the stretch:** change it to per-band quantiles pooled across composites, so years stay comparable, clamped in terra and passed with `domain = c(0, 1)`. | +| B2 | In a composite window, `cover(pre, post)` at the 2022-01-25 offset boundary is not a median. It applies only when a window contains Jan 2022 and has items on both sides. | **Real.** The composite refuses that case with a named error. It is pre-existing for the cube's Jan 2022 layer; note it in the gotchas file. | +| B3a | The network e2e lists `file.path(cache, "sentinel-2-l2a")`, but the cache has been under `v2/` since #48 (lines 405, 444). | **Real.** It was found independently in the #80 run. Fixed on the #80 branch and on this one. | +| B3b/c | A same-key run is a cache hit, and the baseline must come from HEAD. | **Already met.** The identity run used fresh caches, main-tree code and the same gdalcubes 0.7.5 build: identical values, max diff 0. | +| B3d | Nothing pins how `dft_stac_cube()` *calls* the key (argument order at the call site). The frozen test calls the function directly, and the gate tests mock it. | **Real.** Add a characterization test: seed at the real key, stub rstac, and assert the file is served. | + +## Gaps + +| # | Finding | Verdict | +|---|---|---| +| G1 | Chips via buffered points and `tile_size` produce a floodplain-sized, mostly-NA mosaic (merge over the tile union). One key covers all the points, so adding a point invalidates everything. | **Real, and it breaks the approved chip path.** Resolution, which stays within the plan's intent: one composite per point. The documented pattern is a loop over buffered points, one small cached entry per point, which also matches floodplains#93's "map zoomed to the point". The tile-count claim is dropped. | +| G2 | Composite key missing params | The draft already covers aggregation, resampling, url, collection, offset_before and a `"composite"` salt, with order tests. | +| G3 | One empty or clouded year aborts a multi-year call | **Real.** Warn and drop that year. Classed conditions replace message matching. Abort only if every year is empty. | +| G4 | COG write options, NoData, round trip | Pass `COMPRESS=DEFLATE`, `PREDICTOR=YES`, `OVERVIEW_RESAMPLING=AVERAGE` and `BLOCKSIZE=512`. The e2e asserts `LAYOUT=COG` and a NoData value. Make the cache-read warning label generic. | +| G5 | Multi-variable NetCDF read-back order is unverified | **Real risk:** renaming by position would mislabel silently. Assert that terra's layer names match the bands before renaming. | +| G6 | `x = NULL` touch points; rgb/x name clash; en dash in source | Touch points are handled. Add the clash check. Keep the en dash as `–` in code, and write "Jun-Jul" in roxygen. | +| G7 | Reuse `scale_token()` (done). Share the titiler URL base. `stac_cube_items` lacks the #51 paging safeguards. | Share the titiler URL base. The paging safeguards are a behaviour change for the cube and become a follow-up issue. | +| G8 | `clip = TRUE` removes the context a reviewer needs | **Agree:** the composite defaults to `clip = FALSE`. | +| G9 | A window that wraps the year (Dec–Feb) cannot be expressed | Document it; the label names the months actually composited. | + +## Ordering / assumptions / scope + +- A1: verified `nt == 1` for spans equal to dt. A span that is not a multiple of dt extends `t1`, so dt must equal the span, which it does by construction. The layer-count guard is in place. +- A2: a date-only STAC end bound may exclude the last day's ~19:00Z scenes. **Probe it**; it affects the cube too. +- A3: repeat `rescale=` per band. This is done, but was not verified against a live titiler; none is configured. +- A4: `image_mask(bits=)` exists, which answers part of the HLS spike. +- A6: the SCL mask may be read bilinear. Pre-existing; add to the gotchas note. +- A7: record temp disk alongside RSS in the scale test. +- S1: **Real, pre-existing, and it blocks the overlay.** Classified and transition layers are projected to 4326 and then passed with `project = FALSE`. leaflet stretches them linearly into Mercator bounds, so at floodplain extent they do not register with leafem's 3857 RGB. Fix: project to EPSG:3857. +- S2 / AC1: there is no titiler instance and no upload step. The COG path is tested offline by URL shape only; the PR says so. diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R new file mode 100644 index 0000000..2b2c9e0 --- /dev/null +++ b/tests/testthat/test-dft_stac_composite.R @@ -0,0 +1,248 @@ +s2_roles <- dft_stac_config("sentinel-2-l2a")$roles + +aoi_pkg <- function() { + sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift"), + quiet = TRUE) +} + +# Arguments for stac_composite_cache_key(), varied one at a time below +key_args <- function(...) { + aoi <- sf::st_transform(aoi_pkg(), 32609) + base <- list( + aoi_target = aoi, res = 10, target_crs = "EPSG:32609", + datetime = "2017-06-01/2017-07-31", dt = "P2M", aggregation = "median", + resampling = "bilinear", + stac_url = "https://planetarycomputer.microsoft.com/api/stac/v1", + collection = "sentinel-2-l2a", band_assets = c("B04", "B03", "B02"), + bands = c("red", "green", "blue"), cloud_cover_max = 20, + mask_values = c(3L, 8L, 9L, 10L, 11L), scale = 1e-4, offset = -0.1, + offset_before = 0, months = 6:7, clip = TRUE, tile_size = NULL + ) + utils::modifyList(base, list(...)) +} +ckey <- function(...) do.call(drift:::stac_composite_cache_key, key_args(...)) + +test_that("composite_window spans first-of-first to last-of-last month as one step", { + w <- drift:::composite_window(2017, 6:7) + expect_equal(w$t0, "2017-06-01") + expect_equal(w$t1, "2017-07-31") + expect_equal(w$datetime, "2017-06-01/2017-07-31") + expect_equal(w$dt, "P2M") + # a gap still spans first to last; the month filter drops the gap's scenes + expect_equal(drift:::composite_window(2023, c(6, 8))$dt, "P3M") + # February's end follows the calendar, leap year included + expect_equal(drift:::composite_window(2024, 2)$t1, "2024-02-29") + expect_equal(drift:::composite_window(2023, 2)$t1, "2023-02-28") + expect_equal(drift:::composite_window(2023, 12)$t1, "2023-12-31") +}) + +test_that("composite_label names the window without claiming skipped months", { + expect_equal(drift:::composite_label(2017, 6:7), "2017 Jun\u2013Jul") + expect_equal(drift:::composite_label(2023, 8:9), "2023 Aug\u2013Sep") + expect_equal(drift:::composite_label(2023, 7), "2023 Jul") + expect_equal(drift:::composite_label(2023, c(8, 6)), "2023 Jun, Aug") +}) + +test_that("stac_composite_cache_key is deterministic and a 16-char hex", { + expect_identical(ckey(), ckey()) + expect_match(ckey(), "^[0-9a-f]{16}$") +}) + +test_that("stac_composite_cache_key changes with every pixel-affecting parameter", { + base <- ckey() + variants <- list( + ckey(bands = c("nir", "red", "green"), band_assets = c("B08", "B04", "B03")), + ckey(bands = c("blue", "green", "red"), band_assets = c("B02", "B03", "B04")), + ckey(months = 8:9, datetime = "2017-08-01/2017-09-30"), + ckey(months = c(6L, 8L), datetime = "2017-06-01/2017-08-31", dt = "P3M"), + ckey(datetime = "2023-06-01/2023-07-31"), + ckey(res = 20), ckey(target_crs = "EPSG:32610"), + ckey(aggregation = "mean"), ckey(resampling = "near"), + ckey(cloud_cover_max = 60), ckey(mask_values = c(3L, 8L)), + ckey(scale = 1), ckey(offset = 0), ckey(offset_before = -0.1), + ckey(clip = FALSE), ckey(tile_size = 1000) + ) + keys <- vapply(variants, identity, character(1)) + expect_false(any(keys == base)) + expect_equal(length(unique(keys)), length(keys)) +}) + +test_that("stac_composite_cache_key ignores months and mask_values order", { + expect_identical(ckey(months = 7:6), ckey()) + expect_identical(ckey(mask_values = c(11L, 10L, 9L, 8L, 3L)), ckey()) +}) + +test_that("a composite key cannot equal a cube key over the same inputs", { + a <- key_args() + cube <- drift:::stac_cube_cache_key( + a$aoi_target, a$res, a$target_crs, a$dt, a$aggregation, a$resampling, + a$stac_url, a$collection, a$band_assets, a$datetime, "ndvi", + a$cloud_cover_max, a$mask_values, a$scale, a$offset, a$months, + a$offset_before, a$clip + ) + expect_false(identical(cube, ckey())) +}) + +test_that("composite_band_assets resolves roles in order and refuses bad roles", { + expect_equal(drift:::composite_band_assets(c("red", "green", "blue"), s2_roles), + c("B04", "B03", "B02")) + expect_equal(drift:::composite_band_assets(c("nir", "red", "green"), s2_roles), + c("B08", "B04", "B03")) + expect_error(drift:::composite_band_assets("swir22", s2_roles), + "Unknown band role.*Available roles") + expect_error(drift:::composite_band_assets("mask", s2_roles), "Unknown band role") + expect_error(drift:::composite_band_assets(c("red", "red"), s2_roles), "distinct") + expect_error(drift:::composite_band_assets(character(0), s2_roles), "distinct") +}) + +test_that("years and months are validated before any network call", { + expect_error(drift:::composite_years_check(numeric(0)), "whole-number years") + expect_error(drift:::composite_years_check(2017.5), "whole-number years") + expect_error(drift:::composite_years_check(NA_real_), "whole-number years") + expect_equal(drift:::composite_months_check(c(7, 6, 7)), c(6L, 7L)) + expect_error(drift:::composite_months_check(13), "1-12") + expect_error(drift:::composite_months_check(0), "1-12") + expect_error(drift:::composite_months_check(integer(0)), "1-12") +}) + +test_that("dft_stac_composite refuses a categorical source and bad input offline", { + skip_if_not_installed("gdalcubes") + aoi <- aoi_pkg() + # rstac::stac is stubbed: reaching it would mean validation ran too late + testthat::local_mocked_bindings( + stac = function(...) stop("reached the network"), .package = "rstac" + ) + expect_error(dft_stac_composite(aoi, years = 2017, source = "io-lulc"), + "not a cube source") + expect_error(dft_stac_composite(aoi, years = 2017, bands = "bogus"), + "Unknown band role") + expect_error(dft_stac_composite(aoi, years = 2017, months = 13), "1-12") +}) + +test_that("the composite pixel expression folds scale and offset into every band", { + exprs <- vapply(c("B04", "B03", "B02"), drift:::scale_token, character(1), + scale = 1e-4, offset = -0.1, USE.NAMES = FALSE) + expect_equal(exprs, c("(B04 * 0.0001 - 0.1)", "(B03 * 0.0001 - 0.1)", + "(B02 * 0.0001 - 0.1)")) + # post-2022 DN 1500 is reflectance 0.05 + expect_equal(eval(parse(text = exprs[1]), list(B04 = 1500)), 0.05) +}) + +test_that("composite_window gives the STAC query explicit times so the last day counts", { + # A date-only end bound is read as 00:00Z and drops that day's scenes, which in + # BC land ~19:00Z (measured: 22 of 23 items for a window ending on a scene day). + w <- drift:::composite_window(2021, 7:8) + expect_equal(w$query, "2021-07-01T00:00:00Z/2021-08-31T23:59:59Z") +}) + +test_that("a window straddling the offset boundary is refused, not covered", { + expect_error( + drift:::composite_offset_check(c(TRUE, FALSE), "2022 Jan", "2022-01-25"), + class = "drift_composite_offset_split" + ) + expect_true(drift:::composite_offset_check(c(TRUE, TRUE), "2021 Jan", "2022-01-25")) + expect_true(drift:::composite_offset_check(c(FALSE, FALSE), "2023 Jan", "2022-01-25")) +}) + +test_that("composite_layers_check catches a second time step and a band reorder", { + r <- terra::rast(nrows = 2, ncols = 2, nlyrs = 3, vals = 1) + names(r) <- c("red", "green", "blue") + w <- drift:::composite_window(2017, 6:7) + expect_true(drift:::composite_layers_check(r, c("red", "green", "blue"), "x", w)) + expect_error(drift:::composite_layers_check(c(r, r), c("red", "green", "blue"), "x", w), + "expected 3") + names(r) <- c("blue", "green", "red") + expect_error(drift:::composite_layers_check(r, c("red", "green", "blue"), "x", w), + "not in band order") +}) + +test_that("an empty year is dropped with a warning and the others are kept", { + skip_if_not_installed("gdalcubes") + aoi <- aoi_pkg() + cache <- tempfile("drift_composite_skip_") + # 2017 has no scenes; 2018 is served from a seeded cache entry + testthat::local_mocked_bindings( + stac_cube_items = function(...) { + rlang::abort("No STAC items found for sentinel-2-l2a", class = "drift_no_items") + } + ) + seeded <- NULL + testthat::local_mocked_bindings( + stac_composite_cache_key = function(aoi_target, res, target_crs, datetime, ...) { + if (startsWith(datetime, "2018")) "seeded2018000000" else "empty2017000000" + } + ) + dir <- drift:::cache_scheme_dir(cache, "sentinel-2-l2a") + dir.create(dir, recursive = TRUE) + r <- terra::rast(nrows = 4, ncols = 4, nlyrs = 3, vals = 0.1, + crs = "EPSG:32609", extent = c(0, 40, 0, 40)) + terra::writeRaster(r, file.path(dir, "composite_seeded2018000000.tif")) + expect_warning( + out <- suppressMessages( + dft_stac_composite(aoi, years = c(2017, 2018), cache_dir = cache) + ), + "Skipping the 2017 Jun.Jul composite: no scenes" + ) + expect_named(out, "2018 Jun\u2013Jul") + expect_equal(names(out[[1]]), c("red", "green", "blue")) + # every year empty is an error, not an empty list + expect_error( + suppressWarnings(suppressMessages( + dft_stac_composite(aoi, years = 2017, cache_dir = cache) + )), + "No year produced a composite" + ) +}) + +test_that("dft_stac_composite returns a cached, cloud-masked true-colour COG", { + skip_if(Sys.getenv("DRIFT_TEST_NETWORK") != "true", + "network test; set DRIFT_TEST_NETWORK=true to run") + skip_if_not_installed("gdalcubes") + aoi <- aoi_pkg() + cache <- tempfile("drift_composite_") + tc <- suppressMessages(dft_stac_composite(aoi, years = 2021, months = 7:8, + cache_dir = cache)) + expect_named(tc, "2021 Jul\u2013Aug") + r <- tc[[1]] + expect_equal(terra::nlyr(r), 3) + expect_equal(names(r), c("red", "green", "blue")) + expect_equal(terra::time(r)[1], as.Date("2021-07-01")) + v <- terra::values(r) + # surface reflectance, not DN: summer land is well under 0.5 in the visible + expect_gt(mean(!is.na(v)), 0.3) + expect_lt(stats::median(v, na.rm = TRUE), 0.3) + expect_gt(stats::median(v, na.rm = TRUE), 0) + + f <- list.files(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), + pattern = "^composite_.*\\.tif$", full.names = TRUE) + expect_length(f, 1) + # A COG by its layout, not by an exit status: GDAL reports the layout in the + # image structure metadata only for a file written as a COG. + info <- paste(sf::gdal_utils("info", f, quiet = TRUE), collapse = "\n") + expect_match(info, "LAYOUT=COG") + # titiler masks only what the file declares as NoData + expect_match(info, "NoData Value=") + + # second call is a cache hit and returns the same pixels + msgs <- testthat::capture_messages( + tc2 <- dft_stac_composite(aoi, years = 2021, months = 7:8, cache_dir = cache) + ) + expect_true(any(grepl("cached", msgs))) + expect_equal(terra::values(tc2[[1]]), v) + expect_equal(names(tc2[[1]]), c("red", "green", "blue")) +}) + +test_that("false colour keeps band order: NIR is brightest over summer vegetation", { + skip_if(Sys.getenv("DRIFT_TEST_NETWORK") != "true", + "network test; set DRIFT_TEST_NETWORK=true to run") + skip_if_not_installed("gdalcubes") + fc <- suppressMessages(dft_stac_composite( + aoi_pkg(), years = 2021, months = 7:8, bands = c("nir", "red", "green"), + cache_dir = tempfile("drift_composite_fc_") + ))[[1]] + med <- vapply(names(fc), function(b) { + stats::median(terra::values(fc[[b]]), na.rm = TRUE) + }, numeric(1)) + expect_equal(names(med), c("nir", "red", "green")) + expect_gt(med[["nir"]], 2 * med[["red"]]) +}) diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index e535f7b..babdf46 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -660,3 +660,34 @@ test_that("an empty cached cube still ABORTS rather than re-fetching", { "no data on any layer" ) }) + +test_that("dft_stac_cube computes its cache key from its own arguments (call site pinned)", { + # The frozen-key test calls stac_cube_cache_key() directly and the gate tests + # mock it, so neither would notice the call site passing arguments in a new + # order. Seed a healthy cube at the key computed from explicit arguments and + # require dft_stac_cube() to serve it without reaching the network. + skip_if_not_installed("gdalcubes") + aoi <- sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift"), + quiet = TRUE) + cfg <- dft_stac_config("sentinel-2-l2a") + crs <- drift:::auto_utm_epsg(aoi) + aoi_t <- sf::st_transform(aoi, as.integer(gsub("EPSG:", "", crs))) + key <- drift:::stac_cube_cache_key( + aoi_t, 10, crs, "P1M", "median", "bilinear", cfg$stac_url, cfg$collection, + c("B08", "B04"), "2020-06-01/2020-07-31", "ndvi", 60, cfg$mask_values, + cfg$scale, cfg$offset, NULL, cfg$offset_before, TRUE + ) + cache <- tempfile("drift_cube_callsite_") + dir <- drift:::cache_scheme_dir(cache, "sentinel-2-l2a") + dir.create(dir, recursive = TRUE) + r <- terra::rast(terra::ext(aoi_t), resolution = 10, crs = crs, nlyrs = 2, + vals = 0.5) + terra::writeRaster(r, file.path(dir, paste0("cube_", key, ".tif"))) + testthat::local_mocked_bindings( + stac = function(...) stop("fell through to a re-fetch"), .package = "rstac" + ) + out <- suppressMessages(dft_stac_cube(aoi, index = "ndvi", + datetime = "2020-06-01/2020-07-31", + cache_dir = cache)) + expect_equal(terra::nlyr(out), 2) +}) From 515c9ef79fb79a4ffdc30dce3b24b6318ffa820f Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:36:49 -0700 Subject: [PATCH 10/29] Add dated RGB layers to dft_map_interactive (#79) rgb takes composites as rasters (leafem) or COG URLs (titiler), drawn beneath the classified layers with one stretch shared across years, so a difference between years is in the data. x may be NULL for an imagery-only map. Classified and transition layers now reach leaflet in EPSG:3857: with project = FALSE leaflet assumes Mercator pixels, so the 4326 grid was misregistered against anything drawn beneath it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_map_interactive.R | 244 ++++++++++++++++++++-- man/dft_map_interactive.Rd | 37 +++- planning/active/progress.md | 17 ++ planning/active/task_plan.md | 24 +-- tests/testthat/test-dft_map_interactive.R | 115 ++++++++++ 5 files changed, 404 insertions(+), 33 deletions(-) diff --git a/R/dft_map_interactive.R b/R/dft_map_interactive.R index bf66449..f9c36df 100644 --- a/R/dft_map_interactive.R +++ b/R/dft_map_interactive.R @@ -9,16 +9,38 @@ #' type (e.g. Trees -> Rangeland) is added as a checkbox overlay that can be #' shown simultaneously on top of any classified layer. #' +#' When `rgb` is supplied, each composite (e.g. from [dft_stac_composite()]) is +#' added as a switchable overlay beneath the classified and transition layers, +#' so dated imagery can be toggled under a land-cover label to check it. Every +#' composite gets **the same stretch**: each band's display range is set once, +#' from all the composites pooled, so a brightness or colour difference between +#' two years is in the data and not an artefact of stretching each image to its +#' own histogram. +#' #' @param x A named list of classified [terra::SpatRaster]s (e.g. from #' [dft_rast_classify()]) **or** a named character vector of COG URLs. #' A single `SpatRaster` or URL string is auto-wrapped into a length-1 #' list/vector. Names become the layer toggle labels (years, seasons, etc.). +#' May be `NULL` when `rgb` is supplied, for an imagery-only map. #' @param aoi An `sf` polygon for the area of interest outline. `NULL` (default) #' omits the AOI layer. #' @param transition Output of [dft_rast_transition()] — a list with elements #' `raster` (factor `SpatRaster`) and `summary` (tibble). Each transition #' type becomes a toggleable overlay. Stable transitions (same from/to class) #' are excluded by default. `NULL` (default) omits transition overlays. +#' @param rgb Dated reference imagery: a named list of three-band +#' [terra::SpatRaster]s in red, green, blue order (the output of +#' [dft_stac_composite()] with its default `bands`), **or** a named character +#' vector of three-band COG URLs served through `titiler_url`. Names become +#' the layer labels, e.g. `"2017 Jun–Jul"`. Local rasters are drawn with +#' [leafem::addRasterRGB()], which reads every cell into memory; for a +#' floodplain-sized composite, publish the cached COG and pass its URL +#' instead. `NULL` (default) adds no imagery. +#' @param rgb_rescale Numeric length-2 reflectance range stretched to the full +#' display range, the same for every band of every `rgb` layer. When `NULL`, +#' local rasters stretch each band from its 2nd to 98th percentile across all +#' composites pooled, and COG URLs use `c(0, 0.3)`, a range that suits +#' vegetated land in surface reflectance. #' @param class_table A tibble with columns `code`, `class_name`, `color` #' (hex). When `NULL`, loaded via [dft_class_table()] using `source`. #' @param source Character. Used to load a shipped class table when @@ -36,7 +58,8 @@ #' #' @return A [leaflet::leaflet] htmlwidget. The first layer in `x` is visible #' by default; other layers are hidden but toggleable. Transition overlays -#' are visible by default when supplied. +#' are visible by default when supplied. `rgb` layers are hidden when `x` is +#' supplied (toggle them on beneath it); without `x` the first is shown. #' @export #' @examples #' # Single classified raster — returns a leaflet widget @@ -70,10 +93,16 @@ #' cogs <- c("2017" = "https://bucket.s3.amazonaws.com/lulc_2017.tif", #' "2023" = "https://bucket.s3.amazonaws.com/lulc_2023.tif") #' dft_map_interactive(cogs, source = "io-lulc") +#' +#' # Dated true-colour composites beneath the land cover (network + gdalcubes) +#' tc <- dft_stac_composite(aoi, years = c(2017, 2023), months = 6:7) +#' dft_map_interactive(classified, aoi = aoi, rgb = tc) #' } -dft_map_interactive <- function(x, +dft_map_interactive <- function(x = NULL, aoi = NULL, transition = NULL, + rgb = NULL, + rgb_rescale = NULL, class_table = NULL, source = "io-lulc", titiler_url = getOption("drift.titiler_url"), @@ -97,6 +126,10 @@ dft_map_interactive <- function(x, } } + if (is.null(x) && is.null(rgb)) { + cli::cli_abort("Supply {.arg x} (classified layers), {.arg rgb} (imagery), or both.") + } + # Detect mode and normalize input cog_mode <- is.character(x) @@ -117,12 +150,23 @@ dft_map_interactive <- function(x, } } + rgb <- rgb_layers_check(rgb, titiler_url) + rgb_cog <- is.character(rgb) + clash <- intersect(names(rgb), names(x)) + if (length(clash)) { + cli::cli_abort(c( + "{.arg rgb} and {.arg x} share layer name{?s} {.val {clash}}.", + "i" = "Names become layer groups, so a shared name merges two layers." + )) + } + # Compute map center if (!is.null(aoi)) { bbox <- sf::st_bbox(sf::st_transform(aoi, 4326)) - } else if (!cog_mode) { - ext <- terra::ext(terra::project(x[[1]], "EPSG:4326")) - bbox <- c(xmin = ext[1], ymin = ext[3], xmax = ext[2], ymax = ext[4]) + } else if (!is.null(x) && !cog_mode) { + bbox <- raster_bbox_4326(x[[1]]) + } else if (!is.null(rgb) && !rgb_cog) { + bbox <- raster_bbox_4326(rgb[[1]]) } else { bbox <- NULL } @@ -149,17 +193,28 @@ dft_map_interactive <- function(x, } } + # Dated imagery first, so it draws beneath the classified and transition + # layers: leaflet stacks overlays in the order they are added. + if (!is.null(rgb)) { + map <- add_rgb_layers(map, rgb, rgb_rescale, titiler_url) + } + # Add classified layers - if (cog_mode) { + if (is.null(x)) { + # imagery-only map + } else if (cog_mode) { for (nm in names(x)) { tile_url <- build_titiler_url(titiler_url, x[[nm]], class_table) map <- leaflet::addTiles(map, urlTemplate = tile_url, group = nm) } } else { for (nm in names(x)) { + # EPSG:3857, not 4326: with project = FALSE leaflet places the pixels + # linearly between Web Mercator bounds, so a 4326 grid is misregistered + # in latitude, which shows once RGB imagery (drawn in 3857) sits beneath. map <- leaflet::addRasterImage( map, - terra::project(x[[nm]], "EPSG:4326"), + terra::project(x[[nm]], "EPSG:3857", method = "near"), group = nm, project = FALSE ) @@ -167,7 +222,7 @@ dft_map_interactive <- function(x, } # Track overlay groups (classified layers are radio-toggled via hideGroup) - overlay_groups <- names(x) + overlay_groups <- c(names(rgb), names(x)) # Add transition overlays trans_groups <- character(0) @@ -189,8 +244,8 @@ dft_map_interactive <- function(x, overlay_groups <- c(overlay_groups, "AOI") } - # Legend — land cover classes - if (!is.null(legend_position)) { + # Legend — land cover classes (none for an imagery-only map) + if (!is.null(legend_position) && !is.null(x)) { if (cog_mode) { ct_legend <- class_table[class_table$class_name != "No Data", ] } else { @@ -230,14 +285,168 @@ dft_map_interactive <- function(x, overlayGroups = overlay_groups, options = leaflet::layersControlOptions(collapsed = FALSE) ) - # Hide all classified layers except the first (radio-style toggle) - map <- leaflet::hideGroup(map, setdiff(names(x), names(x)[1])) + # Hide all classified layers except the first (radio-style toggle). Imagery + # sits beneath the classified layers, so it starts hidden when they are + # present; on an imagery-only map the first composite is shown. + hidden <- setdiff(names(x), names(x)[1]) + hidden <- c(hidden, if (is.null(x)) setdiff(names(rgb), names(rgb)[1]) else names(rgb)) + if (length(hidden)) map <- leaflet::hideGroup(map, hidden) map <- leaflet.extras::addFullscreenControl(map) map } +#' Validate and normalize the `rgb` argument of dft_map_interactive() +#' +#' Returns `NULL`, a named list of three-band SpatRasters, or a named character +#' vector of COG URLs. A single raster or unnamed URL is wrapped as `"RGB"`. +#' @noRd +rgb_layers_check <- function(rgb, titiler_url) { + if (is.null(rgb)) return(NULL) + if (inherits(rgb, "SpatRaster")) rgb <- list(RGB = rgb) + if (is.character(rgb)) { + if (is.null(names(rgb)) && length(rgb) == 1L) rgb <- stats::setNames(rgb, "RGB") + if (is.null(titiler_url)) { + cli::cli_abort(c( + "COG URLs in {.arg rgb} need a titiler URL.", + "i" = "Set {.code options(drift.titiler_url = \"https://...\")}." + )) + } + } else if (!is.list(rgb) || + !all(vapply(rgb, inherits, logical(1), what = "SpatRaster"))) { + cli::cli_abort( + "{.arg rgb} must be a named list of SpatRasters or a named character vector of COG URLs." + ) + } + nm <- names(rgb) + if (is.null(nm) || anyNA(nm) || any(!nzchar(nm)) || anyDuplicated(nm)) { + cli::cli_abort("{.arg rgb} must be named, with distinct names (they become layer labels).") + } + if (is.list(rgb)) { + n <- vapply(rgb, terra::nlyr, numeric(1)) + bad <- nm[n != 3] + if (length(bad)) { + cli::cli_abort(c( + "Every {.arg rgb} layer must have three bands (red, green, blue).", + "x" = "Not three: {.val {bad}}." + )) + } + } + rgb +} + + +#' The shared stretch for local RGB layers: per band, pooled across composites +#' +#' Returns a 2 x 3 matrix of (low, high) per band: each band's 2nd and 98th +#' percentile over every composite pooled, from a regular sample so a large +#' raster is not read whole to find six numbers. Per band keeps a false-colour +#' NIR channel from saturating; pooled keeps two years on one scale. +#' @noRd +rgb_domain <- function(rgb, probs = c(0.02, 0.98), size = 1e5) { + smp <- do.call(rbind, lapply(rgb, function(r) { + v <- terra::spatSample(r, size = size, method = "regular", as.df = TRUE) + names(v) <- paste0("b", 1:3) + v + })) + dom <- vapply(smp, function(z) { + z <- z[is.finite(z)] + if (!length(z)) return(c(NA_real_, NA_real_)) + unname(stats::quantile(z, probs)) + }, numeric(2)) + if (anyNA(dom)) cli::cli_abort("The {.arg rgb} layers have no data to stretch.") + flat <- dom[2, ] <= dom[1, ] + dom[2, flat] <- dom[1, flat] + 1e-6 + dom +} + + +#' Stretch a three-band raster to 0-1 with a fixed per-band range, clamped +#' @noRd +rgb_stretch <- function(r, dom) { + out <- (r - dom[1, ]) / (dom[2, ] - dom[1, ]) + terra::clamp(out, lower = 0, upper = 1, values = TRUE) +} + + +#' Add dated RGB layers to a leaflet map, with one shared stretch +#' +#' Local rasters are stretched here, in terra, and handed to leafem already on +#' 0-1 with `quantiles = NULL, domain = c(0, 1)`: leafem otherwise stretches +#' each layer to its own histogram (its `quantiles` default wins over `domain`), +#' which is exactly the per-image stretch this avoids. Bands are 1, 2, 3 = +#' red, green, blue; leafem's own default is 3, 2, 1. +#' @noRd +add_rgb_layers <- function(map, rgb, rgb_rescale, titiler_url) { + if (!is.null(rgb_rescale) && + (!is.numeric(rgb_rescale) || length(rgb_rescale) != 2L || + anyNA(rgb_rescale) || rgb_rescale[2] <= rgb_rescale[1])) { + cli::cli_abort("{.arg rgb_rescale} must be two increasing numbers, e.g. {.code c(0, 0.3)}.") + } + if (is.character(rgb)) { + rescale <- rgb_rescale %||% c(0, 0.3) + for (nm in names(rgb)) { + map <- leaflet::addTiles( + map, urlTemplate = build_titiler_rgb_url(titiler_url, rgb[[nm]], rescale), + group = nm + ) + } + return(map) + } + rlang::check_installed("leafem", reason = "to draw RGB layers") + dom <- if (is.null(rgb_rescale)) rgb_domain(rgb) else matrix(rgb_rescale, 2, 3) + for (nm in names(rgb)) { + map <- leafem::addRasterRGB( + map, rgb_stretch(rgb[[nm]], dom), r = 1, g = 2, b = 3, + quantiles = NULL, domain = c(0, 1), + na.color = "#00000000", group = nm + ) + } + map +} + + +#' Build a titiler tile URL template for a three-band COG with a fixed stretch +#' +#' The rescale is repeated once per band so every band gets the same range +#' whatever titiler does with a single value. +#' @noRd +build_titiler_rgb_url <- function(titiler_url, cog_url, rescale) { + # per element: format() on a vector pads to a common width ("0.0,0.3") + rs <- paste(vapply(rescale, format, character(1), scientific = FALSE, + trim = TRUE), collapse = ",") + titiler_tile_url( + titiler_url, cog_url, + paste0("&bidx=1&bidx=2&bidx=3", + strrep(paste0("&rescale=", utils::URLencode(rs, reserved = TRUE)), 3L)) + ) +} + + +#' The titiler tile URL template both COG layer kinds share +#' +#' `query` is appended as given, so it must already be percent-encoded. +#' @noRd +titiler_tile_url <- function(titiler_url, cog_url, query) { + paste0( + titiler_url, "/cog/tiles/WebMercatorQuad/{z}/{x}/{y}.png", + "?url=", utils::URLencode(cog_url, reserved = TRUE), + query + ) +} + + +#' A raster's extent in EPSG:4326 as a named bbox vector +#' @noRd +raster_bbox_4326 <- function(r) { + # terra::rast(r) is deliberately the EMPTY template: only the extent is + # wanted, so projecting geometry alone avoids resampling every cell. + ext <- terra::ext(terra::project(terra::rast(r), "EPSG:4326")) + c(xmin = ext[1], ymin = ext[3], xmax = ext[2], ymax = ext[4]) +} + + #' Add transition overlay layers to a leaflet map #' #' @param map A leaflet map object. @@ -287,7 +496,7 @@ add_transition_layers <- function(map, transition, class_table) { "#999999" } - r_proj <- terra::project(r_mask, "EPSG:4326", method = "near") + r_proj <- terra::project(r_mask, "EPSG:3857", method = "near") map <- leaflet::addRasterImage( map, r_proj, group = label, colors = col, project = FALSE @@ -333,10 +542,9 @@ build_titiler_url <- function(titiler_url, cog_url, class_table) { "}" ) - paste0( - titiler_url, "/cog/tiles/WebMercatorQuad/{z}/{x}/{y}.png", - "?url=", utils::URLencode(cog_url, reserved = TRUE), - "&bidx=1", - "&colormap=", utils::URLencode(colormap_json, reserved = TRUE) + titiler_tile_url( + titiler_url, cog_url, + paste0("&bidx=1", + "&colormap=", utils::URLencode(colormap_json, reserved = TRUE)) ) } diff --git a/man/dft_map_interactive.Rd b/man/dft_map_interactive.Rd index 0427230..ad34693 100644 --- a/man/dft_map_interactive.Rd +++ b/man/dft_map_interactive.Rd @@ -5,9 +5,11 @@ \title{Interactive leaflet map for classified rasters and transitions} \usage{ dft_map_interactive( - x, + x = NULL, aoi = NULL, transition = NULL, + rgb = NULL, + rgb_rescale = NULL, class_table = NULL, source = "io-lulc", titiler_url = getOption("drift.titiler_url"), @@ -22,7 +24,8 @@ dft_map_interactive( \item{x}{A named list of classified \link[terra:SpatRaster-class]{terra::SpatRaster}s (e.g. from \code{\link[=dft_rast_classify]{dft_rast_classify()}}) \strong{or} a named character vector of COG URLs. A single \code{SpatRaster} or URL string is auto-wrapped into a length-1 -list/vector. Names become the layer toggle labels (years, seasons, etc.).} +list/vector. Names become the layer toggle labels (years, seasons, etc.). +May be \code{NULL} when \code{rgb} is supplied, for an imagery-only map.} \item{aoi}{An \code{sf} polygon for the area of interest outline. \code{NULL} (default) omits the AOI layer.} @@ -32,6 +35,21 @@ omits the AOI layer.} type becomes a toggleable overlay. Stable transitions (same from/to class) are excluded by default. \code{NULL} (default) omits transition overlays.} +\item{rgb}{Dated reference imagery: a named list of three-band +\link[terra:SpatRaster-class]{terra::SpatRaster}s in red, green, blue order (the output of +\code{\link[=dft_stac_composite]{dft_stac_composite()}} with its default \code{bands}), \strong{or} a named character +vector of three-band COG URLs served through \code{titiler_url}. Names become +the layer labels, e.g. \code{"2017 Jun–Jul"}. Local rasters are drawn with +\code{\link[leafem:addRasterRGB]{leafem::addRasterRGB()}}, which reads every cell into memory; for a +floodplain-sized composite, publish the cached COG and pass its URL +instead. \code{NULL} (default) adds no imagery.} + +\item{rgb_rescale}{Numeric length-2 reflectance range stretched to the full +display range, the same for every band of every \code{rgb} layer. When \code{NULL}, +local rasters stretch each band from its 2nd to 98th percentile across all +composites pooled, and COG URLs use \code{c(0, 0.3)}, a range that suits +vegetated land in surface reflectance.} + \item{class_table}{A tibble with columns \code{code}, \code{class_name}, \code{color} (hex). When \code{NULL}, loaded via \code{\link[=dft_class_table]{dft_class_table()}} using \code{source}.} @@ -55,7 +73,8 @@ Set to \code{NULL} to suppress the legend.} \value{ A \link[leaflet:leaflet]{leaflet::leaflet} htmlwidget. The first layer in \code{x} is visible by default; other layers are hidden but toggleable. Transition overlays -are visible by default when supplied. +are visible by default when supplied. \code{rgb} layers are hidden when \code{x} is +supplied (toggle them on beneath it); without \code{x} the first is shown. } \description{ Build a toggleable leaflet map from classified \code{SpatRaster}s or remote COG @@ -67,6 +86,14 @@ When only \code{x} is supplied, classified layers appear as radio-toggle overlay (one visible at a time). When \code{transition} is also supplied, each transition type (e.g. Trees -> Rangeland) is added as a checkbox overlay that can be shown simultaneously on top of any classified layer. + +When \code{rgb} is supplied, each composite (e.g. from \code{\link[=dft_stac_composite]{dft_stac_composite()}}) is +added as a switchable overlay beneath the classified and transition layers, +so dated imagery can be toggled under a land-cover label to check it. Every +composite gets \strong{the same stretch}: each band's display range is set once, +from all the composites pooled, so a brightness or colour difference between +two years is in the data and not an artefact of stretching each image to its +own histogram. } \examples{ # Single classified raster — returns a leaflet widget @@ -100,5 +127,9 @@ if (interactive()) map cogs <- c("2017" = "https://bucket.s3.amazonaws.com/lulc_2017.tif", "2023" = "https://bucket.s3.amazonaws.com/lulc_2023.tif") dft_map_interactive(cogs, source = "io-lulc") + +# Dated true-colour composites beneath the land cover (network + gdalcubes) +tc <- dft_stac_composite(aoi, years = c(2017, 2023), months = 6:7) +dft_map_interactive(classified, aoi = aoi, rgb = tc) } } diff --git a/planning/active/progress.md b/planning/active/progress.md index 87eb0b8..53d3408 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -8,3 +8,20 @@ - Next: Phase 1 (NDWI/MNDWI). - Phase 1 done. Added the `green`/`blue` S2 roles and the `ndwi`/`mndwi` registry rows. The tests evaluate the resolved expressions over known reflectance, and over post-2022 DN through the offset. A mutation check swapping NDWI's sign failed 4 tests. The frozen cube key is unchanged. - Phase 2 refactor landed as `stac_cube_session()`, `stac_cube_cache_read()`, `stac_cube_items()` and `stac_cube_assemble(pixel_fn=)`. The offline cube tests pass 90/90, including the frozen key and the mocked cache gates. **Live byte identity:** an untiled NDVI cube (packaged AOI, 2021-07/08, parallel 4) from main against the refactor gives identical values and NA pattern, max diff 0, 77.1 s vs 77.7 s. Recorded in the scratchpad `cube_identity.R`. The tiled check is pending, in the worktree network run. +- Plan review (Plan agent) is in `review-plan.md`, with a verdict on each finding. Folded in: + - the per-band shared stretch, with channel order pinned; + - the composite refuses a window straddling the 2022 offset split; + - the network e2e cache paths (also fixed on #80); + - a cube key call-site characterization test; + - chips are one call per point; + - the composite key set; + - an empty year is dropped with a warning (classed `drift_no_items` / `drift_empty_cube`); + - the COG gdal options, with a NoData assertion; + - a layer-order guard; + - the rgb/x name clash check; + - classified layers are handed to leaflet in 3857 (a pre-existing misregistration); + - a shared titiler URL base; + - the composite defaults to `clip = FALSE`; + - wrap-year windows are documented. +- Probed A2 live: a date-only STAC end bound drops the last day (22 of 23 items). The composite queries with `T00:00:00Z/T23:59:59Z`. The cube has the same issue, pre-existing, and goes to a follow-up issue because fixing it changes what existing cache keys hold. +- Phases 3 and 4 are implemented. Mutation checks: band order, 4326 projection, key call site, offset split, per-image stretch and empty-year skip each turn a test red. The offline suite passes 1089, fails 0, skips 15. diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md index 413b3cb..6302561 100644 --- a/planning/active/task_plan.md +++ b/planning/active/task_plan.md @@ -29,25 +29,25 @@ Plan-gate decisions (2026-09-28): ## Phase 3: `dft_stac_composite()` -- [ ] `R/dft_stac_composite.R` with this signature: `aoi, years, months = 6:7, bands = c("red","green","blue"), source = "sentinel-2-l2a", res = 10, crs = NULL, aggregation = "median", resampling = "bilinear", clip = TRUE, cloud_cover_max = 20, mask_values = NULL, tile_size = NULL, parallel = NULL, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()`. `bands` takes any roles and any number of them. The map needs exactly 3. -- [ ] Each year gets one window: `t0` = 1st of `min(months)`, `t1` = end of `max(months)`, `dt = P{span}M`. Items are filtered to `months`. The result is one time step: a median, cloud-masked **reflectance** stack (scale/offset per band via `scale_token()`) with layers named by role. -- [ ] Return a named list labelled like `"2017 Jun–Jul"`. A single year returns a length-1 list. -- [ ] The cache is `/composite_.tif`, **written as a COG** (`filetype = "COG"`) so the cached file can go to S3 and titiler unchanged. It is written through `cache_write_atomic()`, and read back only after `cache_hit_ok()` and the non-empty check pass. `stac_composite_cache_key()` covers geometry, res, crs, bands (resolved assets, order-sensitive), months (sorted), year window, cloud_cover_max, mask_values, scale/offset, clip and tile_size. -- [ ] **Per-point chips.** A documented example and a test show the chip path: `aoi` = sample points buffered to polygons, `tile_size` set. Through `tile_grid()` only tiles that intersect the points stream. An offline test on the grid confirms scattered buffered points select a small share of tiles. -- [ ] Validation: an unknown band role errors with the available roles; empty `years`/`months` or months outside 1–12 abort; a non-cube source aborts. -- [ ] Tests (offline): key determinism and sensitivity; labels (range, single month, non-contiguous); window derivation, including February's end in a leap year; validation; the pixel expression. One network e2e on the packaged AOI: 3 bands, plausible reflectance, a valid COG (`gdalinfo` layout, not an exit code), cached on the second call. -- [ ] Roxygen with true-colour and false-colour (`c("nir","red","green")`) examples. Run `devtools::document()` and `pkgdown::check_pkgdown()`. +- [x] `R/dft_stac_composite.R` with this signature: `aoi, years, months = 6:7, bands = c("red","green","blue"), source = "sentinel-2-l2a", res = 10, crs = NULL, aggregation = "median", resampling = "bilinear", clip = TRUE, cloud_cover_max = 20, mask_values = NULL, tile_size = NULL, parallel = NULL, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()`. `bands` takes any roles and any number of them. The map needs exactly 3. +- [x] Each year gets one window: `t0` = 1st of `min(months)`, `t1` = end of `max(months)`, `dt = P{span}M`. Items are filtered to `months`. The result is one time step: a median, cloud-masked **reflectance** stack (scale/offset per band via `scale_token()`) with layers named by role. +- [x] Return a named list labelled like `"2017 Jun–Jul"`. A single year returns a length-1 list. +- [x] The cache is `/composite_.tif`, **written as a COG** (`filetype = "COG"`) so the cached file can go to S3 and titiler unchanged. It is written through `cache_write_atomic()`, and read back only after `cache_hit_ok()` and the non-empty check pass. `stac_composite_cache_key()` covers geometry, res, crs, bands (resolved assets, order-sensitive), months (sorted), year window, cloud_cover_max, mask_values, scale/offset, clip and tile_size. +- [x] **Per-point chips.** *(Revised after the plan review, G1: one `aoi` of buffered points plus `tile_size` gives a floodplain-sized, mostly-NA mosaic under a single cache key. The chip contract is now one call per buffered point, so each chip is small and cached separately. It is documented in the roxygen chips section with an example. The tile-count test was dropped because the claim it tested no longer holds.)* +- [x] Validation: an unknown band role errors with the available roles; empty `years`/`months` or months outside 1–12 abort; a non-cube source aborts. +- [x] Tests (offline): key determinism and sensitivity; labels (range, single month, non-contiguous); window derivation, including February's end in a leap year; validation; the pixel expression. One network e2e on the packaged AOI: 3 bands, plausible reflectance, a valid COG (`gdalinfo` layout, not an exit code), cached on the second call. +- [x] Roxygen with true-colour and false-colour (`c("nir","red","green")`) examples. Run `devtools::document()` and `pkgdown::check_pkgdown()`. ## Phase 4: RGB layers in `dft_map_interactive()` -- [ ] New `rgb = NULL` argument: a named list of 3-band SpatRasters **or** a named character vector of COG URLs. That mirrors `x`'s dual mode. +- [x] New `rgb = NULL` argument: a named list of 3-band SpatRasters **or** a named character vector of COG URLs. That mirrors `x`'s dual mode. - Local: `leafem::addRasterRGB()`. leafem is already in Suggests; call `check_installed()` only when it is used. - COG: a titiler URL with `bidx=1&bidx=2&bidx=3&rescale=…`, built by a sibling of `build_titiler_url()`. -- [ ] Every RGB layer gets **one shared stretch**, so a brightness difference between years is real: +- [x] Every RGB layer gets **one shared stretch**, so a brightness difference between years is real: - local mode uses a pooled 2–98% `domain` across all composites; - COG mode uses an `rgb_rescale` argument, defaulting to a documented reflectance range. -- [ ] RGB groups are switchable overlays beneath the classified and transition layers. The first is visible when `x` is `NULL`, and they are hidden otherwise. `x` may be `NULL` when `rgb` is given. Centring and legends handle that case. -- [ ] Tests: synthetic 3-band rasters; groups in the layer control; `x = NULL`; no land-cover legend without `x`; a non-3-band layer errors by name; the shared domain is applied; the COG URL has 3 `bidx` and the shared rescale, is percent-encoded, and errors without `titiler_url`. +- [x] RGB groups are switchable overlays beneath the classified and transition layers. The first is visible when `x` is `NULL`, and they are hidden otherwise. `x` may be `NULL` when `rgb` is given. Centring and legends handle that case. +- [x] Tests: synthetic 3-band rasters; groups in the layer control; `x = NULL`; no land-cover legend without `x`; a non-3-band layer errors by name; the shared domain is applied; the COG URL has 3 `bidx` and the shared rescale, is percent-encoded, and errors without `titiler_url`. - [ ] Live check: 2017 Jun–Jul and 2023 Aug–Sep on the packaged AOI over Esri/Google. Save the widget, view it in the browser, and run the cartography self-review on the screenshot. ## Phase 5: HLS spike → own issue; reconcile #79 diff --git a/tests/testthat/test-dft_map_interactive.R b/tests/testthat/test-dft_map_interactive.R index 6dd7312..84a460c 100644 --- a/tests/testthat/test-dft_map_interactive.R +++ b/tests/testthat/test-dft_map_interactive.R @@ -172,3 +172,118 @@ test_that("COG mode auto-wraps single unnamed URL", { titiler_url = "https://titiler.example.com") expect_s3_class(map, "leaflet") }) + +# -- RGB imagery (#79) --------------------------------------------------------- + +# A synthetic three-band "composite" on the packaged grid, in reflectance +make_rgb <- function(red = 0.1, green = 0.05, blue = 0.02) { + tmpl <- terra::rast(system.file("extdata", "example_2020.tif", package = "drift")) + tmpl <- terra::aggregate(tmpl, 10) + r <- c(terra::init(tmpl, red), terra::init(tmpl, green), terra::init(tmpl, blue)) + names(r) <- c("red", "green", "blue") + r +} + +test_that("rgb layers become overlay groups beneath the classified layers", { + classified <- load_classified_list() + rgb <- list("2017 Jun-Jul" = make_rgb(), "2023 Aug-Sep" = make_rgb(0.2)) + map <- dft_map_interactive(classified, rgb = rgb) + ctrl <- map$x$calls[[which(map_methods(map) == "addLayersControl")]]$args[[2]] + expect_equal(ctrl[1:2], names(rgb)) + # drawn first, so beneath: every rgb image call precedes the first classified one + img <- which(map_methods(map) == "addRasterImage") + groups <- vapply(map$x$calls[img], function(c) c$args[[4]], character(1)) + expect_equal(groups[1:2], names(rgb)) + # imagery starts hidden under classified layers + hidden <- unlist(lapply(which(map_methods(map) == "hideGroup"), + function(i) map$x$calls[[i]]$args)) + expect_true(all(names(rgb) %in% hidden)) +}) + +test_that("an imagery-only map works with x = NULL and has no land-cover legend", { + rgb <- list("2017 Jun-Jul" = make_rgb(), "2023 Aug-Sep" = make_rgb(0.2)) + map <- dft_map_interactive(rgb = rgb, aoi = load_aoi()) + expect_s3_class(map, "leaflet") + expect_false("addLegend" %in% map_methods(map)) + hidden <- unlist(lapply(which(map_methods(map) == "hideGroup"), + function(i) map$x$calls[[i]]$args)) + expect_false("2017 Jun-Jul" %in% hidden) + expect_true("2023 Aug-Sep" %in% hidden) + expect_error(dft_map_interactive(), "Supply") +}) + +test_that("band 1 draws as red: channel order is r, g, b = 1, 2, 3", { + # leafem defaults to r = 3, b = 1, which would paint this pure blue + rgb <- list(a = make_rgb(red = 0.3, green = 0, blue = 0), + b = make_rgb(red = 0, green = 0, blue = 0)) + cols <- NULL + testthat::local_mocked_bindings( + addRasterImage = function(map, x, colors, ...) { + cols <<- c(cols, list(colors(1))) + map + }, + .package = "leafem" # leafem calls addRasterImage through its own imports + ) + dft_map_interactive(rgb = rgb) + # reprojection to 3857 leaves NA corners, drawn transparent; every data pixel + # of the all-red composite must be pure red + drawn <- unique(toupper(cols[[1]][cols[[1]] != "#00000000"])) + expect_equal(drawn, "#FF0000FF") # rgb(alpha = 1) carries the alpha byte +}) + +test_that("one stretch serves every composite, so a darker year stays darker", { + # Per-image stretching would render both as the same full-range colour. + dom <- drift:::rgb_domain(list(make_rgb(0.1, 0.1, 0.1), make_rgb(0.3, 0.3, 0.3))) + expect_equal(dim(dom), c(2L, 3L)) + dark <- drift:::rgb_stretch(make_rgb(0.1, 0.1, 0.1), dom) + bright <- drift:::rgb_stretch(make_rgb(0.3, 0.3, 0.3), dom) + expect_lt(terra::global(dark, "max")[1, 1], terra::global(bright, "min")[1, 1]) + # per band: a band scaled 10x (NIR in false colour) does not flatten the others + dom2 <- drift:::rgb_domain(list(make_rgb(0.5, 0.05, 0.02), make_rgb(0.4, 0.04, 0.01))) + s <- drift:::rgb_stretch(make_rgb(0.4, 0.04, 0.01), dom2) + expect_equal(unname(unlist(terra::global(s, "max"))), c(0, 0, 0)) + s <- drift:::rgb_stretch(make_rgb(0.5, 0.05, 0.02), dom2) + expect_equal(unname(unlist(terra::global(s, "max"))), c(1, 1, 1)) +}) + +test_that("rgb input is validated by name and band count", { + classified <- load_classified_list() + two_band <- make_rgb()[[1:2]] + expect_error(dft_map_interactive(rgb = list(a = two_band)), "three bands.*a") + expect_error(dft_map_interactive(rgb = list(make_rgb())), "named") + expect_error(dft_map_interactive(classified, rgb = list("2017" = make_rgb())), + "share layer name") + expect_error(dft_map_interactive(rgb = list(a = make_rgb()), rgb_rescale = c(1, 0)), + "two increasing") + expect_error(dft_map_interactive(rgb = "https://x/y.tif", titiler_url = NULL), + "titiler URL") +}) + +test_that("COG rgb layers get a three-band titiler URL with the shared rescale", { + url <- drift:::build_titiler_rgb_url("https://ti.example", "https://b/c 1.tif", + c(0, 0.3)) + expect_match(url, "&bidx=1&bidx=2&bidx=3", fixed = TRUE) + expect_equal(lengths(regmatches(url, gregexpr("&rescale=0%2C0.3", url, fixed = TRUE))), 3L) + expect_match(url, "url=https%3A%2F%2Fb%2Fc%201.tif", fixed = TRUE) + map <- dft_map_interactive(rgb = c("2017 Jun-Jul" = "https://b/c.tif"), + titiler_url = "https://ti.example") + tiles <- map$x$calls[map_methods(map) == "addTiles"] + tmpl <- vapply(tiles, function(c) c$args[[1]], character(1)) + expect_true(any(grepl("bidx=3", tmpl, fixed = TRUE))) +}) + +test_that("classified layers are handed to leaflet in EPSG:3857", { + # project = FALSE means leaflet assumes Web Mercator pixels; a 4326 grid would + # be misregistered against the RGB imagery beneath it. + crs_seen <- character(0) + testthat::local_mocked_bindings( + addRasterImage = function(map, x, ...) { + crs_seen <<- c(crs_seen, terra::crs(x, describe = TRUE)$code) + map + }, + .package = "leaflet" + ) + dft_map_interactive(load_classified_list()) + expect_true(length(crs_seen) > 0) + expect_true(all(crs_seen == "3857")) +}) From 5d215c79f80f6945ea5c7105faefbfbfc68f11a4 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:36:56 -0700 Subject: [PATCH 11/29] Regenerate dft_stac_config docs for the green and blue roles (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- man/dft_stac_config.Rd | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/man/dft_stac_config.Rd b/man/dft_stac_config.Rd index 1e5d112..a753ad2 100644 --- a/man/dft_stac_config.Rd +++ b/man/dft_stac_config.Rd @@ -14,7 +14,8 @@ L2A surface reflectance, a cube source).} \value{ A list. Categorical sources have elements \code{stac_url}, \code{collection}, \code{asset}, \code{available_years}. Cube sources have \code{stac_url}, \code{collection}, -\code{cube = TRUE}, \code{roles} (a named list mapping \code{red}/\code{nir}/\code{swir16}/\code{mask} +\code{cube = TRUE}, \code{roles} (a named list mapping +\code{red}/\code{green}/\code{blue}/\code{nir}/\code{swir16}/\code{mask} to asset names), \code{mask_values} (integer mask classes to exclude), \code{scale}/\code{offset} (DN → reflectance affine transform), and \code{available_datetime} (an ISO 8601 interval string). The \code{cube} field is @@ -31,7 +32,7 @@ Sources are of two kinds. \strong{Categorical} sources (\code{"io-lulc"}, \code{"esa-worldcover"}) host single-band classified rasters and carry a flat \code{asset} name for \code{\link[=dft_stac_fetch]{dft_stac_fetch()}}. \strong{Cube} sources (\code{"sentinel-2-l2a"}) host multi-band reflectance imagery and instead carry a role-based band map -(\code{red}/\code{nir}/\code{swir16}/\code{mask}), mask values, and reflectance scale/offset for +(\code{red}/\code{green}/\code{blue}/\code{nir}/\code{swir16}/\code{mask}), mask values, and reflectance scale/offset for \code{\link[=dft_stac_cube]{dft_stac_cube()}}; they are marked with \code{cube = TRUE}. The role-based schema means a new reflectance source (e.g. Landsat C2 L2) drops in with no API change — only the role→asset map and scale/offset differ. From 2c911e6c22180ed3672c4b3bc135ef7d8fa1da12 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:39:58 -0700 Subject: [PATCH 12/29] Order composite layers by name, not position (#79) terra reads gdalcubes' multi-variable NetCDF with its variables in alphabetical order, so a true-colour request came back blue, green, red. The live e2e's layer-order guard caught it; renaming by position would have swapped red and blue silently. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_composite.R | 22 ++++++++++++---------- planning/active/findings.md | 20 ++++++++++++++++++++ tests/testthat/test-dft_stac_composite.R | 20 ++++++++++++-------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R index 6043ef4..2ebd975 100644 --- a/R/dft_stac_composite.R +++ b/R/dft_stac_composite.R @@ -214,7 +214,7 @@ dft_stac_composite <- function(aoi, offset = offset, offset_before = offset_before, pixel_fn = pixel_fn, tile_size = tile_size ) - composite_layers_check(stk, bands, label, w) + stk <- composite_layers_order(stk, bands, label, w) if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) cube_check_nonempty(stk, cfg$collection, w$datetime, cached = FALSE) stk <- composite_finish(stk, bands, w$t0) @@ -276,14 +276,15 @@ composite_offset_check <- function(is_pre, label, boundary) { } -#' Check a composite has one layer per band, in band order +#' Put a composite's layers in band order, by name, checking there is one per band #' -#' Two ways it can go wrong, both silent downstream: a window and `dt` that -#' disagree give more than one time step, and a multi-variable NetCDF read back -#' by terra in an order other than the bands' would put the wrong band in a -#' channel when the layers are renamed by position. +#' terra reads a multi-variable gdalcubes NetCDF with its variables in +#' ALPHABETICAL order (measured: `blue`, `green`, `red` for a true-colour +#' request), so renaming the layers by position would silently swap red and +#' blue. Layers are therefore selected by name. A window and `dt` that disagree +#' would give more than one time step, which is refused too. #' @noRd -composite_layers_check <- function(stk, bands, label, w) { +composite_layers_order <- function(stk, bands, label, w) { if (terra::nlyr(stk) != length(bands)) { cli::cli_abort(c( "The {label} composite has {terra::nlyr(stk)} layer{?s}; expected \\ @@ -292,13 +293,14 @@ composite_layers_check <- function(stk, bands, label, w) { a single time step." )) } - if (!all(startsWith(names(stk), bands))) { + idx <- match(bands, names(stk)) + if (anyNA(idx)) { cli::cli_abort(c( - "The {label} composite's layers are not in band order.", + "The {label} composite's layers do not match its bands.", "x" = "Layers: {.val {names(stk)}}; bands: {.val {bands}}." )) } - invisible(TRUE) + stk[[idx]] } #' Name a composite's layers by band role and stamp the window start diff --git a/planning/active/findings.md b/planning/active/findings.md index f4733a0..04a3c80 100644 --- a/planning/active/findings.md +++ b/planning/active/findings.md @@ -29,7 +29,27 @@ drift already fetches Sentinel-2 L2A cubes (`dft_stac_cube()`, with cloud maskin - **floodplains#93:** "for each sample point: a map zoomed to the point, the dated composites as switchable layers". The review runs over whole floodplains, not a reach, which is why this issue has to hold at floodplain scale. - The new_graphiti post `2026-01-08-stac-ortho-mosaics` is the starting point: rstac, bbox search, `eo:cloud_cover <= 20`, B04/B03/B02, `dt = P2M`, median, `leafem::addRasterRGB(quantiles = c(0.02, 0.98))`. It has no cloud mask and no offset handling, and stretches each layer on its own. +## HLS spike (Phase 5, probed live 2026-09-28) + +Probe scripts are `hls_probe.R`, `hls_read*.R` and `hls_query*.R` in the session scratchpad. The AOI is the packaged Neexdzii Kwa reach, July 2023. + +- **Search needs no auth.** `https://cmr.earthdata.nasa.gov/stac/LPCLOUD`, collections `HLSS30_2.0` (Sentinel-2, from 2015-11-28) and `HLSL30_2.0` (Landsat). The reach returned 11 S30 and 7 L30 items for July 2023, tile `T09UXA`. +- **CMR-STAC is not Planetary Computer, and drift's query shape fails on it in two ways:** + - `intersects` returns **HTTP 500** on POST, while `bbox` works on both GET and POST. An HLS source needs a bbox query plus the client-side AOI clip drift already does. + - The CQL2 `eo:cloud_cover <= 20` filter is **silently ignored**. It returned covers 3, 19, 21, 29, 41, 55, 69, 73, 75, 88 and 100. The cloud pre-filter has to be client-side. +- **Reading needs Earthdata Login.** + - Assets are under `data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/`. An unauthenticated `/vsicurl/` read returns 404, not 401, which is misleading. + - `earthdatalogin::edl_netrc()` with its **built-in default credentials no longer works**: `edl_set_token()` returns HTTP 401, and the netrc read comes back "not recognized as being in a supported file format", which is the login page. + - A real Earthdata Login account is required. Passing it via `edl_netrc(username, password)` or `EARTHDATA_USER`/`EARTHDATA_PASSWORD`, which sets `GDAL_HTTP_NETRC_FILE` and the cookie jar, was **not verified end to end**, because no account was available to the session. +- **Band names differ between the two collections.** Both have `B02`/`B03`/`B04` for blue/green/red. NIR is `B8A` in S30 (the narrow NIR HLS harmonizes on) and `B05` in L30. SWIR16 is `B11` in S30 and `B06` in L30; SWIR22 is `B12` and `B07`. A single "hls" source therefore needs per-collection role maps, or two sources. +- **Masking.** `Fmask` is a **bit** mask (1 cloud, 2 adjacent cloud, 3 cloud shadow, 4 snow/ice, 5 water; 6-7 aerosol), not SCL class values. `gdalcubes::image_mask()` has `bits =` (args: band, min, max, values, bits, invert), so it is expressible. The source config needs a mask-bits field beside `mask_values`. +- **Scale.** HLS v2.0 reflectance is scale 1e-4 with no offset, so there is no baseline split. + ## Errors Encountered | Error | Resolution | |-------|------------| +| Network e2e `expect_length(list.files(file.path(cache, src)))` fails on every run | Since #48 the cache lives under `v2/`. Use `cache_scheme_dir()` (fixed on #80 and here) | +| The Write tool turned `"\\u2013"` escapes into literal en dashes | Re-escape string literals in code; en dashes in comments are fine (the package already has them) | +| `format(c(0, 0.3))` gives `"0.0" "0.3"` (common width) | Format per element with `vapply` | +| Mocking `leaflet::addRasterImage` did not intercept leafem's call | leafem calls it through its own imports; mock with `.package = "leafem"` | diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R index 2b2c9e0..0b66a2e 100644 --- a/tests/testthat/test-dft_stac_composite.R +++ b/tests/testthat/test-dft_stac_composite.R @@ -144,16 +144,20 @@ test_that("a window straddling the offset boundary is refused, not covered", { expect_true(drift:::composite_offset_check(c(FALSE, FALSE), "2023 Jan", "2022-01-25")) }) -test_that("composite_layers_check catches a second time step and a band reorder", { - r <- terra::rast(nrows = 2, ncols = 2, nlyrs = 3, vals = 1) - names(r) <- c("red", "green", "blue") +test_that("composite_layers_order selects layers by name, not position", { + # terra reads gdalcubes' multi-variable NetCDF alphabetically (measured live: + # blue, green, red), so position would swap red and blue. + r <- terra::rast(nrows = 2, ncols = 2, nlyrs = 3, vals = 0) + terra::values(r) <- cbind(rep(3, 4), rep(2, 4), rep(1, 4)) + names(r) <- c("blue", "green", "red") w <- drift:::composite_window(2017, 6:7) - expect_true(drift:::composite_layers_check(r, c("red", "green", "blue"), "x", w)) - expect_error(drift:::composite_layers_check(c(r, r), c("red", "green", "blue"), "x", w), + out <- drift:::composite_layers_order(r, c("red", "green", "blue"), "x", w) + expect_equal(names(out), c("red", "green", "blue")) + expect_equal(unname(unlist(terra::global(out, "max"))), c(1, 2, 3)) + expect_error(drift:::composite_layers_order(c(r, r), c("red", "green", "blue"), "x", w), "expected 3") - names(r) <- c("blue", "green", "red") - expect_error(drift:::composite_layers_check(r, c("red", "green", "blue"), "x", w), - "not in band order") + expect_error(drift:::composite_layers_order(r, c("nir", "red", "green"), "x", w), + "do not match") }) test_that("an empty year is dropped with a warning and the others are kept", { From 92d0ec8c93542756de77801e8fcc00d3e07fee23 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:50:57 -0700 Subject: [PATCH 13/29] Record the live map check and the HLS spike (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- planning/active/progress.md | 8 ++++++++ planning/active/task_plan.md | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/planning/active/progress.md b/planning/active/progress.md index 53d3408..56f1d50 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -25,3 +25,11 @@ - wrap-year windows are documented. - Probed A2 live: a date-only STAC end bound drops the last day (22 of 23 items). The composite queries with `T00:00:00Z/T23:59:59Z`. The cube has the same issue, pre-existing, and goes to a follow-up issue because fixing it changes what existing cache keys hold. - Phases 3 and 4 are implemented. Mutation checks: band order, 4326 projection, key call site, offset split, per-image stretch and empty-year skip each turn a test red. The offline suite passes 1089, fails 0, skips 15. +- Live network e2e in the worktree: + - The cube passes 110, including the tiled-vs-untiled test, so the refactor's tiled path is verified live. + - The composite e2e first failed on the layer-order guard. terra reads gdalcubes' multi-variable NetCDF alphabetically (blue, green, red), which the guard caught. Fixed by selecting by name (cf230bc) and re-running. +- Live map check (Phase 4): + - Built 2017 Jun-Jul and 2023 Aug-Sep composites on the packaged AOI in 140.3 s; the second call was a cache hit. + - Served the widgets on localhost and inspected them in Chrome. Both layers render opaque and register with the Esri basemap. 2023 Aug-Sep reads darker and hazier than 2017 Jun-Jul under the shared stretch. Medians (red, green, blue): 2017 0.030, 0.050, 0.022; 2023 0.036, 0.050, 0.030. One scene seam is visible in 2017. + - Pre-existing and unrelated: the default "Light" basemap (CartoDB.Positron) now serves "API KEY REQUIRED" tiles. +- Phase 5 done: re-read #79 (unchanged since the gate), ran the HLS spike (findings.md), filed #82, filed #83 (pre-existing cube-query defects), and edited the #79 body (HLS -> #82, scale pivot, #80 dependency). diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md index 6302561..d27ebd8 100644 --- a/planning/active/task_plan.md +++ b/planning/active/task_plan.md @@ -48,19 +48,19 @@ Plan-gate decisions (2026-09-28): - COG mode uses an `rgb_rescale` argument, defaulting to a documented reflectance range. - [x] RGB groups are switchable overlays beneath the classified and transition layers. The first is visible when `x` is `NULL`, and they are hidden otherwise. `x` may be `NULL` when `rgb` is given. Centring and legends handle that case. - [x] Tests: synthetic 3-band rasters; groups in the layer control; `x = NULL`; no land-cover legend without `x`; a non-3-band layer errors by name; the shared domain is applied; the COG URL has 3 `bidx` and the shared rescale, is percent-encoded, and errors without `titiler_url`. -- [ ] Live check: 2017 Jun–Jul and 2023 Aug–Sep on the packaged AOI over Esri/Google. Save the widget, view it in the browser, and run the cartography self-review on the screenshot. +- [x] Live check: 2017 Jun–Jul and 2023 Aug–Sep on the packaged AOI over Esri/Google. Save the widget, view it in the browser, and run the cartography self-review on the screenshot. ## Phase 5: HLS spike → own issue; reconcile #79 -- [ ] Re-read #79, since scope may have been edited again. -- [ ] Probe live: +- [x] Re-read #79, since scope may have been edited again. +- [x] Probe live: - `earthdatalogin` netrc / GDAL config; - a CMR-STAC `LPCLOUD` `HLSS30.v2.0` / `HLSL30.v2.0` search via rstac; - one COG read through gdalcubes; - S30 vs L30 band names (NIR `B8A` vs `B05`, and so on); - the Fmask **bit** mask against SCL classes (`image_mask(bits=)`?). Record the results in `findings.md`. -- [ ] File the HLS issue with the worked auth path and source-config shape. Edit the #79 body: point item 4 at it, and add the scale pivot (chips and COG) and the #80 dependency. +- [x] File the HLS issue with the worked auth path and source-config shape. Edit the #79 body: point item 4 at it, and add the scale pivot (chips and COG) and the #80 dependency. ## Phase 6: scale test, docs, release From 4d333a0fc0d42545fe6247ca86f8dcf6657d67d4 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:55:35 -0700 Subject: [PATCH 14/29] Return the cached composite so every call gives the same values (#79) The first call returned the in-memory double stack while the cache holds Float32, so a cache hit differed in the eighth significant figure. The live e2e's same-values-on-second-call assertion caught it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_composite.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R index 2ebd975..d2d18d6 100644 --- a/R/dft_stac_composite.R +++ b/R/dft_stac_composite.R @@ -225,7 +225,10 @@ dft_stac_composite <- function(aoi, "OVERVIEW_RESAMPLING=AVERAGE") ) }) - stk + # Return what was cached, not the in-memory double stack: the file is + # Float32, so returning `stk` would make the first call differ from every + # later cache hit in the eighth significant figure. + composite_finish(terra::rast(cache_file), bands, w$t0) } # An empty or fully clouded year is a result, not a failure: warn and drop # it, keeping the years already built (and cached) before it. From f4e3477dcb413706772312c0c626545f07add0fb Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 07:59:47 -0700 Subject: [PATCH 15/29] Route the composite's gdalcubes check through check_gdalcubes (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_composite.R | 2 +- tests/testthat/test-dft_stac_composite.R | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R index d2d18d6..54e1b4b 100644 --- a/R/dft_stac_composite.R +++ b/R/dft_stac_composite.R @@ -141,7 +141,7 @@ dft_stac_composite <- function(aoi, cache_dir = NULL, force = FALSE, sign_fn = rstac::sign_planetary_computer()) { - rlang::check_installed("gdalcubes", reason = "to fetch STAC composites") + check_gdalcubes("to fetch STAC composites") cfg <- dft_stac_config(source) if (!isTRUE(cfg$cube)) { diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R index 0b66a2e..f76cb89 100644 --- a/tests/testthat/test-dft_stac_composite.R +++ b/tests/testthat/test-dft_stac_composite.R @@ -250,3 +250,9 @@ test_that("false colour keeps band order: NIR is brightest over summer vegetatio expect_equal(names(med), c("nir", "red", "green")) expect_gt(med[["nir"]], 2 * med[["red"]]) }) + +test_that("a missing gdalcubes names the GitHub install (#80)", { + testthat::local_mocked_bindings(gdalcubes_available = function() FALSE) + expect_error(dft_stac_composite(aoi_pkg(), years = 2023), + "appelmar/gdalcubes", fixed = TRUE) +}) From e100d63a930226a76bf79fe589bb25104de53704 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 08:10:33 -0700 Subject: [PATCH 16/29] Fix a stranded COG sidecar and the transition legend with x = NULL (#79) From code-check round 1: - Stamping time before the COG write made terra emit a .aux.json sidecar that cache_write_atomic() does not move, so every composite build left an orphan in the cache. Time is now stamped only on read. A new offline test drives the whole build and write path with the STAC query and gdalcubes assembly stubbed. - With x = NULL the transition legend sat inside the land-cover legend's gate and was dropped, and a map with no AOI and COG imagery was not centred. Both now follow the transition raster. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_map_interactive.R | 26 ++++++++++++---------- R/dft_stac_composite.R | 6 ++++- tests/testthat/test-dft_map_interactive.R | 13 +++++++++++ tests/testthat/test-dft_stac_composite.R | 27 +++++++++++++++++++++++ 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/R/dft_map_interactive.R b/R/dft_map_interactive.R index f9c36df..1042222 100644 --- a/R/dft_map_interactive.R +++ b/R/dft_map_interactive.R @@ -167,6 +167,8 @@ dft_map_interactive <- function(x = NULL, bbox <- raster_bbox_4326(x[[1]]) } else if (!is.null(rgb) && !rgb_cog) { bbox <- raster_bbox_4326(rgb[[1]]) + } else if (!is.null(transition)) { + bbox <- raster_bbox_4326(transition$raster) } else { bbox <- NULL } @@ -263,19 +265,19 @@ dft_map_interactive <- function(x = NULL, title = "Land Cover", opacity = 1 ) + } - # Legend — transitions - if (length(trans_groups) > 0) { - trans_colors <- attr(trans_result, "colors") - map <- leaflet::addLegend( - map, - position = legend_position, - colors = trans_colors, - labels = trans_groups, - title = "Transitions", - opacity = 1 - ) - } + # Legend — transitions (independent of `x`, which may be NULL) + if (!is.null(legend_position) && length(trans_groups) > 0) { + trans_colors <- attr(trans_result, "colors") + map <- leaflet::addLegend( + map, + position = legend_position, + colors = trans_colors, + labels = trans_groups, + title = "Transitions", + opacity = 1 + ) } # Layer control + fullscreen diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R index 54e1b4b..a7a0b31 100644 --- a/R/dft_stac_composite.R +++ b/R/dft_stac_composite.R @@ -217,7 +217,11 @@ dft_stac_composite <- function(aoi, stk <- composite_layers_order(stk, bands, label, w) if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) cube_check_nonempty(stk, cfg$collection, w$datetime, cached = FALSE) - stk <- composite_finish(stk, bands, w$t0) + # Names only, NOT time, before the write: a raster carrying a time makes + # terra's COG writer emit a `.aux.json` sidecar, which cache_write_atomic() + # does not move, so it would be stranded under the temp name on every build. + # The time is stamped on the re-read below, as it is on a cache hit. + names(stk) <- bands cache_write_atomic(cache_file, function(path) { terra::writeRaster( stk, path, filetype = "COG", datatype = "FLT4S", overwrite = TRUE, diff --git a/tests/testthat/test-dft_map_interactive.R b/tests/testthat/test-dft_map_interactive.R index 84a460c..55d0b2b 100644 --- a/tests/testthat/test-dft_map_interactive.R +++ b/tests/testthat/test-dft_map_interactive.R @@ -287,3 +287,16 @@ test_that("classified layers are handed to leaflet in EPSG:3857", { expect_true(length(crs_seen) > 0) expect_true(all(crs_seen == "3857")) }) + +test_that("transitions keep their legend and centre the map when x is NULL", { + classified <- load_classified_list() + trans <- dft_rast_transition(classified, from = "2017", to = "2023", + from_class = "Trees") + map <- dft_map_interactive(transition = trans, + rgb = c(img = "https://b/c.tif"), + titiler_url = "https://ti.example") + legends <- map$x$calls[map_methods(map) == "addLegend"] + titles <- vapply(legends, function(c) c$args[[1]]$title, character(1)) + expect_equal(titles, "Transitions") + expect_false(is.null(map$x$setView)) # centred from the transition raster +}) diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R index f76cb89..a82060d 100644 --- a/tests/testthat/test-dft_stac_composite.R +++ b/tests/testthat/test-dft_stac_composite.R @@ -256,3 +256,30 @@ test_that("a missing gdalcubes names the GitHub install (#80)", { expect_error(dft_stac_composite(aoi_pkg(), years = 2023), "appelmar/gdalcubes", fixed = TRUE) }) + +test_that("a built composite leaves only its COG in the cache, no sidecar", { + # Drives the real build and write path offline: the STAC query and the + # gdalcubes assembly are stubbed with a synthetic three-band stack. A time + # stamped before a COG write makes terra emit a .aux.json that + # cache_write_atomic() does not move, which strands under the temp name. + skip_if_not_installed("gdalcubes") + aoi <- aoi_pkg() + aoi_t <- sf::st_transform(aoi, 32609) + testthat::local_mocked_bindings( + stac_cube_items = function(...) list(features = list(), is_pre = logical(0)), + stac_cube_assemble = function(...) { + r <- terra::rast(terra::ext(aoi_t), resolution = 50, crs = "EPSG:32609", + nlyrs = 3, vals = 0.05) + names(r) <- c("blue", "green", "red") # alphabetical, as terra reads it + r + } + ) + cache <- tempfile("drift_composite_build_") + out <- suppressMessages(dft_stac_composite(aoi, years = 2023, cache_dir = cache)) + dir <- drift:::cache_scheme_dir(cache, "sentinel-2-l2a") + files <- list.files(dir, all.files = TRUE, no.. = TRUE) + expect_length(files, 1L) + expect_match(files, "^composite_[0-9a-f]{16}\\.tif$") + expect_equal(names(out[[1]]), c("red", "green", "blue")) + expect_equal(terra::time(out[[1]])[1], as.Date("2023-06-01")) +}) From 7266fe77733219e9ff0a51646d8d0e575af46ccd Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 08:15:21 -0700 Subject: [PATCH 17/29] Strip time before the COG write, and move .aux.json sidecars (#79) Code-check round 2: the round-1 sidecar fix only stopped the code stamping a time, but the stack already carries one read from the gdalcubes NetCDF, so the orphan persisted on the real path. The test fixture had no time and could not reach it. - The composite strips time before writing and re-stamps on read. - cache_write_atomic() now moves a terra .aux.json the same way it moves a GDAL .aux.xml. A COG gets one for time, units, varnames, longnames, metags or scoff, so a later metadata addition cannot strand one again. - Both fixtures now carry a time. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_composite.R | 10 ++++---- R/dft_stac_fetch.R | 30 ++++++++++++++---------- tests/testthat/test-dft_stac_composite.R | 3 +++ tests/testthat/test-dft_stac_cube.R | 20 ++++++++++++++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/R/dft_stac_composite.R b/R/dft_stac_composite.R index a7a0b31..3fed511 100644 --- a/R/dft_stac_composite.R +++ b/R/dft_stac_composite.R @@ -217,11 +217,13 @@ dft_stac_composite <- function(aoi, stk <- composite_layers_order(stk, bands, label, w) if (isTRUE(clip)) stk <- stac_cube_clip(stk, aoi_target) cube_check_nonempty(stk, cfg$collection, w$datetime, cached = FALSE) - # Names only, NOT time, before the write: a raster carrying a time makes - # terra's COG writer emit a `.aux.json` sidecar, which cache_write_atomic() - # does not move, so it would be stranded under the temp name on every build. - # The time is stamped on the re-read below, as it is on a cache hit. + # Names, and NO time, before the write. The stack arrives carrying a time + # read from the gdalcubes NetCDF, and a raster with a time makes terra's + # COG writer emit a `.aux.json` sidecar: an extra file beside a COG that is + # meant to be publishable on its own. The time is stamped on the re-read + # below, as it is on a cache hit. names(stk) <- bands + terra::time(stk) <- NULL cache_write_atomic(cache_file, function(path) { terra::writeRaster( stk, path, filetype = "COG", datatype = "FLT4S", overwrite = TRUE, diff --git a/R/dft_stac_fetch.R b/R/dft_stac_fetch.R index 10584ee..bf4147b 100644 --- a/R/dft_stac_fetch.R +++ b/R/dft_stac_fetch.R @@ -678,12 +678,15 @@ cache_write_atomic <- function(path, write_fn) { basename(tempfile("")), tools::file_ext(path)) ) - # GDAL's PAM sidecar rides along with whatever the writer produced. Measured: - # terra::writeRaster() to `.nc` emits one, to `.tif` does not. Renaming only - # the raster would strand it under the dead temp name, leaving cache litter - # that dft_cache_info() then counts. - tmp_pam <- paste0(tmp, ".aux.xml") - on.exit(unlink(c(tmp, tmp_pam)), add = TRUE) + # Sidecars ride along with whatever the writer produced. Measured: + # terra::writeRaster() to `.nc` emits a GDAL PAM `.aux.xml`, to a plain `.tif` + # nothing, and to a COG a terra `.aux.json` whenever the raster carries a time, + # units, varnames, longnames, metags or scoff (#79, terra 1.9.50). Renaming + # only the raster would strand them under the dead temp name, leaving cache + # litter that dft_cache_info() then counts. + sidecars <- c(".aux.xml", ".aux.json") + tmp_side <- paste0(tmp, sidecars) + on.exit(unlink(c(tmp, tmp_side)), add = TRUE) write_fn(tmp) @@ -706,12 +709,15 @@ cache_write_atomic <- function(path, write_fn) { )) } - # Sidecar first, raster second, so the canonical name never exists without it. - if (file.exists(tmp_pam) && - !isTRUE(file.rename(tmp_pam, paste0(path, ".aux.xml")))) { - cli::cli_abort( - "The sidecar for {.file {basename(path)}} could not be moved into place." - ) + # Sidecars first, raster second, so the canonical name never exists without + # them. + for (i in seq_along(sidecars)) { + if (file.exists(tmp_side[i]) && + !isTRUE(file.rename(tmp_side[i], paste0(path, sidecars[i])))) { + cli::cli_abort( + "The sidecar for {.file {basename(path)}} could not be moved into place." + ) + } } if (!isTRUE(file.rename(tmp, path))) { cli::cli_abort(c( diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R index a82060d..93ab833 100644 --- a/tests/testthat/test-dft_stac_composite.R +++ b/tests/testthat/test-dft_stac_composite.R @@ -271,6 +271,9 @@ test_that("a built composite leaves only its COG in the cache, no sidecar", { r <- terra::rast(terra::ext(aoi_t), resolution = 50, crs = "EPSG:32609", nlyrs = 3, vals = 0.05) names(r) <- c("blue", "green", "red") # alphabetical, as terra reads it + # and with a time, as terra reads the gdalcubes NetCDF: a fixture without + # one cannot reach the sidecar (code-check round 2) + terra::time(r) <- rep(as.Date("1970-01-01"), 3) r } ) diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index babdf46..dc110dc 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -691,3 +691,23 @@ test_that("dft_stac_cube computes its cache key from its own arguments (call sit cache_dir = cache)) expect_equal(terra::nlyr(out), 2) }) + +test_that("cache_write_atomic moves a terra .aux.json sidecar with the raster", { + # A COG written from a raster that carries a time (or units, metags...) gets a + # terra .aux.json; only .aux.xml used to be moved, so the json was stranded + # under the temp name on every write (#79). + dir <- tempfile("drift_sidecar_") + dir.create(dir) + r <- terra::rast(nrows = 4, ncols = 4, nlyrs = 2, vals = 1, + crs = "EPSG:32609", extent = c(0, 40, 0, 40)) + terra::time(r) <- as.Date(c("2023-06-01", "2023-07-01")) + path <- file.path(dir, "entry.tif") + drift:::cache_write_atomic(path, function(out) { + terra::writeRaster(r, out, filetype = "COG") + }) + files <- sort(list.files(dir, all.files = TRUE, no.. = TRUE)) + expect_false(any(grepl("tmp", files))) + expect_true("entry.tif" %in% files) + # the sidecar, when the driver writes one, sits under the canonical name + expect_true(all(files %in% c("entry.tif", "entry.tif.aux.json", "entry.tif.aux.xml"))) +}) From 467ad699e10b1e6a3be655464ad2f28a3bfdca59 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 08:24:30 -0700 Subject: [PATCH 18/29] Pin the composite's layer reorder and sidecar-free cache in tests (#79) Code-check round 3 enumerated every raster write in R/ and every test stub against the object it stands in for. There was no production defect, but two tests could not reach what they claimed to cover: - the offline build stub gave every band the same value, so dropping composite_layers_order() stayed green (verified with that mutant); - the network test listed only *.tif, so a sidecar beside the COG was invisible to it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- planning/active/findings.md | 8 ++ planning/active/review-79-round1.md | 43 +++++++++++ planning/active/review-79-round2.md | 37 +++++++++ planning/active/review-79-round3.md | 97 ++++++++++++++++++++++++ tests/testthat/test-dft_stac_composite.R | 17 ++++- 5 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 planning/active/review-79-round1.md create mode 100644 planning/active/review-79-round2.md create mode 100644 planning/active/review-79-round3.md diff --git a/planning/active/findings.md b/planning/active/findings.md index 04a3c80..ec24f63 100644 --- a/planning/active/findings.md +++ b/planning/active/findings.md @@ -45,6 +45,14 @@ Probe scripts are `hls_probe.R`, `hls_read*.R` and `hls_query*.R` in the session - **Masking.** `Fmask` is a **bit** mask (1 cloud, 2 adjacent cloud, 3 cloud shadow, 4 snow/ice, 5 water; 6-7 aerosol), not SCL class values. `gdalcubes::image_mask()` has `bits =` (args: band, min, max, values, bits, invert), so it is expressible. The source config needs a mask-bits field beside `mask_values`. - **Scale.** HLS v2.0 reflectance is scale 1e-4 with no offset, so there is no baseline split. +## Per-chip cost (measured 2026-09-28, BULK, 2023 Jul-Aug) + +- The chips benchmark is about 52 s per chip. Three chips were split by stage while that benchmark was also streaming, so the times are contended: + - `stac_cube_items()` takes 1.4-2.5 s and returns 16 items; + - the full composite takes 39-95 s. +- The query is not the cost; the COG reads are: 16 scenes × 4 assets (B04/B03/B02/SCL), about 64 remote opens and block reads for a 600 m chip. Sharing one STAC query across chips would save about 2 s of about 50. +- What would help is running chips **concurrently**. That means PSOCK workers, not forks: `mclapply` over a remote raster aborts every fork on macOS (`code-check-spatial.md`). Left as a follow-up; a floodplains#93 pilot of ~180 points × 2-3 windows is several hours serially. + ## Errors Encountered | Error | Resolution | diff --git a/planning/active/review-79-round1.md b/planning/active/review-79-round1.md new file mode 100644 index 0000000..7259ce3 --- /dev/null +++ b/planning/active/review-79-round1.md @@ -0,0 +1,43 @@ +# Review round 1: drift #79 (dft_stac_composite, rgb in dft_map_interactive, cube refactor) + +## Findings + +- **[severity: bug]** R/dft_stac_composite.R:220-227 (with R/dft_stac_fetch.R:685-686): every composite cache write leaves an orphaned `.aux.json` sidecar in the cache. + `composite_finish()` stamps `terra::time()` on `stk` **before** it is written. When a raster has time set, terra's COG writer puts it in a `.aux.json` sidecar. The plain GTiff path used by `dft_stac_cube()` does not do this, which is why the existing `cache_write_atomic()` comment says `.tif` produces no sidecar. `cache_write_atomic()` only handles `.aux.xml`: it unlinks that file on exit and renames it with the raster. The `.aux.json` is therefore left under the dead temp name. + Reproduced on this machine (terra 1.9.50) with the package's own `composite_finish()` and `cache_write_atomic()`, using the exact `writeRaster(filetype = "COG", ...)` call from the diff: + ``` + list.files(cache_dir) + #> "composite_abc.tif" "composite_abc.tmp26686-683e3de428c5.tif.aux.json" + ``` + The same write with no time set produces no sidecar, so the time stamp is the cause. + Effects: + - One orphan per cache miss, which for chips means one per point per year. + - `dft_cache_info()` counts them. This is the same litter class that `cache_write_atomic()`'s own comment says the sidecar handling exists to prevent. + - The published COG does not carry the time. This is harmless only because `composite_finish()` re-stamps it on read. + + Two possible fixes: + - Write `stk` before stamping the time, i.e. call `composite_finish()` only on the re-read `terra::rast(cache_file)`, which the code already does. + - Or have `cache_write_atomic()` treat `.aux.json` the same as `.aux.xml`: unlink it on exit and move it into place. + + The first also keeps the cached COG free of a sidecar that titiler/S3 publishing would not carry. + +- **[severity: bug, minor]** R/dft_map_interactive.R:248-279: with `x = NULL`, the transition legend is dropped along with the land-cover legend. + The new `!is.null(x)` gate wraps the whole legend block, and the transition legend (`# Legend — transitions`) is nested inside it. `x = NULL` is newly allowed, so `dft_map_interactive(transition = tr, rgb = tc)` is now reachable. That call draws every transition overlay in its to-class colour with no legend. Verified: the call list contains `addRasterImage` × N and no `addLegend`. + Also on this path: with `aoi = NULL` and `rgb` given as COG URLs, `bbox` is `NULL` even though `transition$raster` has an extent, so no `setView` is issued. + Only the land-cover half should depend on `x`. + +## Checked and found sound (not findings) + +- **`stac_cube_session()` restore ordering.** `cube_parallel_check()` runs before any mutation. Nothing can error between the return and the `on.exit()` registration in either caller (it is a single assignment). +- **`tryCatch(build(), drift_no_items=, drift_empty_cube=)`.** Neither class can come from the cache-read path, which sits outside `build()`. `drift_composite_offset_split` is deliberately not caught. `cli_abort(class = ..., c(...))` puts the named `class` into `...`, so the class is attached (confirmed by the passing skip test). +- **`rgb_stretch`.** A 3-layer SpatRaster minus or divided by a length-3 vector recycles per layer (measured: `(0.1-0)/1, (0.2-0)/0.5, (0.3-0.1)/0.1` gives `0.1, 0.4, 2.0`). `clamp(values = TRUE)` saturates to 0/1 and leaves NA as NA. +- **`leafem::addRasterRGB(quantiles = NULL, domain = c(0,1))`** (leafem 0.2.5 source read). The `quantiles` branch is skipped and `rscl(from = c(0,1))` is the identity, so the terra stretch is what is drawn. +- **Classified layers projected to 3857 with `method = "near"`.** Factor levels and coltab survive (checked on the packaged classified raster: same values 1,2,4,5,7,9,11, coltab intact). leaflet 2.2.3 `addRasterImage(project = FALSE)` computes bounds by projecting the extent from the raster's CRS through 3857, so 3857 input is correct. +- **Z-order after toggling.** Re-added GridLayers append to the DOM, but `L.Control.Layers(autoZIndex = TRUE)` assigns z-indices in `overlay_groups` order. `rgb` comes first there, so it stays beneath `x` after toggling. +- **Tiled composite.** `terra::merge(sprc(...))` keeps layer names (measured), so `composite_layers_order()`'s by-name select also works on the tiled path. +- **`composite_window`.** Month-end arithmetic (Feb and leap year, Dec) is correct. `dt = P{last-first+1}M` always gives one step, and `composite_layers_order()` refuses anything else. +- **Cache key.** Uses `cache_key_hash` type tags. The WKB list goes through the `is.list` branch. The key is distinct from the cube key via the leading `"composite"` tag. +- **cli pluralisation.** Every `{?s}` in the new messages renders correctly at n = 1 and n = 2 (probed). +- **Offline tests.** `test-dft_map_interactive.R` and `test-dft_stac_composite.R` run green from a copy (`NOT_CRAN=true`); only the two network tests skip. + +/private/tmp/claude-501/-Users-airvine-Projects-repo-drift/15f0d12b-1b2b-4965-8f08-91fb4f6a71a8/scratchpad/review-79-round1.md diff --git a/planning/active/review-79-round2.md b/planning/active/review-79-round2.md new file mode 100644 index 0000000..61eff17 --- /dev/null +++ b/planning/active/review-79-round2.md @@ -0,0 +1,37 @@ +# Review #79 — round 2 (fixes first) + +## Findings + +- **[severity: bug]** R/dft_stac_composite.R:220-231 — The round-1 sidecar fix does not fix it on the real path. The comment says "Names only, NOT time, before the write", but the stack never lacked a time to begin with: `stac_cube_assemble()` returns `terra::rast()` of the gdalcubes NetCDF, and terra reads that file's time dimension (measured: `has.time() = TRUE`, step `yearmonths`, value shown as `1970-01-00`). `composite_layers_order()` (`stk[[idx]]`), `stac_cube_clip()` (`mask`) and `names<-` all keep that time, so the COG write still emits `.aux.json`, and `cache_write_atomic()` still strands it. + + Measured on terra 1.9.50 / GDAL 3.13.0 / gdalcubes 0.7.5. I built a real gdalcubes cube from three local synthetic scenes (B02/B03/B04/SCL), then `apply_pixel(names = c("red","green","blue"))` and `write_ncdf`. I mocked only `stac_cube_items` and `stac_cube_assemble` (to return `terra::rast()`) and ran the current `dft_stac_composite()` from a copy of the tree: + - clip = FALSE: cache dir held `composite_.tif` and `composite_.tmp-.tif.aux.json`, and `dft_cache_info()$n_files` was 2. + - clip = TRUE: the same orphan. + - The orphan was still there after a second (cache-hit) call. A new one appears on every build or `force = TRUE` rebuild. + + With `terra::time(stk) <- NULL` added after `names(stk) <- bands`, both arms leave only the `.tif` (measured). The returned object still gets its time from `composite_finish()` on the re-read. + + **Why the new test stays green:** its `stac_cube_assemble` stub builds the raster with `terra::rast(ext, nlyrs = 3, vals = 0.05)`, which has no time. It therefore cannot reach the failure mode, which is inherited from the NetCDF reader and not from the code's own stamp. The stub would catch the old explicit `composite_finish()` stamp, but not this. To reach it, the fixture must carry a time, for example `terra::time(r) <- rep(as.Date("2023-06-01"), 3)`, or be a real gdalcubes NetCDF. + + **Sidecar triggers under the COG driver** (single-variable probes on a 3-band FLT4S raster, same `gdal=` options): `time`, `units`, `varnames` (a non-default value), `longnames`, `metags` and `scoff` each produce `.aux.json`. Plain `names` does not. On the real gdalcubes stack only `time` is set. Its varnames equal the layer names, units and longnames are empty, scoff is 1/0 and NAflag is NaN, and with time stripped it wrote no sidecar. So `time` is the only live trigger today. + + A more robust option than stripping one property is for `cache_write_atomic()` to move or unlink `paste0(tmp, ".aux.json")` as it already does for `.aux.xml`. Then a later `units`/`metags` addition cannot bring the orphan back. + + **Cube path (question 1b): not affected.** `dft_stac_cube()` writes a plain GTiff (no `filetype`), and GTiff with a monthly time and repeated names wrote no `.aux.json` and no `.aux.xml` (measured). The raw gdalcubes stack written as GTiff also wrote no sidecar. + +## Checked, sound + +- **dft_map_interactive x = NULL paths:** + - `cog_mode <- is.character(NULL)` is FALSE, and the else branch's `inherits(NULL, "SpatRaster")` is FALSE, so `x` stays NULL. + - `intersect(names(rgb), NULL)` is empty. + - The classified-layer loop is skipped by the explicit `is.null(x)` arm. + - `overlay_groups` / `hidden`: `setdiff(NULL, NULL)` is empty. The first rgb layer is shown, and the transitions are visible. + - `class_table` is loaded unconditionally and used only by `add_transition_layers()` when `x` is NULL, which is harmless. + - The transition legend reads `trans_result` only when `length(trans_groups) > 0`, which implies it was assigned. + - The bbox falls back to `transition$raster` through the empty-template projection. + - `x = NULL`, `rgb = NULL` with only `transition` aborts with the "Supply x or rgb" message. That is consistent with the documented contract (`x` may be NULL "when rgb is supplied"). +- **The new `green`/`blue` roles** do not change any existing cube cache key. The cube key hashes only the index's `roles_needed` assets. `index_resolve_expr()` substitutes longest-first with `\b` boundaries, so `green` and `red` cannot clobber each other. + +## Evidence + +Scratch copy and scripts are in `scratchpad/r2/`: `nc.R` builds the gdalcubes NetCDF, `w.R` holds the per-property sidecar probes, and `e2e.R` drives the current composite build with the real NetCDF stack. The worktree `/Users/airvine/Projects/repo/drift-79` was not modified. diff --git a/planning/active/review-79-round3.md b/planning/active/review-79-round3.md new file mode 100644 index 0000000..8e4391e --- /dev/null +++ b/planning/active/review-79-round3.md @@ -0,0 +1,97 @@ +# Review #79, round 3 (mechanism + enumeration) + +Tree reviewed: `git archive HEAD` of `/Users/airvine/Projects/repo/drift-79` at 7266fe7. It was copied to `scratchpad/r3/tree`, and the worktree was not modified. +Stack: terra 1.9.50, GDAL 3.13.0, gdalcubes 0.7.5. + +## Mechanism + +R1 and R2 share one mechanism, the second framing: **the fixture was built from the properties the author set, not from what the upstream object carries**. + +- The first framing (a writer serialises raster properties into a sidecar the publisher does not know about) describes the *symptom* in R1. It is now closed by enumeration: + - Every publishing writer was measured on real gdalcubes output. None emits a sidecar (table A). + - `cache_write_atomic()` handles both sidecar kinds. +- What turned R1 into R2 was the second framing. The R1 fix was verified against a `stac_cube_assemble` stub with no time. The failure lived in a property the real NetCDF reader supplies and the code never sets. + +Round 3 finds a third instance of that same framing, below. + +The discriminating instrument this round was to stop stubbing `stac_cube_assemble` altogether: +- `scratchpad/r3/col.R` builds a local 5-band gdalcubes image collection (B02/B03/B04/B08/SCL, distinct per-band DN levels). +- `scratchpad/r3/e2e3.R` mocks only `drift:::stac_cube_items` and `gdalcubes::stac_image_collection`. + +So the real `stac_cube_assemble()` runs, along with `write_ncdf`, `terra::rast` of the multi-variable NetCDF, `mosaic_stacks`, `stac_cube_clip` and the COG publish, for: +- `dft_stac_composite()`: clip x tile, 4 arms, plus false colour. +- `dft_stac_cube()`: untiled and tiled. + +## Findings + +- **[test gap: medium]** tests/testthat/test-dft_stac_composite.R:260-289. The offline build test cannot detect the layer-order defect it models. Its `stac_cube_assemble` stub names the layers `blue, green, red` ("alphabetical, as terra reads it"), but every layer holds `vals = 0.05` (line 272). The name assertion `names(out[[1]]) == c("red","green","blue")` holds whatever the order, because `composite_finish()` renames unconditionally. + - **Mutation, measured.** Deleting `stk <- composite_layers_order(stk, bands, label, w)` (R/dft_stac_composite.R:217) leaves the whole offline file green (`NOT_CRAN=true`, 0 failures; only the two network tests skip). + - **Same mutant on the real NetCDF path.** It returns means `red 0.02, green 0.05, blue 0.04`, i.e. red and blue are swapped. The unmutated tree gives `0.04, 0.05, 0.02`, and the source DN levels make that correct. + - **What still guards it.** The `composite_layers_order` unit test covers the helper, not that the build calls it. Only the network false-colour test covers the call, and it is opt-in. + - **Fix (verified).** Give the stub distinct per-band values in alphabetical order, e.g. `cbind(0.02, 0.05, 0.1)` for `blue, green, red`, and assert `unname(unlist(terra::global(out[[1]], "max"))) == c(0.1, 0.05, 0.02)`. That kills the mutant (actual `0.02 0.05 0.1`) and passes on the unmutated tree. + - **Stronger alternative.** Drop the `stac_cube_assemble` stub for a local-collection fixture like `r3/col.R`, which runs in seconds. That closes the whole stub boundary that produced R2 and this finding. + +- **[test gap: low]** tests/testthat/test-dft_stac_composite.R:220-222. The network test lists the cache with `pattern = "^composite_.*\\.tif$"`, which cannot match a sidecar (`...tif.aux.json`). It is the only composite test that runs on a real gdalcubes object, and it ran green over the R1 defect: + - `scratchpad/test79_composite_network2.log` is 07:55, `[ FAIL 0 | ... | PASS 64 ]`. + - That run was on code that stamped the time before the COG write, which R1 then reproduced as an orphan. + - The `expect_length(f, 1)` therefore passes whatever litter sits beside the COG. + - Use `list.files(dir, all.files = TRUE, no.. = TRUE)` and expect exactly the one `.tif`, as the offline test now does. + - The two cube network tests (test-dft_stac_cube.R:406, :445) use the same filtered listing. That is pre-existing, and the cube writer emits no sidecar today (table A). + +No production-code defect found this round. + +## Enumeration A: every raster/file write in R/ (grep writeRaster|write_ncdf|writeVector|st_write|file.copy|file.rename|filename=) + +No `writeVector`, `st_write` or `file.copy` calls exist in R/. The only `file.rename` calls are the ones inside `cache_write_atomic()`. + +| # | site | driver | properties the object carries at write (measured on the real path) | sidecar emitted (measured) | publish path | +|---|---|---|---|---|---| +| 1 | dft_stac_composite.R:227-233 | COG, FLT4S, via `cache_write_atomic` | Names are the bands. Time is stripped at :226. Untiled: varnames equal the band names. Tiled `merge`: varnames `""`. Units and longnames empty, scoff 1/0, NAflag NaN (`NoData Value=nan` declared, `LAYOUT=COG`). | **None**, in all 4 arms (clip F/T x tile NULL/500) and in false colour. Nothing new after a cache hit, and hit values are identical to miss values. | Moves `.aux.xml` and `.aux.json`, and unlinks both on exit. | +| 2 | dft_stac_cube.R:291-293 | GTiff (default FLT4S), via `cache_write_atomic` | Time is `month_times`, names are `rep(index)`, and varnames come from the NetCDF. | **None**, untiled or tiled. A GTiff probe with time, units, longnames, varnames and metags all set also wrote none. | Same as #1. | +| 3 | dft_stac_fetch.R:213-216 (`fetch_extent_to`, :618) | gdalcubes `write_ncdf` `.nc`, via `cache_write_atomic` | Time is yearly. Varnames are the asset names. | **None**: not after publish, not after `cache_hit_ok()`, and not after `mask(rast())` plus `global()`. | Same as #1. | +| 4 | dft_stac_fetch.R:227 (`mosaic_tiles`, :635) | GTiff `.tif`, via `cache_write_atomic` | `merge` of the tile NetCDFs: `has.time` TRUE, varnames, units and longnames empty. | **None** | Same as #1. | +| 5 | dft_stac_cube.R:497 (`build_stack`) | gdalcubes `write_ncdf` to `tempfile(".nc")` | A float64, uncompressed NetCDF. | None | Not published and never unlinked; lives in the session tempdir (see Notes). | +| 6 | dft_rast_break_class.R:194/200/222/237/248/255; dft_rast_break_category.R:131/140/146; dft_transition_artifact.R:310-319 | GTiff `tempfile(".tif")` intermediates, plus a user `filename` | Not changed by this diff. | GTiff emits no `.aux.json` (probe in #2). The `.aux.xml` RAT sidecar is unlinked alongside. | n/a (not a cache) | + +**Canonical-stale-sidecar check.** `cache_write_atomic()` never removes a sidecar already sitting at the canonical name when the new write produces none. That is not reachable today: +- No writer above emits one. +- The R1-era orphans were always under the temp name, never the canonical one (checked in e100d63). + +The only producer would be an outside reader, for example QGIS or `gdalinfo -stats` writing PAM stats. drift restamps names and time on read and never uses `minmax()` on a cache file, so a stale sidecar there changes no drift output. Not flagged. + +## Enumeration B: test stubs and fixtures standing in for stac_cube_assemble / gdalcubes / rast(NetCDF) / STAC items + +| test (file:line) | stands in for | omits vs real | code under test that depends on the omission? | +|---|---|---|---| +| composite "built composite leaves only its COG" (composite:260) | `stac_cube_items` (empty `features`), `stac_cube_assemble` (synthetic 3-layer) | **Distinct per-band values** (all 0.05). Multi-source structure (one source per NetCDF variable). The `yearmonths` time step (uses Date days). NA cells. varnames. | **Yes: band order.** Finding 1, mutant measured green. Time step: no, because `time<- NULL` strips either kind. NA and varnames: no (real path measured clean). | +| composite "empty year dropped" (composite:163) | `stac_cube_items` aborting `drift_no_items`; `stac_composite_cache_key`; a seeded GTiff (not COG) with no time and names `lyr.1..3` | COG layout, time, band names | No. The hit path (`stac_cube_cache_read`, then `composite_finish`) renames and stamps unconditionally. The call-site argument order was checked by reading: it matches the signature positionally. | +| composite "refuses a categorical source" (composite:108) | `rstac::stac` stopping | none needed | No (validation only). | +| composite `composite_layers_order` unit (composite:149) | `terra::rast` of the NetCDF (alphabetical names, distinct values) | Multi-source structure | No. It is a correct helper-level test, but it does not pin the call site (finding 1). | +| composite network x2 (composite:201, :239) | real | nothing | **Its listing filter hides sidecars** (finding 2). | +| cube `cube_seed_cache` / "healthy cache served" / "call site pinned" (cube:497, :590, :666) | cached GTiff with no time and distinct values; `rstac::stac` stop; `stac_cube_cache_key` mock | Time, varnames | No. The hit path sets time from `month_times` and names come from the file. The call-site test does not mock the key. | +| cube "publishes atomically" (cube:508) | `terra::writeRaster` writing 2 KB and dying | Sidecar emission mid-write | No. `on.exit` unlinks the tmp file and both tmp sidecars regardless. | +| cube "moves a terra .aux.json" (cube:696) | COG with a time | none | Reaches the failure on terra 1.9.50: the canonical `entry.tif.aux.json` is produced and moved (measured). It carries no premise assertion that the sidecar exists, so on a terra that emits none it would pass vacuously. Not flagged: it reaches the failure on the pinned stack. | +| cube `mosaic_stacks` / offset-cover / tile-gap tests (cube:270-360) | in-memory synthetic stacks | NetCDF-backed sources, time | No. `merge` is positional per layer, and real tiled arms were measured to give correct per-band means. | +| map `make_rgb()` (map_interactive) | `dft_stac_composite()` output | Time, NA cells, Float32 | No. A real-path composite (time set, 1,572 NA cells from clip) drives `dft_map_interactive(rgb = …)` to a leaflet widget with `addRasterImage`. `rgb_domain` filters with `is.finite`. | +| index `eval_expr` (index_expr) | tinyexpr in gdalcubes | Evaluator differences | No. The real gdalcubes NDWI on the local collection gave -0.678, against a hand value of (0.05-0.26)/0.31 = -0.677. | +| fetch STAC paging stubs (fetch:268-530) | `rstac` `get_request` / `items_fetch` / `items_sign` | Not touched by this diff | n/a | + +## Notes (not findings) + +- **`build_stack` tempfiles (pre-existing, moved code).** The per-build NetCDF is float64 and uncompressed (240,364 bytes for 100x100x3), and it is never unlinked until the session ends. + - A floodplain-wide composite over the BULK bbox (~14,600 x 11,500 cells) is on the order of 4 GB of tempdir per year, before tile culling. + - `dft_stac_cube()` has always done the same. + - Worth watching in the running `floodplain` benchmark stage. Not a correctness defect. +- **Cube miss and hit differ (pre-existing, not in this diff).** `dft_stac_cube()` still returns the in-memory stack on a miss. Miss vs hit values differ by at most 2.98e-08 (measured), which is the Float32 round trip that 4d333a0 fixed for the composite. The composite is fine: miss and hit are identical in all arms. + +## Evidence + +Scripts in `scratchpad/r3/`: +- `col.R`: local collection. +- `e2e3.R`: real assemble, composite 4 arms, false colour, cube x2. +- `e2e3_mut.R` with `mut/`: ordering mutant. +- `fixok/`: strengthened fixture on the unmutated tree. +- `fetchw.R`: fetch writers. +- `nodata.R`: COG NoData and layout. +- `maprgb.R`: map with a real composite. +- `cubehit.R`: cube miss vs hit. diff --git a/tests/testthat/test-dft_stac_composite.R b/tests/testthat/test-dft_stac_composite.R index 93ab833..356aafb 100644 --- a/tests/testthat/test-dft_stac_composite.R +++ b/tests/testthat/test-dft_stac_composite.R @@ -217,9 +217,12 @@ test_that("dft_stac_composite returns a cached, cloud-masked true-colour COG", { expect_lt(stats::median(v, na.rm = TRUE), 0.3) expect_gt(stats::median(v, na.rm = TRUE), 0) - f <- list.files(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), - pattern = "^composite_.*\\.tif$", full.names = TRUE) - expect_length(f, 1) + # every file, not a .tif pattern: a sidecar beside the COG must show up here + all_files <- list.files(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), + all.files = TRUE, no.. = TRUE) + expect_length(all_files, 1) + expect_match(all_files, "^composite_[0-9a-f]{16}\\.tif$") + f <- file.path(drift:::cache_scheme_dir(cache, "sentinel-2-l2a"), all_files) # A COG by its layout, not by an exit status: GDAL reports the layout in the # image structure metadata only for a file written as a COG. info <- paste(sf::gdal_utils("info", f, quiet = TRUE), collapse = "\n") @@ -269,7 +272,11 @@ test_that("a built composite leaves only its COG in the cache, no sidecar", { stac_cube_items = function(...) list(features = list(), is_pre = logical(0)), stac_cube_assemble = function(...) { r <- terra::rast(terra::ext(aoi_t), resolution = 50, crs = "EPSG:32609", - nlyrs = 3, vals = 0.05) + nlyrs = 3) + # distinct per band, so a build that renamed by position instead of + # reordering by name would put 0.02 in red (code-check round 3) + terra::values(r) <- matrix(rep(c(0.02, 0.05, 0.1), each = terra::ncell(r)), + ncol = 3) names(r) <- c("blue", "green", "red") # alphabetical, as terra reads it # and with a time, as terra reads the gdalcubes NetCDF: a fixture without # one cannot reach the sidecar (code-check round 2) @@ -284,5 +291,7 @@ test_that("a built composite leaves only its COG in the cache, no sidecar", { expect_length(files, 1L) expect_match(files, "^composite_[0-9a-f]{16}\\.tif$") expect_equal(names(out[[1]]), c("red", "green", "blue")) + expect_equal(unname(unlist(terra::global(out[[1]], "max"))), c(0.1, 0.05, 0.02), + tolerance = 1e-6) expect_equal(terra::time(out[[1]])[1], as.Date("2023-06-01")) }) From 2c30c3c4b51836254585cf9344b9fd20099b4f3a Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 08:24:50 -0700 Subject: [PATCH 19/29] Document the composite and per-point chip pattern in CLAUDE.md (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- CLAUDE.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3155504..9a11c62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,6 +74,17 @@ breaks <- dft_rast_break(cube, start = c(2022, 1)) # abrupt change, dated (BFA trend <- dft_rast_trend(cube) # gradual change (Theil-Sen slope + Mann-Kendall) ``` +Dated reference imagery (#79): per-year median reflectance composites on the same read path as +the cube. They are cached as COGs and drawn beneath the land cover with one stretch shared across years: + +```r +tc <- dft_stac_composite(aoi, years = c(2017, 2023), months = 6:7) # true colour; bands = roles +dft_map_interactive(classified, aoi = aoi, rgb = tc) # or rgb = COG URLs via titiler +# sample points: ONE CALL PER BUFFERED POINT (small, separately cached chips). +# All points in one aoi spans the floodplain bbox under one cache key. +chips <- lapply(seq_len(nrow(buf)), \(i) dft_stac_composite(buf[i, ], years = 2023)) +``` + ## Key Patterns - **Dual-mode maps:** `dft_map_interactive()` uses `addRasterImage()` for local SpatRasters, `addTiles()` via titiler for remote COGs From fecdae931b61a31f3c959bb4e72c627438256fa7 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 09:22:33 -0700 Subject: [PATCH 20/29] Query a floodplain-sized AOI by its convex hull (#79) The BULK scale test failed in 9 s: Planetary Computer returns HTTP 413 when the POST search body passes about 1 MiB (measured: accepted at 21,348 vertices / 860 KB, rejected at 26,508 / 1.07 MB). BULK's floodplain is 104,584 vertices, so neither the cube nor the composite could query it at all. Above 20,000 vertices the intersects geometry is now the convex hull, a superset that changes no pixel over the AOI. Below that the query is exactly the union, as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_cube.R | 25 ++++++++++++++++++++++++- tests/testthat/test-dft_stac_cube.R | 18 ++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/R/dft_stac_cube.R b/R/dft_stac_cube.R index 267bce1..5d52d42 100644 --- a/R/dft_stac_cube.R +++ b/R/dft_stac_cube.R @@ -399,7 +399,7 @@ stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, # union so a multi-feature AOI queries its whole footprint, matching the # cube extent and the terra::mask() clip (a single first-feature geometry # would leave silent NoData holes over the other features) - intersects = sf::st_geometry(sf::st_union(aoi_wgs84))[[1]], + intersects = stac_query_geometry(aoi_wgs84), datetime = datetime, limit = 500 ) |> @@ -447,6 +447,29 @@ stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, } +#' The geometry sent as a STAC `intersects` query: the AOI union, or its hull +#' +#' Planetary Computer rejects a search whose POST body is over about 1 MiB with +#' HTTP 413 (measured 2026-09-28, #79: accepted at 21,348 vertices / 860 KB of +#' GeoJSON, rejected at 26,508 / 1.07 MB). A floodplain polygon crosses that +#' easily; BULK's is 104,584 vertices (4.2 MB), so neither dft_stac_cube() nor +#' dft_stac_composite() could query it at all. Above `max_vertices` the query +#' uses the convex hull instead. That is a superset, so no scene over the AOI is +#' lost; the extra scenes it admits lie outside the AOI, gdalcubes reads only +#' images overlapping the view, and the output is clipped, so pixels over the AOI +#' are unchanged. At or below the threshold the query is exactly the union, as +#' before, so any AOI that could be queried until now gets the same item set. +#' @noRd +stac_query_geometry <- function(aoi_wgs84, max_vertices = 20000L) { + u <- sf::st_union(aoi_wgs84) + n <- nrow(sf::st_coordinates(u)) + if (n > max_vertices) { + message(" AOI has ", n, " vertices; querying its convex hull (STAC body limit)") + u <- sf::st_convex_hull(u) + } + sf::st_geometry(u)[[1]] +} + #' Assemble a masked cube stack over the AOI, tiled or not #' #' Shared by [dft_stac_cube()] and [dft_stac_composite()]; the two differ only in diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index dc110dc..1fbc907 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -711,3 +711,21 @@ test_that("cache_write_atomic moves a terra .aux.json sidecar with the raster", # the sidecar, when the driver writes one, sits under the canonical name expect_true(all(files %in% c("entry.tif", "entry.tif.aux.json", "entry.tif.aux.xml"))) }) + +test_that("stac_query_geometry sends the union below the vertex limit and a hull above", { + # Planetary Computer returns HTTP 413 above ~1 MiB of POST body; BULK's + # floodplain (104,584 vertices) could not be queried at all (#79). + aoi <- sf::st_transform( + sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift"), + quiet = TRUE), 4326) + u <- sf::st_geometry(sf::st_union(aoi))[[1]] + n <- nrow(sf::st_coordinates(u)) + # below the limit: exactly the union, so existing queries are unchanged + expect_identical(drift:::stac_query_geometry(aoi, max_vertices = n), u) + # above it: a hull, far smaller, still covering the AOI + expect_message(h <- drift:::stac_query_geometry(aoi, max_vertices = n - 1L), + "convex hull") + expect_lt(nrow(sf::st_coordinates(h)), n / 10) + expect_true(isTRUE(sf::st_covers(sf::st_sfc(h, crs = 4326), + sf::st_sfc(u, crs = 4326), sparse = FALSE)[1, 1])) +}) From e9e46a7b2ead3e351e6354f7a83e961adb74505a Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 09:23:53 -0700 Subject: [PATCH 21/29] BULK scale test for dft_stac_composite: 100 per-point chips (#79) 100 chips over the BULK floodplain, 2023 Jul-Aug: 84.1 min, median 40.2 s per chip, peak RSS 0.48 GiB. The cost is COG reads, not the STAC query (#85). The floodplain-wide stage follows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- data-raw/benchmark_composite_bulk-run.sh | 32 + data-raw/benchmark_composite_bulk.R | 95 + .../chips_per_chip.csv | 101 + .../benchmark_composite_bulk/rss_chips.txt | 2364 +++++++++++++++++ .../timings_chips.csv | 2 + .../wallclock_chips.txt | 2 + planning/active/progress.md | 7 + planning/active/task_plan.md | 4 +- 8 files changed, 2605 insertions(+), 2 deletions(-) create mode 100755 data-raw/benchmark_composite_bulk-run.sh create mode 100644 data-raw/benchmark_composite_bulk.R create mode 100644 data-raw/logs/benchmark_composite_bulk/chips_per_chip.csv create mode 100644 data-raw/logs/benchmark_composite_bulk/rss_chips.txt create mode 100644 data-raw/logs/benchmark_composite_bulk/timings_chips.csv create mode 100644 data-raw/logs/benchmark_composite_bulk/wallclock_chips.txt diff --git a/data-raw/benchmark_composite_bulk-run.sh b/data-raw/benchmark_composite_bulk-run.sh new file mode 100755 index 0000000..b592076 --- /dev/null +++ b/data-raw/benchmark_composite_bulk-run.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Run data-raw/benchmark_composite_bulk.R for one stage, sampling the Rscript +# process's RSS every 2 s and recording the wall clock (drift#79). +# +# bash data-raw/benchmark_composite_bulk-run.sh chips +# bash data-raw/benchmark_composite_bulk-run.sh floodplain +# +# Rscript is started alone with `&` so $! is its PID. Gated on the script's +# `ALL STAGES DONE` marker, not on the exit status. +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 +[ $# -eq 1 ] || { echo "usage: $0 " >&2; exit 2; } +s="$1" +d="data-raw/logs/benchmark_composite_bulk" +mkdir -p "$d" +: > "$d/rss_$s.txt" +date -u +"%Y-%m-%dT%H:%M:%SZ start $s" > "$d/wallclock_$s.txt" +Rscript data-raw/benchmark_composite_bulk.R "$s" > "$d/run_$s.log" 2>&1 & +pid=$! +while kill -0 "$pid" 2>/dev/null; do + ps -o rss= -p "$pid" >> "$d/rss_$s.txt" + sleep 2 +done +wait "$pid"; rc=$? +date -u +"%Y-%m-%dT%H:%M:%SZ end $s (exit $rc)" >> "$d/wallclock_$s.txt" +peak=$(sort -n "$d/rss_$s.txt" | tail -1 | awk '{printf "%.2f", $1/1024/1024}') +if grep -q "ALL STAGES DONE" "$d/run_$s.log"; then + echo "OK $s peak RSS ${peak} GiB $(grep -E '^(chips|floodplain):' "$d/run_$s.log" | tail -1)" +else + echo "FAIL $s (exit $rc) $(grep -iE 'error|halted' "$d/run_$s.log" | head -3)" + exit 1 +fi diff --git a/data-raw/benchmark_composite_bulk.R b/data-raw/benchmark_composite_bulk.R new file mode 100644 index 0000000..ce08929 --- /dev/null +++ b/data-raw/benchmark_composite_bulk.R @@ -0,0 +1,95 @@ +# Scale test for dft_stac_composite() on the BULK floodplain (drift#79). +# +# The bundled Neexdzii Kwa reach cannot reach memory or runtime failure modes, +# so the CLAUDE.md convention is to run new raster functions on the BULK +# floodplain (bulk_co_ff04, 386.5 km2 inside a ~146 x 115 km bbox) before the PR. +# +# Two stages, because #79 serves two paths: +# chips 100 sample points in the floodplain, each buffered 300 m and +# composited on its own: the floodplains#93 review path. +# floodplain one floodplain-wide true-colour composite, tiled: the COG path +# for titiler browsing. +# +# Rscript data-raw/benchmark_composite_bulk.R chips +# Rscript data-raw/benchmark_composite_bulk.R floodplain +# +# Run through data-raw/benchmark_composite_bulk-run.sh, which samples RSS every +# 2 s from outside the process and records the wall clock. + +# Load the SOURCE tree, never the installed package (code-check-r.md). +pkgload::load_all(quiet = TRUE) + +stage <- commandArgs(trailingOnly = TRUE)[1] +stopifnot(stage %in% c("chips", "floodplain")) + +out_dir <- file.path("data-raw", "logs", "benchmark_composite_bulk") +dir.create(out_dir, recursive = TRUE, showWarnings = FALSE) +cache <- file.path(tempdir(), "drift_composite_bulk") + +# --- BULK floodplain polygon (the `floodplain` asset of bulk_co_ff04) ------- +gpkg <- file.path(out_dir, "floodplain.gpkg") +if (!file.exists(gpkg)) { + url <- "https://stac-floodplains-bc.s3.us-west-2.amazonaws.com/bulk_co_ff04/floodplain.gpkg" + tmp <- tempfile(fileext = ".gpkg") + resp <- curl::curl_fetch_disk(url, tmp) + if (resp$status_code != 200) stop("floodplain.gpkg fetch returned HTTP ", resp$status_code) + stopifnot(file.copy(tmp, gpkg)) +} +# co_ff04 explicitly: the file also carries ff02/ff06 and st_read() takes ff02 +aoi <- sf::st_read(gpkg, layer = "co_ff04", quiet = TRUE) + +timings <- list() +tick <- function(label, expr) { + t0 <- Sys.time() + force(expr) + secs <- as.numeric(difftime(Sys.time(), t0, units = "secs")) + timings[[label]] <<- secs + message(sprintf("%s: %.1f s", label, secs)) + invisible(secs) +} + +if (stage == "chips") { + set.seed(79) + pts <- sf::st_as_sf(sf::st_sample(sf::st_union(aoi), 100)) + buf <- sf::st_buffer(pts, 300) + per_chip <- numeric(nrow(buf)) + cells <- numeric(nrow(buf)) + tick("wall", { + for (i in seq_len(nrow(buf))) { + t0 <- Sys.time() + chip <- suppressMessages( + dft_stac_composite(buf[i, ], years = 2023, months = 7:8, cache_dir = cache) + ) + per_chip[i] <- as.numeric(difftime(Sys.time(), t0, units = "secs")) + cells[i] <- terra::ncell(chip[[1]]) + } + }) + utils::write.csv(data.frame(chip = seq_along(per_chip), secs = per_chip, + cells = cells), + file.path(out_dir, "chips_per_chip.csv"), row.names = FALSE) + cache_mb <- sum(file.size(list.files(cache, recursive = TRUE, + full.names = TRUE))) / 2^20 + message(sprintf("chips: %d, median %.1f s, max %.1f s, cells/chip %d, cache %.1f MB", + length(per_chip), stats::median(per_chip), max(per_chip), + as.integer(stats::median(cells)), cache_mb)) +} + +if (stage == "floodplain") { + tick("wall", { + fp <- dft_stac_composite(aoi, years = 2023, months = 7:8, clip = TRUE, + tile_size = 20000, cache_dir = cache) + }) + r <- fp[[1]] + f <- list.files(cache, pattern = "^composite_.*\\.tif$", recursive = TRUE, + full.names = TRUE) + message(sprintf("floodplain: %d x %d x %d, notNA %.1f%%, COG %.1f MB", + terra::nrow(r), terra::ncol(r), terra::nlyr(r), + 100 * terra::global(r[[1]], "notNA")[1, 1] / terra::ncell(r), + file.size(f[1]) / 2^20)) +} + +utils::write.csv(data.frame(stage = stage, what = names(timings), + secs = unlist(timings)), + file.path(out_dir, paste0("timings_", stage, ".csv")), + row.names = FALSE) +message("ALL STAGES DONE") diff --git a/data-raw/logs/benchmark_composite_bulk/chips_per_chip.csv b/data-raw/logs/benchmark_composite_bulk/chips_per_chip.csv new file mode 100644 index 0000000..672e0a4 --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/chips_per_chip.csv @@ -0,0 +1,101 @@ +"chip","secs","cells" +1,36.6357309818268,3660 +2,41.3755371570587,3660 +3,39.5575051307678,3660 +4,41.8905329704285,3660 +5,32.8213548660278,3660 +6,52.275456905365,3660 +7,47.8954379558563,3660 +8,92.898992061615,3660 +9,41.6701691150665,3660 +10,53.8903439044952,3660 +11,39.2086861133575,3660 +12,39.479728937149,3660 +13,54.4840381145477,3660 +14,92.3897070884705,3660 +15,76.9595301151276,3660 +16,76.5363321304321,3660 +17,39.6369619369507,3660 +18,46.846911907196,3660 +19,64.0215938091278,3660 +20,41.1457362174988,3660 +21,40.6029620170593,3660 +22,46.6963188648224,3660 +23,38.8354349136353,3660 +24,48.1532070636749,3660 +25,104.130614995956,3660 +26,74.1742479801178,3660 +27,43.74281001091,3660 +28,39.7970249652863,3660 +29,28.0558130741119,3660 +30,89.8523728847504,3660 +31,39.949254989624,3660 +32,140.90344285965,3660 +33,29.463161945343,3660 +34,40.6727838516235,3660 +35,39.270476102829,3660 +36,80.3213851451874,3660 +37,40.5032680034637,3660 +38,67.6864511966705,3660 +39,40.3847680091858,3660 +40,39.1982610225677,3660 +41,25.9964740276337,3660 +42,39.7723071575165,3660 +43,39.2951791286469,3660 +44,21.7727229595184,3660 +45,40.1111578941345,3660 +46,38.9243609905243,3660 +47,89.7190139293671,3660 +48,40.3862979412079,3660 +49,63.141930103302,3660 +50,39.4003620147705,3660 +51,38.4212319850922,3660 +52,39.5450019836426,3660 +53,74.9018950462341,3660 +54,70.9632270336151,3660 +55,39.211667060852,3660 +56,25.3026850223541,3660 +57,39.4397349357605,3660 +58,95.6239521503448,3660 +59,39.7771890163422,3660 +60,38.9563360214233,3660 +61,29.0469450950623,3660 +62,95.102972984314,3660 +63,38.8466799259186,3660 +64,24.3591551780701,3660 +65,39.4318840503693,3660 +66,39.0439019203186,3660 +67,25.5492069721222,3660 +68,38.8576078414917,3660 +69,31.7570109367371,3660 +70,38.4806900024414,3660 +71,106.015300989151,3660 +72,39.6910409927368,3660 +73,41.3182570934296,3660 +74,72.3575150966644,3660 +75,28.7851848602295,3660 +76,38.2615630626678,3660 +77,39.110358953476,3660 +78,40.1957859992981,3660 +79,41.3105008602142,3660 +80,63.1865417957306,3660 +81,26.8262450695038,3660 +82,40.5082311630249,3660 +83,26.8884909152985,3660 +84,39.8314001560211,3660 +85,28.8725209236145,3660 +86,76.9768869876862,3660 +87,40.2793688774109,3660 +88,92.1693139076233,3660 +89,40.1662518978119,3660 +90,40.0150110721588,3660 +91,82.8547601699829,3660 +92,39.4981660842896,3660 +93,110.509310007095,3660 +94,41.3110089302063,3660 +95,39.7005550861359,3660 +96,39.1111390590668,3660 +97,42.0266511440277,3660 +98,71.8980669975281,3660 +99,39.4107091426849,3660 +100,71.7192101478577,3660 diff --git a/data-raw/logs/benchmark_composite_bulk/rss_chips.txt b/data-raw/logs/benchmark_composite_bulk/rss_chips.txt new file mode 100644 index 0000000..dbe551e --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/rss_chips.txt @@ -0,0 +1,2364 @@ + 32 +268992 +336432 +337408 +342736 +342736 +342736 +342736 +342736 +342736 +342736 +342752 +342752 +342752 +342752 +342752 +342752 +342752 +342752 +355360 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +356336 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +361968 +363936 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425376 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +425648 +426288 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426304 +426944 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427408 +427456 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +428176 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429264 +429408 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430464 +430512 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +431104 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +441856 +451344 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +452144 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455376 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +455408 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457696 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457840 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +457968 +458000 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458240 +458400 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458480 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458928 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458960 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +458976 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +458976 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459008 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459328 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459408 +459456 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459760 +459840 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +460000 +461024 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462112 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +462656 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +465824 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +466528 +467120 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467440 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +467712 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +468384 +471520 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +471552 +473056 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +473248 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +474256 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487376 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487408 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487456 +487728 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +487968 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488016 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488560 +488768 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +488784 +490992 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493584 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493984 +493968 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +494000 +503296 +473840 +473840 +473840 +473840 +473840 +473840 +473840 +473840 +473840 +473840 +473840 +478544 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +479248 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +481760 +403440 +403440 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +388400 +403840 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +407376 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +410224 +399136 +399136 +399136 +399136 +399136 +399136 +399136 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +408848 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419616 +419568 +419568 +419392 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416784 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +416768 +427152 +427296 +427296 +418000 +418000 +418000 +396304 +383792 +383792 +383792 +383792 +383792 +383792 +383792 +383792 +383792 +383792 +383792 +387920 +388672 +388672 +388672 +388672 +388672 +388672 +388672 +388672 +388672 +388672 +396688 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +389840 +392224 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +392256 +401728 +402048 +402048 +402048 +402048 +402048 +402048 +402048 +402048 +402048 +402048 +402048 +408272 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408384 +408592 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +408752 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +410096 +411056 +412816 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +413088 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +403104 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +417904 +422384 +423216 +423216 +423216 +423216 +423216 +423216 +423216 +423216 +423216 +423216 +391936 +389184 +389136 +368832 +368672 +368672 +368672 +368672 +308944 +350128 +350128 +350128 +350128 +350128 +350112 +350112 +350000 +349472 +349472 +349472 +349472 +349472 +349472 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +349584 +370384 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +371120 +386848 +375232 +375232 +375232 +373568 +373568 +371056 +371056 +370944 +370944 +370944 +370944 +370944 +370944 +370944 +370944 +370944 +370944 +384384 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +385040 +398064 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +399040 +406768 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +407024 +409728 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +409840 +411632 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411776 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +411856 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422784 +422928 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +422944 +423744 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +423792 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425216 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +425232 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426064 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426192 +426208 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431904 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +431952 +432016 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +432048 +433696 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +433840 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434032 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434176 +434352 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435072 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +435104 +438528 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +440944 +442192 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442608 +442832 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 +442848 diff --git a/data-raw/logs/benchmark_composite_bulk/timings_chips.csv b/data-raw/logs/benchmark_composite_bulk/timings_chips.csv new file mode 100644 index 0000000..30ff16d --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/timings_chips.csv @@ -0,0 +1,2 @@ +"stage","what","secs" +"chips","wall",5045.98043298721 diff --git a/data-raw/logs/benchmark_composite_bulk/wallclock_chips.txt b/data-raw/logs/benchmark_composite_bulk/wallclock_chips.txt new file mode 100644 index 0000000..882a00c --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/wallclock_chips.txt @@ -0,0 +1,2 @@ +2026-09-28T14:55:40Z start chips +2026-09-28T16:19:50Z end chips (exit 0) diff --git a/planning/active/progress.md b/planning/active/progress.md index 56f1d50..47764dc 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -33,3 +33,10 @@ - Served the widgets on localhost and inspected them in Chrome. Both layers render opaque and register with the Esri basemap. 2023 Aug-Sep reads darker and hazier than 2017 Jun-Jul under the shared stretch. Medians (red, green, blue): 2017 0.030, 0.050, 0.022; 2023 0.036, 0.050, 0.030. One scene seam is visible in 2017. - Pre-existing and unrelated: the default "Light" basemap (CartoDB.Positron) now serves "API KEY REQUIRED" tiles. - Phase 5 done: re-read #79 (unchanged since the gate), ran the HLS spike (findings.md), filed #82, filed #83 (pre-existing cube-query defects), and edited the #79 body (HLS -> #82, scale pivot, #80 dependency). +- Code-check (`review-79-round{1,2,3}.md`): + - R1: the COG sidecar orphan and the transition legend with x = NULL, both fixed. + - R2: **a defect inside the R1 fix.** The NetCDF-read time persisted, and the fixture had no time. Fixed by stripping time and moving `.aux.json` in `cache_write_atomic()`. + - R3: the enumeration of every raster write and every stub. No production defect; two test gaps fixed, and the layer-reorder mutant is now killed offline. The loop ended on that enumeration. +- **BULK chips:** 100 buffered points (300 m), 2023 Jul-Aug, 84.1 min wall. Per chip: median 40.2 s, mean 50.5 s, max 140.9 s. 3,660 cells per chip, a 3.8 MB cache, peak RSS 0.48 GiB. The time is COG reads (the query is about 2 s), so concurrent chips are filed as #85. +- **BULK floodplain, first attempt:** HTTP 413 in 9 s. The 104,584-vertex floodplain made a 4.2 MB POST body; PC accepts at 860 KB and rejects at 1.07 MB. Pre-existing for `dft_stac_cube()` too. Fixed by querying with the convex hull above 20,000 vertices (fecdae9). The re-run got past the query: 72 items, 30 tiles. +- Filed #85 (concurrent chips) and soul#281 (report times in Vancouver local). diff --git a/planning/active/task_plan.md b/planning/active/task_plan.md index d27ebd8..e05258d 100644 --- a/planning/active/task_plan.md +++ b/planning/active/task_plan.md @@ -64,11 +64,11 @@ Plan-gate decisions (2026-09-28): ## Phase 6: scale test, docs, release -- [ ] BULK scale test with the RSS sampler. Record wall time and peak RSS for the PR: +- [x] BULK scale test (chips done; floodplain stage re-running after the 413 fix) with the RSS sampler. Record wall time and peak RSS for the PR: - `dft_stac_composite()` on `bulk_co_ff04`, one year, `tile_size` set; - a chip run over ~100 scattered buffered points. Also note what `dft_map_interactive(rgb=)` does with a floodplain-size local raster (maxBytes), to confirm the doc points to the COG path. -- [ ] Update NEWS.md, the CLAUDE.md Core Pipeline (the composite call and the chip pattern), and the gotchas note if anything turns up. +- [ ] Update NEWS.md, the CLAUDE.md Core Pipeline (CLAUDE.md done, 2c30c3c) (the composite call and the chip pattern), and the gotchas note if anything turns up. - [ ] Bump the version (minor) as the final commit. ## Validation From 5d8769d1a1876cd5ebf97909b0077501e3a56970 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 10:21:16 -0700 Subject: [PATCH 22/29] Re-sign STAC features before each read extent (#79) Planetary Computer SAS tokens last about 45 minutes and features were signed once, at query time. The BULK floodplain composite read for 54 minutes over 30 tiles, and 15 tiles came back with every chunk failed. rstac's signer refreshes an expired token, so the features are now re-signed per extent. Verified live: corrupted tokens read 102,364 cells after re-signing, 0 without. Detecting failed chunk reads is #87; gdalcubes reports them only on worker stderr, which R cannot reliably capture. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- R/dft_stac_cube.R | 42 ++++++++++++++++++++++++----- tests/testthat/test-dft_stac_cube.R | 24 +++++++++++++++++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/R/dft_stac_cube.R b/R/dft_stac_cube.R index 5d52d42..09e4dcb 100644 --- a/R/dft_stac_cube.R +++ b/R/dft_stac_cube.R @@ -387,8 +387,10 @@ stac_cube_cache_read <- function(cache_file, collection, datetime, #' box; and a scene-level `eo:cloud_cover` pre-filter shrinks the collection #' before any pixel is read. #' -#' @return A list with `features` (the signed, month-filtered STAC features) and -#' `is_pre` (logical, one per feature: acquired before `cfg$offset_boundary`). +#' @return A list with `features` (the signed, month-filtered STAC features), +#' `is_pre` (logical, one per feature: acquired before `cfg$offset_boundary`), +#' and `items` and `sign_fn`, which stac_cube_assemble() uses to re-sign the +#' features before each read extent (see there). #' @noRd stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, sign_fn) { @@ -443,10 +445,36 @@ stac_cube_items <- function(cfg, aoi_wgs84, datetime, cloud_cover_max, months, message(" offset split at ", cfg$offset_boundary, ": ", sum(is_pre), " pre / ", sum(!is_pre), " post") } - list(features = items$features, is_pre = is_pre) + list(features = items$features, is_pre = is_pre, items = items, + sign_fn = sign_fn) } +#' Re-sign STAC features just before they are read +#' +#' Planetary Computer SAS tokens last about 45 minutes (measured 2026-09-28: +#' a token issued at 17:16:55Z expired at 18:01:55Z), and signing happens once, +#' at query time. A tiled read over a floodplain outlasts that: on BULK (#79) +#' 15 of 30 tiles came back with every chunk failed. rstac's signer refreshes an +#' expired token and replaces the `sig` parameter of an already-signed href, so +#' re-signing before each extent keeps every read inside a live token (verified +#' live: features with a corrupted `sig` read 102,364 cells after re-signing, 0 +#' without). A single extent whose read outlives one token is still exposed, and +#' gdalcubes reports a failed chunk only on the worker's stderr, which R cannot +#' reliably capture (#87). A `fetched` without `sign_fn` (a test stub) passes +#' through unchanged. +#' @noRd +stac_features_resign <- function(fetched, feats) { + if (is.null(fetched$sign_fn) || is.null(fetched$items) || !length(feats)) { + return(feats) + } + it <- fetched$items + it$features <- feats + rstac::items_sign(it, sign_fn = fetched$sign_fn)$features +} + + + #' The geometry sent as a STAC `intersects` query: the AOI union, or its hull #' #' Planetary Computer rejects a search whose POST body is over about 1 MiB with @@ -536,13 +564,15 @@ stac_cube_assemble <- function(fetched, cfg, aoi_target, target_crs, t0, t1, dx = res, dy = res, dt = dt, aggregation = aggregation, resampling = resampling ) + # re-sign here, per extent, so no read outlives its token + feats <- stac_features_resign(fetched, features) if (any(is_pre) && !all(is_pre)) { terra::cover( - build_stack(features[is_pre], offset_before, v), - build_stack(features[!is_pre], offset, v) + build_stack(feats[is_pre], offset_before, v), + build_stack(feats[!is_pre], offset, v) ) } else { - build_stack(features, if (all(is_pre)) offset_before else offset, v) + build_stack(feats, if (all(is_pre)) offset_before else offset, v) } } diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index 1fbc907..c8f7f6c 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -729,3 +729,27 @@ test_that("stac_query_geometry sends the union below the vertex limit and a hull expect_true(isTRUE(sf::st_covers(sf::st_sfc(h, crs = 4326), sf::st_sfc(u, crs = 4326), sparse = FALSE)[1, 1])) }) + +test_that("stac_features_resign re-signs features before a read, stubs pass through", { + # Planetary Computer tokens last ~45 min and a tiled floodplain read outlives + # one (#79: 15 of 30 BULK tiles failed). Re-signing per extent refreshes them. + feat <- function(sig) list(type = "Feature", id = "a", + assets = list(B04 = list(href = paste0("https://x/a.tif?sig=", sig)))) + items <- structure(list(type = "FeatureCollection", features = list(feat("old"))), + class = c("doc_items", "rstac_doc", "list")) + calls <- 0L + signer <- function(item, ...) { + calls <<- calls + 1L + item$assets$B04$href <- sub("sig=[^&]*", "sig=fresh", item$assets$B04$href) + item + } + fetched <- list(features = items$features, is_pre = FALSE, items = items, + sign_fn = signer) + out <- drift:::stac_features_resign(fetched, list(feat("expired"))) + expect_equal(out[[1]]$assets$B04$href, "https://x/a.tif?sig=fresh") + expect_equal(calls, 1L) + # a stub without items/sign_fn, and an empty subset, pass through untouched + expect_identical(drift:::stac_features_resign(list(features = list()), list(feat("x"))), + list(feat("x"))) + expect_identical(drift:::stac_features_resign(fetched, list()), list()) +}) From 27f6b55f1d6cd2f31ac0a3e5fa0be4426b7967c4 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 12:24:36 -0700 Subject: [PATCH 23/29] Record the BULK floodplain runs 2 and 3 (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- planning/active/progress.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/planning/active/progress.md b/planning/active/progress.md index 47764dc..b6dcaa3 100644 --- a/planning/active/progress.md +++ b/planning/active/progress.md @@ -40,3 +40,5 @@ - **BULK chips:** 100 buffered points (300 m), 2023 Jul-Aug, 84.1 min wall. Per chip: median 40.2 s, mean 50.5 s, max 140.9 s. 3,660 cells per chip, a 3.8 MB cache, peak RSS 0.48 GiB. The time is COG reads (the query is about 2 s), so concurrent chips are filed as #85. - **BULK floodplain, first attempt:** HTTP 413 in 9 s. The 104,584-vertex floodplain made a 4.2 MB POST body; PC accepts at 860 KB and rejects at 1.07 MB. Pre-existing for `dft_stac_cube()` too. Fixed by querying with the convex hull above 20,000 vertices (fecdae9). The re-run got past the query: 72 items, 30 tiles. - Filed #85 (concurrent chips) and soul#281 (report times in Vancouver local). +- **BULK floodplain, run 2** (after the hull fix): 72 items, 30 tiles, 54 min, peak RSS 5.09 GiB. 15 of 30 tiles reported "9 out of 9 chunks" failed: PC SAS tokens last ~45 min and features were signed once. The final COG write then failed. A synthetic floodplain-size COG writes fine (82 s, 459 MB), so the failure is probably downstream of the failed tiles. Fixed by re-signing per extent (5d8769d); detection is #87, because stderr capture caught the report in only 1 of 4 configurations. +- **BULK floodplain, run 3** (re-sign, 2-h cap): killed at the cap. 16 tiles done, at 5.4-11.9 min each (~7 min mean), with **no chunk failures** even well past the 45-min token life. That is the scale evidence for the re-sign fix. A full run projects to ~3.5 h; run 4 is uncapped (5-h safety cap). From df2fd14dc0d5a75df809b88fbc7037d039a8d871 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:44:47 -0700 Subject: [PATCH 24/29] Record the BULK floodplain-wide run; keep temp-file evidence on failure (#79, #88) Run 4 read all 30 tiles in 3 h 16 min with no chunk failures (the per-extent re-signing holds at scale), peak RSS 3.90 GiB, then failed: terra::mask could not read the merged mosaic's temp GeoTIFF. It was not reproduced offline, and the evidence went with the session tempdir, so the stage now inventories temp files and gdalinfo -checksum on failure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- data-raw/benchmark_composite_bulk.R | 28 +- .../rss_floodplain.txt | 5487 +++++++++++++++++ .../wallclock_floodplain.txt | 2 + 3 files changed, 5513 insertions(+), 4 deletions(-) create mode 100644 data-raw/logs/benchmark_composite_bulk/rss_floodplain.txt create mode 100644 data-raw/logs/benchmark_composite_bulk/wallclock_floodplain.txt diff --git a/data-raw/benchmark_composite_bulk.R b/data-raw/benchmark_composite_bulk.R index ce08929..1109581 100644 --- a/data-raw/benchmark_composite_bulk.R +++ b/data-raw/benchmark_composite_bulk.R @@ -75,10 +75,30 @@ if (stage == "chips") { } if (stage == "floodplain") { - tick("wall", { - fp <- dft_stac_composite(aoi, years = 2023, months = 7:8, clip = TRUE, - tile_size = 20000, cache_dir = cache) - }) + # On failure, inventory R's temp files BEFORE the session deletes them: run 4 + # (2026-09-28) read all 30 tiles, then terra::mask() could not read the merged + # mosaic's spat_*.tif, and the evidence went with the tempdir (#88). + # tick() returns the seconds, so the composite is assigned inside the block + withCallingHandlers( + tick("wall", { + fp <- dft_stac_composite(aoi, years = 2023, months = 7:8, clip = TRUE, + tile_size = 20000, cache_dir = cache) + }), + error = function(e) { + message("FAILED: ", conditionMessage(e)) + tmp <- list.files(tempdir(), full.names = TRUE) + info <- file.info(tmp) + message("tempdir: ", tempdir(), " (", length(tmp), " files, ", + round(sum(info$size, na.rm = TRUE) / 2^30, 2), " GiB)") + for (f in tmp[grepl("^spat_", basename(tmp))]) { + message("--- ", basename(f), " ", info[f, "size"], " bytes, mtime ", + format(info[f, "mtime"])) + message(paste(tryCatch(system2("gdalinfo", c("-checksum", f), stdout = TRUE, + stderr = TRUE), error = function(x) "gdalinfo failed"), + collapse = "\n")) + } + } + ) r <- fp[[1]] f <- list.files(cache, pattern = "^composite_.*\\.tif$", recursive = TRUE, full.names = TRUE) diff --git a/data-raw/logs/benchmark_composite_bulk/rss_floodplain.txt b/data-raw/logs/benchmark_composite_bulk/rss_floodplain.txt new file mode 100644 index 0000000..f9647c4 --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/rss_floodplain.txt @@ -0,0 +1,5487 @@ + 32 +277152 +440128 +472192 +532240 +595488 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +595472 +593648 +592000 +591520 +591520 +510608 +510608 +510608 +510608 +453728 +440800 +372368 +277904 +277904 +299648 +299648 +299648 +299648 +299648 +299648 +332736 +332736 +332736 +332736 +332736 +332736 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +355328 +369280 +369280 +369280 +369280 +369280 +369280 +369280 +369280 +369280 +369280 +362992 +362992 +362992 +362992 +363008 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +362944 +393328 +407152 +407152 +407152 +407152 +406960 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +434640 +448944 +448944 +448944 +448944 +448944 +448944 +448944 +448944 +448944 +448944 +448944 +448960 +448960 +448960 +448960 +448960 +448960 +448960 +558848 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625520 +625680 +625680 +625680 +625680 +625680 +625680 +625680 +625680 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +630144 +655680 +668256 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673424 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673440 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +673408 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +674608 +706336 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +724432 +737936 +737920 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737984 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +737952 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +715712 +645520 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +651264 +659632 +659632 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +673456 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +685520 +699344 +699344 +699344 +699344 +699344 +699344 +699344 +699344 +699344 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +709104 +755216 +755904 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +756464 +761824 +761824 +761824 +761824 +761824 +761824 +761824 +775648 +775648 +775648 +775648 +775648 +775648 +775648 +775648 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +775664 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +784416 +811904 +812160 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812736 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +812752 +817392 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824576 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +824544 +820416 +820416 +820416 +820416 +820416 +820416 +820416 +809632 +705968 +705968 +705968 +705968 +705968 +705968 +720320 +720320 +720320 +720320 +720320 +720320 +733424 +733424 +733424 +733424 +733424 +733424 +733424 +733424 +733424 +733424 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +747536 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +761360 +774096 +780928 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +782048 +759872 +759872 +774624 +774624 +774624 +774624 +774624 +774624 +774624 +774624 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +788416 +796288 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +807648 +847648 +779888 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780384 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +780352 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +763520 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +765488 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +782560 +800672 +802048 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802352 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +802368 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +803360 +840592 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841312 +841328 +841328 +841328 +841328 +841328 +841328 +841328 +841328 +841328 +841328 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841296 +841344 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841792 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +841760 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +842768 +868688 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +867312 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853488 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +853456 +867280 +867280 +867280 +867280 +867280 +867280 +867280 +867280 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +880848 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +862672 +853136 +788560 +787856 +787856 +787856 +787856 +785376 +784272 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +780752 +746608 +615600 +615600 +615600 +615600 +615600 +615600 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +625360 +639184 +639184 +639184 +639184 +639184 +639184 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +619728 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +633552 +662192 +716240 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721184 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +721200 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +722208 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +725056 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +732976 +770368 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772160 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772128 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +772144 +781200 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +781312 +789680 +789680 +797904 +797904 +797904 +797904 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +793456 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +807280 +826496 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +835488 +827264 +827264 +736096 +736096 +736096 +736096 +736096 +737616 +737616 +737616 +737616 +737616 +737616 +737616 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +751440 +765296 +765296 +765296 +765296 +765296 +765296 +765296 +765296 +765296 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +779120 +810800 +819696 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824080 +824112 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832896 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +832864 +838112 +838112 +838112 +838112 +853504 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857632 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +857600 +841184 +841184 +841184 +841184 +818608 +803072 +788496 +747184 +747088 +727696 +649728 +575696 +575696 +575696 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +575120 +586960 +586960 +586960 +586960 +586960 +600784 +600784 +600784 +600784 +600784 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +614992 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +624720 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +631088 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +639824 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +643920 +679232 +702944 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710128 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +710144 +696384 +696384 +696384 +696384 +696384 +696384 +696384 +710208 +710208 +710208 +710208 +710208 +710208 +710208 +710208 +710208 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +724032 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +738400 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +743008 +799376 +759616 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +760128 +761072 +761072 +761072 +761072 +761072 +761072 +761072 +761072 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769264 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +769600 +790800 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783696 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +783728 +790304 +790272 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790448 +790464 +790464 +790464 +790464 +790464 +790464 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795024 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +795968 +808832 +809184 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +809568 +834544 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +834704 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836368 +836384 +836352 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +836624 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +837584 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +851408 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +837616 +754640 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +754416 +762768 +762768 +762768 +762768 +762768 +762768 +762768 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +776592 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +777920 +780080 +780080 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +793904 +825552 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +825584 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +831152 +844048 +1553248 +1555856 +1620096 +1624864 +1625248 +1625792 +1625856 +1630608 +1630608 +1630640 +1629520 +1630928 +1630928 +1630928 +1630976 +1631328 +4086272 diff --git a/data-raw/logs/benchmark_composite_bulk/wallclock_floodplain.txt b/data-raw/logs/benchmark_composite_bulk/wallclock_floodplain.txt new file mode 100644 index 0000000..930d1ad --- /dev/null +++ b/data-raw/logs/benchmark_composite_bulk/wallclock_floodplain.txt @@ -0,0 +1,2 @@ +2026-09-28T19:24:35Z start floodplain +2026-09-28T22:40:18Z end floodplain (exit 1) From c528f00587bc7dc65461ce2f4d5c08b7af28d9cb Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:44:57 -0700 Subject: [PATCH 25/29] NEWS for dft_stac_composite and the floodplain-scale findings (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- NEWS.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/NEWS.md b/NEWS.md index 27a4a45..656cab3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,23 @@ +# drift 0.18.0 + +- **Dated reference imagery (#79).** The Esri and Google basemaps say what a place is but not when it changed. New `dft_stac_composite()` returns a cloud-masked median composite of any Sentinel-2 band roles for each year's run of calendar months, for example true colour for June–July in 2017 and in 2023. Values are surface reflectance, not display bytes, so a composite is data and can later be classifier input. It shares its read path with `dft_stac_cube()`. The cache entry is written as a Cloud Optimized GeoTIFF, so it can be served through titiler unchanged. floodplains#93 is the first consumer: a stratified accuracy assessment of IO LULC, reviewed at sample points across whole floodplains. +- **Chips, one per point.** To review sample points, call `dft_stac_composite()` once per buffered point. Each chip is small and cached on its own, so adding a point fetches one chip. The obvious alternative, all the points as one AOI with read tiling, was the plan until review: it returns a raster the size of the floodplain under one cache key. On the BULK floodplain, 100 chips (300 m, 2023 Jul–Aug) took 84.1 min: median 40.2 s per chip, peak RSS 0.48 GiB. Almost all of that is remote COG reads (the STAC query is about 2 s), so running chips concurrently is #85. +- **`dft_map_interactive(rgb =)`.** Composites go on the map as switchable layers beneath the land cover, as rasters (leafem) or as COG URLs (titiler). Every composite gets the same per-band stretch, pooled across years, so a brightness difference between two years is in the data, not an artefact of stretching each image to its own histogram. `x` may be `NULL` for an imagery-only map. +- **NDWI and MNDWI** join `dft_index_table()`, with the `green` and `blue` Sentinel-2 roles they need. The tests evaluate the resolved expressions on known reflectance and on post-2022 DN through the offset, not only their strings. +- **Floodplain scale, two defects fixed and one open.** Both are in the read path the cube shares, so `dft_stac_cube()` had them too. + - Planetary Computer returns HTTP 413 once a search body passes about 1 MiB. BULK's floodplain is 104,584 vertices (4.2 MB of GeoJSON), so it could not be queried at all. Above 20,000 vertices the query now uses the AOI's convex hull; below that it is unchanged. + - Signed URLs expire in about 45 minutes and were signed once, so a 54-minute tiled read lost 15 of 30 tiles, silently. Features are now re-signed before each tile. A full BULK read then got all 30 tiles in 3 h 16 min with no failures. gdalcubes reports a failed chunk only on stderr, which R cannot reliably capture, so detection is #87. + - **Open (#88):** after that full read, clipping the merged mosaic failed ("cannot read from" terra's temp file). It did not reproduce offline. A floodplain-wide composite over an AOI the size of BULK does not complete yet; chips, and reach-scale composites, do. +- **Four defects found on the way, three of them silent.** + - terra reads a multi-variable gdalcubes NetCDF alphabetically. A true-colour request came back blue, green, red, and renaming layers by position would have swapped red and blue. The live end-to-end caught it through the layer-order guard. + - Classified and transition layers were projected to EPSG:4326 and handed to leaflet with `project = FALSE`, which assumes Web Mercator pixels. At reach scale the misregistration was invisible; beneath imagery at floodplain scale it is not. + - A date-only STAC end bound excludes that day's scenes, 22 of 23 in one measured window. The composite queries with explicit times; `dft_stac_cube()` has the same defect (#83). + - Five network end-to-end assertions had listed a pre-#48 cache path since that change, so they failed on every opt-in run. +- **What a composite refuses.** A window with Sentinel-2 scenes on both sides of the 2022-01-25 offset change has no single median, because gdalcubes aggregates before the offset is applied. It is refused rather than returned as a pre-side cover. The monthly cube's January 2022 layer has that bias today (#83). A year with no usable scenes is dropped with a warning, and the other years are kept. +- **`cache_write_atomic()` moves a `.aux.json` sidecar as well as `.aux.xml`.** A COG written from a raster carrying a time (or units, varnames, longnames, metags or scoff) gets one from terra. Before this, it was stranded under the temp name on every write. The composite also strips the time the gdalcubes NetCDF gives it before writing, so the cached COG stands alone. The first fix for this was incomplete, and code review caught it: the fixture had no time and could not reach the failure. +- **Refactor, measured.** The STAC query, offset split and tiled assembly moved out of `dft_stac_cube()` into helpers shared with the composite. A live untiled NDVI cube from main and from the refactor are identical cell for cell (max difference 0), and a new test pins how `dft_stac_cube()` calls its cache key. Both matter because the frozen-key test cannot see the call site. +- **HLS is #82.** A live spike found that CMR-STAC rejects `intersects` and ignores the CQL2 cloud filter, that reading needs a real Earthdata Login (`earthdatalogin`'s default credentials now return 401), and that S30 and L30 name NIR and SWIR differently. #82 records these and the source shape they imply. + # drift 0.17.1 - **gdalcubes installs from GitHub (#80).** gdalcubes was archived on CRAN on 2026-09-16. That broke the pkgdown workflow's dependency resolve, which had failed on every push since 2026-09-26, and it left new users unable to install what `dft_stac_fetch()` and `dft_stac_cube()` need. `appelmar/gdalcubes` is now in `Remotes:` with no suffix, and the README says to install it with `pak::pak("appelmar/gdalcubes")`. So does the prompt `dft_stac_fetch()`, `dft_stac_cube()` and `dft_index_expr()` raise when it is missing. rlang's own install offer means CRAN and would fail. Upstream master (0.7.5) has merged the `filter_geom` segfault fix from `NewGraphEnvironment/gdalcubes@newgraph`, so the fork is no longer needed. On 0.7.5 the offline suite passes (1007 tests), and so does the opt-in network end-to-end for the cube and fetch paths (109 and 165). That run also surfaced five network assertions that had listed the pre-#48 cache path since that change, so they failed on every opt-in run. They now use `cache_scheme_dir()`. From 505218d1e8c2cb18d41d62007f294791a130a647 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:45:26 -0700 Subject: [PATCH 26/29] gotchas: floodplain-scale and multi-band read findings (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- inst/notes/gdalcubes-pc-gotchas.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/inst/notes/gdalcubes-pc-gotchas.md b/inst/notes/gdalcubes-pc-gotchas.md index 9e4cdf3..9f686c7 100644 --- a/inst/notes/gdalcubes-pc-gotchas.md +++ b/inst/notes/gdalcubes-pc-gotchas.md @@ -194,3 +194,20 @@ Do not install the fork over 0.7.5. See `planning/archive/2026-07-issue-30-index-trajectory/findings.md` and `planning/archive/2026-07-issue-30-vignette-qa-map/findings.md` for the full empirical journey. + +## Floodplain scale and multi-band reads (#79, 2026-09-28) + +Measured on gdalcubes 0.7.5, rstac 1.0.1 and terra 1.9.50, with the BULK floodplain (`bulk_co_ff04`) and the packaged AOI. Scripts are `data-raw/benchmark_composite_bulk.R`; the numbers are in `planning/archive/2026-09-issue-79-dated-reference-imagery/`. + +- **Planetary Computer rejects a search body over about 1 MiB with HTTP 413.** It accepted 21,348 vertices (860 KB of GeoJSON) and rejected 26,508 (1.07 MB). BULK's floodplain is 104,584 vertices (4.2 MB), so an `intersects` query on it failed in 9 s. `stac_query_geometry()` switches to the convex hull above 20,000 vertices. The hull is a superset, and pixels over the AOI are unchanged. +- **PC SAS tokens last about 45 minutes** (a token issued at 17:16:55Z expired at 18:01:55Z), and features were signed once, at query time. A 54-minute tiled read lost 15 of 30 tiles. + - `stac_features_resign()` re-signs before each extent. rstac's signer refreshes an expired token and replaces the `sig` parameter of an already-signed href. + - Verified live: corrupted tokens read 102,364 cells after re-signing, 0 without. With re-signing, a full 3 h 16 min BULK read had no failed tiles. +- **gdalcubes reports failed chunks only on stderr**, as `[WARNING] n out of m chunks have repoprted errors / incompleteness`. It is not an R warning. The cube is written with those chunks NA and passes an any-data check. + - `capture.output(type = "message")` caught the line in 1 of 4 configurations, and never with `parallel > 1`, where workers write to the process's stderr directly. + - Do not build a guard on it. Detection is #87. +- **terra reads a multi-variable gdalcubes NetCDF with its variables in ALPHABETICAL order.** A true-colour `apply_pixel(names = c("red","green","blue"))` reads back as `blue, green, red`, so select layers by name, never by position. The composite does this in `composite_layers_order()`. + - The layers also arrive carrying a time (step `yearmonths`). + - Writing that to a COG makes terra emit a `.aux.json` sidecar. So do units, varnames, longnames, metags and scoff; names alone do not. +- **A date-only STAC end bound is read as 00:00Z**, so it drops that day's scenes, which in BC land around 19:00Z. A window ending on a scene day returned 22 of 23 items; the same window with `T23:59:59Z` returned 23. The composite uses explicit times. The cube still passes the user's string (#83). +- **CMR-STAC (NASA LPCLOUD, for HLS) differs from PC.** It returns 500 on `intersects` (use `bbox`) and ignores the CQL2 `eo:cloud_cover` filter (filter client-side). Its assets need a real Earthdata Login; `earthdatalogin`'s defaults return 401. See #82. From 481ad571939623e3114fac8ee0d1e277858f06ba Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:49:09 -0700 Subject: [PATCH 27/29] Fix two lints in the new cube tests (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- tests/testthat/test-dft_stac_cube.R | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/testthat/test-dft_stac_cube.R b/tests/testthat/test-dft_stac_cube.R index c8f7f6c..6557579 100644 --- a/tests/testthat/test-dft_stac_cube.R +++ b/tests/testthat/test-dft_stac_cube.R @@ -715,9 +715,8 @@ test_that("cache_write_atomic moves a terra .aux.json sidecar with the raster", test_that("stac_query_geometry sends the union below the vertex limit and a hull above", { # Planetary Computer returns HTTP 413 above ~1 MiB of POST body; BULK's # floodplain (104,584 vertices) could not be queried at all (#79). - aoi <- sf::st_transform( - sf::st_read(system.file("extdata", "example_aoi.gpkg", package = "drift"), - quiet = TRUE), 4326) + aoi_path <- system.file("extdata", "example_aoi.gpkg", package = "drift") + aoi <- sf::st_transform(sf::st_read(aoi_path, quiet = TRUE), 4326) u <- sf::st_geometry(sf::st_union(aoi))[[1]] n <- nrow(sf::st_coordinates(u)) # below the limit: exactly the union, so existing queries are unchanged @@ -733,8 +732,10 @@ test_that("stac_query_geometry sends the union below the vertex limit and a hull test_that("stac_features_resign re-signs features before a read, stubs pass through", { # Planetary Computer tokens last ~45 min and a tiled floodplain read outlives # one (#79: 15 of 30 BULK tiles failed). Re-signing per extent refreshes them. - feat <- function(sig) list(type = "Feature", id = "a", - assets = list(B04 = list(href = paste0("https://x/a.tif?sig=", sig)))) + feat <- function(sig) { + href <- paste0("https://x/a.tif?sig=", sig) + list(type = "Feature", id = "a", assets = list(B04 = list(href = href))) + } items <- structure(list(type = "FeatureCollection", features = list(feat("old"))), class = c("doc_items", "rstac_doc", "list")) calls <- 0L From 6bb3f3a184e1b2f3c4a4e49e3888db120c80926a Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:49:31 -0700 Subject: [PATCH 28/29] Archive planning files for issue #79 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- .../README.md | 61 +++++++++++++++++++ .../findings.md | 0 .../progress.md | 0 .../review-79-round1.md | 0 .../review-79-round2.md | 0 .../review-79-round3.md | 0 .../review-plan.md | 0 .../task_plan.md | 16 ++--- 8 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 planning/archive/2026-09-issue-79-dated-reference-imagery/README.md rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/findings.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/progress.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/review-79-round1.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/review-79-round2.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/review-79-round3.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/review-plan.md (100%) rename planning/{active => archive/2026-09-issue-79-dated-reference-imagery}/task_plan.md (91%) diff --git a/planning/archive/2026-09-issue-79-dated-reference-imagery/README.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/README.md new file mode 100644 index 0000000..e3c077c --- /dev/null +++ b/planning/archive/2026-09-issue-79-dated-reference-imagery/README.md @@ -0,0 +1,61 @@ +## Outcome + +Added dated reference imagery for floodplains#93's accuracy assessment: + +- `dft_stac_composite()`: per-year, cloud-masked median reflectance composites, cached as COGs, on a read path now shared with `dft_stac_cube()`; +- dated RGB layers in `dft_map_interactive(rgb =)`, with one stretch shared across years; +- NDWI and MNDWI. + +HLS was split to #82 after a live spike. The plan gate pivoted the work twice. #80 (gdalcubes archived on CRAN) went first as its own PR, #84. The scale requirement, sample points across whole floodplains, set the design. The plan review then showed the planned chip path (one tiled call over all points) would return a floodplain-sized raster under a single key, so chips became one call per point. + +What was learned is mostly about failure modes that stay silent at reach scale: + +- terra reads gdalcubes' multi-variable NetCDF alphabetically, so a true-colour request came back blue-green-red. The live e2e's layer-order guard caught it. +- A NetCDF-derived time made the COG writer strand a sidecar. The first fix missed it because its fixture carried no time. Code-check R2 caught that, and R3 closed the class by enumerating every raster write and every stub. +- Classified layers had always reached leaflet misregistered (4326 passed as 3857). +- A date-only STAC end bound drops the last day. +- At BULK scale: Planetary Computer's ~1 MiB search body limit made the floodplain unqueryable (fixed: convex hull), and ~45-min SAS tokens silently holed a long tiled read (fixed: re-sign per extent). A full floodplain-wide read then completed, but clipping the merged mosaic failed and did not reproduce offline (#88, open). + +The durable findings are in `inst/notes/gdalcubes-pc-gotchas.md`, section "Floodplain scale and multi-band reads". + +## Measurement + +**Refactor identity.** An untiled NDVI cube (packaged AOI, 2021-07/08, parallel 4) from main and from the refactor gave identical values and NA pattern, max diff 0 (77.1 s vs 77.7 s). The network e2e passed 110 for the cube, including tiled vs untiled, and 64 for the composite. + +**Per-point chips on BULK** (100 points, 300 m buffers, 2023 Jul-Aug): + +- 84.1 min total; +- per chip: median 40.2 s, mean 50.5 s, max 140.9 s; +- 3,660 cells per chip, a 3.8 MB cache, peak RSS 0.48 GiB. + +Splitting by stage: the STAC query is 1.4-2.5 s, and the rest is about 64 remote COG opens per chip. That turned "query once for all chips" into "run chips concurrently" (#85). + +**STAC body limit.** Planetary Computer accepted 21,348 vertices (860 KB) and returned HTTP 413 at 26,508 (1.07 MB). BULK is 104,584 vertices (4.2 MB). The hull is used above 20,000 vertices. + +**Tokens.** A token issued at 17:16:55Z expired at 18:01:55Z (about 45 min). Run 2 lost 15 of 30 tiles in 54 min. With corrupted tokens, re-signing read 102,364 cells against 0. + +**Detecting failed chunks.** Capturing stderr caught the gdalcubes chunk-error line in 1 of 4 configurations (never at parallel > 1), so it was not shipped as a guard. + +**Floodplain-wide BULK** (`tile_size = 20000`, 30 tiles), in order: + +1. Run 1: HTTP 413 in 9 s. +2. Run 2: 54 min, 15 tiles lost, then the COG write failed. The first diagnosis, that the write failed because of the failed tiles, was **retracted**: run 4 failed the same way with every tile read. +3. Run 3: killed at a 2-h cap after 16 tiles (5.4-11.9 min each), no chunk failures. +4. Run 4: all 30 tiles in 3 h 16 min, peak RSS 3.90 GiB, then `terra::mask` "cannot read from" the merged mosaic's temp file. + +Ruled out for run 4 (#88): + +- disk space (1 TB free); +- a floodplain-size COG write (82 s, 459 MB); +- terra deleting the temp file on GC; +- merging and masking 30 synthetic NetCDF tiles on the same grid with the real polygon, which took 0.7 min. + +The live run is not reproduced. + +## Evidence + +- `data-raw/logs/benchmark_composite_bulk/*`: chips per-chip times, RSS and wall clock for the chips run and the last floodplain run. Earlier floodplain runs were overwritten by the wrapper; their numbers are above. +- `data-raw/benchmark_composite_bulk*.{R,sh}`: the benchmark. Its floodplain stage now inventories temp files on failure. +- `review-plan.md`, `review-79-round{1,2,3}.md`: the plan review and code-check rounds, each with its verdicts. + +Closed by: PR #86 diff --git a/planning/active/findings.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/findings.md similarity index 100% rename from planning/active/findings.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/findings.md diff --git a/planning/active/progress.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/progress.md similarity index 100% rename from planning/active/progress.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/progress.md diff --git a/planning/active/review-79-round1.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round1.md similarity index 100% rename from planning/active/review-79-round1.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round1.md diff --git a/planning/active/review-79-round2.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round2.md similarity index 100% rename from planning/active/review-79-round2.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round2.md diff --git a/planning/active/review-79-round3.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round3.md similarity index 100% rename from planning/active/review-79-round3.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/review-79-round3.md diff --git a/planning/active/review-plan.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/review-plan.md similarity index 100% rename from planning/active/review-plan.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/review-plan.md diff --git a/planning/active/task_plan.md b/planning/archive/2026-09-issue-79-dated-reference-imagery/task_plan.md similarity index 91% rename from planning/active/task_plan.md rename to planning/archive/2026-09-issue-79-dated-reference-imagery/task_plan.md index e05258d..8ce4c7a 100644 --- a/planning/active/task_plan.md +++ b/planning/archive/2026-09-issue-79-dated-reference-imagery/task_plan.md @@ -25,7 +25,7 @@ Plan-gate decisions (2026-09-28): - STAC query, `months` filter and offset split (`stac_cube_items()`); - extent → cube_view → pre/post `cover` → tiling/mosaic (`stac_cube_assemble()`), taking a pixel function. - [x] `dft_stac_cube()` calls these helpers and passes a `dft_index_expr()` closure as its pixel function. -- [ ] Guard: the frozen legacy key and every offline cube test pass unchanged. Run the network e2e (untiled + tiled) before and after, and record both. +- [x] Guard: the frozen legacy key and every offline cube test pass unchanged. Run the network e2e (untiled + tiled) before and after, and record both. ## Phase 3: `dft_stac_composite()` @@ -68,13 +68,13 @@ Plan-gate decisions (2026-09-28): - `dft_stac_composite()` on `bulk_co_ff04`, one year, `tile_size` set; - a chip run over ~100 scattered buffered points. Also note what `dft_map_interactive(rgb=)` does with a floodplain-size local raster (maxBytes), to confirm the doc points to the COG path. -- [ ] Update NEWS.md, the CLAUDE.md Core Pipeline (CLAUDE.md done, 2c30c3c) (the composite call and the chip pattern), and the gotchas note if anything turns up. -- [ ] Bump the version (minor) as the final commit. +- [x] Update NEWS.md (c528f00), the CLAUDE.md Core Pipeline (2c30c3c) (the composite call and the chip pattern), and the gotchas note (505218d). +- [ ] Bump the version (minor) as the final commit. *(Lands after this archive, as the branch's last commit.)* ## Validation -- [ ] Tests pass (`devtools::test()`, plus `DRIFT_TEST_NETWORK=true` e2e for cube and composite) -- [ ] `lintr::lint_package()` clean; `pkgdown::check_pkgdown()` passes -- [ ] `/code-check` clean on each commit -- [ ] PWF checkboxes match landed work -- [ ] `/planning-archive` on completion +- [x] Tests pass (`devtools::test()`, plus `DRIFT_TEST_NETWORK=true` e2e for cube and composite) +- [x] `lintr::lint_package()`: 0 lints on changed lines (193 pre-existing elsewhere, untouched); `pkgdown::check_pkgdown()` passes +- [x] `/code-check`: three rounds on the branch diff (not per commit), ended by R3's enumeration +- [x] PWF checkboxes match landed work +- [x] `/planning-archive` on completion From 0c3cdceffa116ad07a7760f22ad6b04a4d6a3342 Mon Sep 17 00:00:00 2001 From: almac2022 Date: Mon, 28 Sep 2026 15:49:40 -0700 Subject: [PATCH 29/29] Release v0.18.0 (#79) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PRhUJsuKABLfpBGktPoiBN --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 35718c9..03ecb95 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: drift Title: Detecting Riparian and Inland Floodplain Transitions -Version: 0.17.1 +Version: 0.18.0 Date: 2026-09-06 Authors@R: c( person("Allan", "Irvine", , "al@newgraphenvironment.com", role = c("aut", "cre"),