Cpp data loader

.Rbuildignore

@@ -27,3 +27,4 @@ tests/testthat/Rplots.pdf
 ^CLAUDE\.md$
 ^AUDIT\.md$
 ^VISIBILITY\.md$
+^src/Makevars$

.github/workflows/R-CMD-check.yaml

@@ -31,6 +31,7 @@ jobs:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
       KMP_DUPLICATE_LIB_OK: TRUE
+      _R_CHECK_FORCE_SUGGESTS_: false
     steps:
       - uses: actions/checkout@v4
 
@@ -42,39 +43,25 @@ jobs:
 
       - uses: r-lib/actions/setup-pandoc@v2
 
-      - name: Query dependencies
-        run: |
-          install.packages("pak", repos = "https://r-lib.github.io/p/pak/dev/")
-          saveRDS(pak::pkg_deps("local::.", dependencies = TRUE), ".github/r-depends.rds")
-        shell: Rscript {0}
-
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v4
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ matrix.config.os }}-${{ steps.install-r.outputs.installed-r-version }}-1-${{ hashFiles('.github/r-depends.rds') }}
-          restore-keys: ${{ matrix.config.os }}-${{ steps.install-r.outputs.installed-r-version }}-1-
-
-      - name: Install curl
-        if: runner.os == 'Linux'
-        run: sudo apt-get install -y curl
-      - name: Install system dependencies
+      # libjpeg, libpng and libtiff are what the compiled image decoder links against;
+      # without them configure degrades that format to the R reader and the decoder is untested
+      - name: Install system dependencies (Linux)
         if: runner.os == 'Linux'
         run: |
-          pak::local_system_requirements(execute = TRUE)
-          pak::pkg_system_requirements("rcmdcheck", execute = TRUE)
-        shell: Rscript {0}
+          sudo apt-get update -y
+          sudo apt-get install -y libcurl4-openssl-dev libssl-dev libxml2-dev \
+                                  libjpeg-dev libpng-dev libtiff5-dev \
+                                  libfontconfig1-dev libharfbuzz-dev libfribidi-dev \
+                                  libfreetype6-dev libgit2-dev
 
-      - name: Install dependencies
-        run: |
-          pak::local_install_dev_deps(upgrade = TRUE)
-          pak::pkg_install("rcmdcheck")
-        shell: Rscript {0}
+      - name: Install system dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: brew install jpeg libpng libtiff pkg-config
 
-      - name: Install Pkg
-        run: |
-          Rscript -e "pak::local_install()"
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::rcmdcheck
+          needs: check
 
       - name: Check I
         env:

.gitignore

@@ -22,3 +22,6 @@ HISTORY.md
 INSIGHTS.md
 AUDIT.md
 VISIBILITY.md
+src/Makevars
+src/*.o
+src/*.so

DESCRIPTION

@@ -26,11 +26,13 @@ Description: The 'cito' package provides a user-friendly interface for training
 Encoding: UTF-8
 LazyData: true
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.3
 Depends:
     R (>= 4.1.0)
 Imports:
+    Rcpp,
     coro,
+    jpeg,
+    png,
     checkmate,
     torch,
     gridExtra,
@@ -55,10 +57,12 @@ Suggests:
     ggplot2,
     knitr,
     tiff,
-    jpeg,
-    png,
     raster
+LinkingTo:
+    Rcpp
+SystemRequirements: libjpeg, libpng, libtiff (all optional; without them cito falls back to the R image readers)
 VignetteBuilder: knitr
 BugReports: https://github.com/citoverse/cito/issues
 URL: https://citoverse.github.io/cito/
 Language: en-US
+Config/roxygen2/version: 8.1.0

NAMESPACE

@@ -62,3 +62,7 @@ export(transfer)
 export(tune)
 import(checkmate)
 import(tibble)
+importFrom(Rcpp,sourceCpp)
+importFrom(jpeg,readJPEG)
+importFrom(png,readPNG)
+useDynLib(cito, .registration = TRUE)

NEWS.md

@@ -65,10 +65,16 @@
 * conditionalEffects(data = ) works when the new data contains NAs. Those rows are dropped from the model frame, but the row indices were still drawn from the full data, which aborted the analytical method inside torch ("index out of bounds") and made the finite method fail on a length mismatch
 * the mvp likelihood returns one value per observation like every other likelihood, instead of a single reduced value
 * the development notes at the repository root (and E-CNN_and_MMN.md) are excluded from the build, they were shipped in the tarball before
-* multi-band TIFFs that store their bands as separate planes (GDAL's INTERLEAVE=BAND) are no longer read incorrectly. tiff::readTIFF() returns wrong pixel values for that layout without raising an error, so models were trained on corrupted inputs. Such files are now refused with an explanatory message instead of being read silently
 * images read from a folder are no longer scaled down by a factor of 255. base_loader() returns values in [0, 1] and cito divided by 255 on top of that, so JPEG and PNG inputs reached the network in [0, 1/255] while TIFF inputs, which were not divided, reached it in [0, 1]
-* folder inputs with .JPG, .PNG, .JPEG or .tif extensions are read instead of being rejected. The file type was matched case-sensitively, .tif was missing entirely, and the pattern was unanchored with an unescaped dot, so a file called "x.png.bak" was passed to the image reader while "photo.JPG" was not
+* multi-band TIFFs that store their bands as separate planes (GDAL's INTERLEAVE=BAND) are no longer read incorrectly. tiff::readTIFF() returns wrong pixel values for that layout without raising an error, so models were trained on corrupted inputs. cito now reads such files through its compiled decoder, and the fallback reader refuses them with an explanatory message instead of continuing silently
+* cloud-optimised GeoTIFFs can be read at all. A COG stores its overviews, and usually an internal mask, as further TIFF directories; cito passed every directory to readTIFF() and tried to stack images of different sizes and bit depths, which cannot succeed. Only full-resolution directories are used now
+* palette and white-is-zero TIFFs are read by the compiled decoder instead of falling back; a palette image is expanded through its colormap, and white-is-zero returns the stored samples, which is what GDAL and tiff::readTIFF both do
+* 32-bit float and signed 16-bit TIFFs are supported, which is what per-pixel embedding products and elevation models are distributed as. Their values pass through unscaled, since only unsigned integer samples carry an implied [0, 1] range
+* folder images are decoded by a compiled reader when libjpeg, libpng and libtiff are available at install time, which is 9 to 14 times faster than the previous path and never materialises an R double array. Without those libraries cito falls back to the R readers, one format at a time
 * GeoTIFF tags no longer raise four warnings per file
+* folder inputs with .JPG, .PNG, .JPEG or .tif extensions are read instead of being rejected. The file type was matched case-sensitively, .tif was missing entirely, and the pattern was unanchored with an unescaped dot, so a file called "x.png.bak" was passed to the image reader while "photo.JPG" was not
+* RGBA images keep their alpha channel on every decode path
+* cnn() gains num_workers for folder inputs. The dataset no longer stores tensors, which previously made torch's parallel dataloader fail with "external pointer is not valid"
 * weighted cross-validation in the hyperparameter tuning subsets matrix weights by row
 * print() of a citoarchitecture without an input_shape no longer errors
 * models fitted with cito <= 1.1 are rejected with an informative message instead of an internal error

R/RcppExports.R

@@ -0,0 +1,11 @@
+# Generated by using Rcpp::compileAttributes() -> do not edit by hand
+# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393
+
+cito_decoder_available <- function() {
+    .Call(`_cito_cito_decoder_available`)
+}
+
+cito_decode_batch <- function(paths, nthreads = 1L) {
+    .Call(`_cito_cito_decode_batch`, paths, nthreads)
+}
+

R/cito.R

@@ -29,6 +29,10 @@
 #'
 #' @example /inst/examples/dnn-example.R
 #'
+#' @importFrom jpeg readJPEG
+#' @importFrom png readPNG
+#' @useDynLib cito, .registration = TRUE
+#' @importFrom Rcpp sourceCpp
 #' @aliases cito-package
 #' @name cito
 "_PACKAGE"

R/cnn.R

@@ -16,6 +16,8 @@
 #' @param validation Proportion of the data to be used for validation. Alternatively, a vector containing the indices of the validation samples can be provided. Default is 0.0.
 #' @param batchsize Batch size for training. If NULL, batchsize is 10% of the training data. Default is NULL.
 #' @param shuffle Whether to shuffle the data before each epoch. Default is TRUE.
+#' @param num_workers (integer) Number of worker processes for the dataloader. Only effective for folder inputs. 0 decodes in the main process.
+#' @param num_threads (integer) Number of threads the compiled image decoder uses per batch. Only effective for folder inputs.
 #' @param data_augmentation A list of functions used for data augmentation. Elements must be either functions or strings corresponding to inbuilt data augmentation functions. See details for more information.
 #' @param epochs Number of epochs to train the model. Default is 100.
 #' @param weights weights or other values (can be also a matrix) that the likelihood has access to
@@ -108,6 +110,8 @@ cnn <- function(X,
                 batchsize = NULL,
                 shuffle = TRUE,
                 data_augmentation = NULL,
+                num_workers = 0L,
+                num_threads = 4L,
                 epochs = 100,
                 weights = NULL,
                 early_stopping = Inf,
@@ -139,6 +143,8 @@ cnn <- function(X,
   checkmate::qassert(baseloss, c("0", "N1", "B1"))
   checkmate::qassert(device, "S+[3,)")
   checkmate::qassert(plot, "B1")
+  checkmate::qassert(num_workers, "X1[0,)")
+  checkmate::qassert(num_threads, "X1[1,)")
   checkmate::qassert(verbose, "B1")
 
   # Only return the model properties if no Y specified (Used in mmn())
@@ -223,7 +229,8 @@ cnn <- function(X,
   weights_torch <- if(is.null(weights)) NULL else torch::torch_tensor(weights)
   dls <- build_loaders(list(X), Y, weights = weights_torch, valid = valid,
                        batch_size = batchsize, shuffle = shuffle,
-                       from_folder = from_folder, data_augmentation = data_augmentation)
+                       from_folder = from_folder, data_augmentation = data_augmentation,
+                       num_workers = num_workers, num_threads = num_threads)
   train_dl <- dls$train_dl
   valid_dl <- dls$valid_dl
 
@@ -249,6 +256,8 @@ cnn <- function(X,
                               batchsize = batchsize,
                               shuffle = shuffle,
                               data_augmentation = data_augmentation,
+                              num_workers = num_workers,
+                              num_threads = num_threads,
                               epochs = epochs, #redundant?
                               early_stopping = early_stopping,
                               burnin = burnin,
@@ -345,7 +354,9 @@ predict.citocnn <- function(object,
     stop(paste0("Wrong dimension of newdata: [", paste(dim(newdata), collapse = ", "), "]   Correct input dimension: [", paste(c("N", dim(object$data$X)[-1]), collapse = ", "), "]"))
   }
 
-  dl <- get_data_loader(newdata, batch_size = batchsize, shuffle = FALSE, from_folder = from_folder)
+  nt <- if(is.null(object$training_properties$num_threads)) 4L else object$training_properties$num_threads
+  dl <- get_data_loader(newdata, batch_size = batchsize, shuffle = FALSE, from_folder = from_folder,
+                        num_threads = nt)
 
   pred <- NULL
   coro::loop(for(b in dl) {

R/continue_training.R

@@ -269,8 +269,10 @@ continue_training.citocnn <- function(model,
     }
   }
 
+  nw <- if(is.null(model$training_properties$num_workers)) 0L else model$training_properties$num_workers
+  nt <- if(is.null(model$training_properties$num_threads)) 4L else model$training_properties$num_threads
   dls <- build_loaders(list(X), Y, weights = weights, valid = valid, batch_size = bs, shuffle = sh,
-                       from_folder = from_folder)
+                       from_folder = from_folder, num_workers = nw, num_threads = nt)
   train_dl <- dls$train_dl
   valid_dl <- dls$valid_dl
 
@@ -348,8 +350,11 @@ continue_training.citommn <- function(model,
     }
   }
 
+  nw <- if(is.null(model$training_properties$num_workers)) 0L else model$training_properties$num_workers
+  nt <- if(is.null(model$training_properties$num_threads)) 4L else model$training_properties$num_threads
   dls <- build_loaders(X, Y, valid = valid, batch_size = model$training_properties$batchsize,
-                       shuffle = model$training_properties$shuffle, from_folder = from_folder)
+                       shuffle = model$training_properties$shuffle, from_folder = from_folder,
+                       num_workers = nw, num_threads = nt)
   train_dl <- dls$train_dl
   valid_dl <- dls$valid_dl
 

R/mmn.R

@@ -302,7 +302,9 @@ predict.citommn <- function(object,
     return(x)
   })
 
-  dl <- do.call(get_data_loader, append(newdata, list(batch_size = batchsize, shuffle = FALSE, from_folder=from_folder)))
+  nt <- if(is.null(object$training_properties$num_threads)) 4L else object$training_properties$num_threads
+  dl <- do.call(get_data_loader, append(newdata, list(batch_size = batchsize, shuffle = FALSE,
+                                                      from_folder = from_folder, num_threads = nt)))
 
   pred <- NULL
   coro::loop(for(b in dl) {