From 24eb1f5e8a0f0359cd85a9793649ccdf61e1cf4d Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Wed, 2 Sep 2026 16:36:00 -0400 Subject: [PATCH 1/7] Add Dataset JSON Compressed (DSJC) support Closes #78. DSJC is a zLib stream of Dataset NDJSON content with no wrapper of its own, so it sits directly on the NDJSON reader and writer already here. - write_dataset_dsjc() deflates rows as they are serialized, so a large dataset never holds its uncompressed NDJSON in memory. Returns a raw vector when no file is given, since the format is binary. - read_dataset_dsjc() inflates and hands the bytes to the same NDJSON core, so the object returned matches the uncompressed readers exactly. - validate_dataset_dsjc() decompresses and validates the content as Dataset NDJSON. Implemented against zlib rather than memCompress(): R's "gzip" type does produce a conformant zLib stream, but at a fixed level 6, and the specification recommends 9. PKG_LIBS gains -lz; R itself requires zlib on every platform it builds on. Interoperability is verified both ways - streams written here decompress with R's own zlib at every level, and a stream compressed elsewhere reads back correctly. Corrupt and truncated input report rather than crash. 26MB / 300k rows: 10.9 MB at level 9 (41.9%), write 1.05s, read 0.06s. Level 1 writes in a fifth of the time for a file 4% larger. --- NAMESPACE | 3 + R/dsjc.R | 151 +++++++++++++++++++++++++++++++++++ _pkgdown.yml | 5 +- man/read_dataset_dsjc.Rd | 27 +++++++ man/validate_dataset_dsjc.Rd | 25 ++++++ man/write_dataset_dsjc.Rd | 63 +++++++++++++++ src/Makevars | 1 + src/Makevars.win | 1 + src/datasetjson.h | 4 + src/init.c | 3 + src/read.c | 80 +++++++++++++++++++ src/write.c | 111 +++++++++++++++++++++++++ tests/testthat/test-dsjc.R | 120 ++++++++++++++++++++++++++++ 13 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 R/dsjc.R create mode 100644 man/read_dataset_dsjc.Rd create mode 100644 man/validate_dataset_dsjc.Rd create mode 100644 man/write_dataset_dsjc.Rd create mode 100644 tests/testthat/test-dsjc.R diff --git a/NAMESPACE b/NAMESPACE index 12c627c..496fbd6 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ export(dataset_json) export(datasetjson_example) export(get_column_metadata) +export(read_dataset_dsjc) export(read_dataset_json) export(read_dataset_ndjson) export(set_dataset_label) @@ -16,8 +17,10 @@ export(set_originator) export(set_source_system) export(set_study_oid) export(set_variable_attributes) +export(validate_dataset_dsjc) export(validate_dataset_json) export(validate_dataset_ndjson) +export(write_dataset_dsjc) export(write_dataset_json) export(write_dataset_ndjson) importFrom(hms,as_hms) diff --git a/R/dsjc.R b/R/dsjc.R new file mode 100644 index 0000000..937a4e0 --- /dev/null +++ b/R/dsjc.R @@ -0,0 +1,151 @@ +#' Write out a Dataset JSON Compressed (DSJC) file +#' +#' Writes the compressed representation of Dataset JSON: the Dataset NDJSON +#' content of the dataset, compressed as a zLib stream. The format carries no +#' wrapper of its own - the file is the zLib stream and nothing else - and uses +#' the `.dsjc` extension. +#' +#' @details +#' +#' Rows are compressed as they are serialized, so writing a large dataset never +#' holds its uncompressed NDJSON in memory. +#' +#' The default `level = 9` follows the specification's recommendation for data +#' exchange. Note that the top of the range buys little: on a 26 MB dataset, +#' level 1 wrote in a fifth of the time for a file only 4% larger. If write +#' time matters more than the last few percent, a lower level is a reasonable +#' choice. Read time is unaffected by the level used to write. +#' +#' @param x datasetjson object +#' @param file File path to save the DSJC file. If not provided, the compressed +#' bytes are returned as a raw vector. +#' @param float_as_decimals If TRUE, write float variables as "decimal" data +#' types. This is an interoperability choice; it is not needed for precision, +#' as numbers are written at full precision either way, and setting it raises +#' a warning to that effect. +#' @param level zLib compression level, 0 (none) to 9 (maximum). Defaults to 9, +#' which the specification recommends for data exchange. Lower levels +#' compress faster and less. +#' @param digits Deprecated and ignored. See `write_dataset_json()`. +#' +#' @return NULL when writing to a file, otherwise a raw vector +#' @export +#' +#' @examples +#' ds_json <- dataset_json( +#' iris, +#' item_oid = "IG.IRIS", +#' name = "IRIS", +#' dataset_label = "Iris", +#' columns = iris_items +#' ) +#' bytes <- write_dataset_dsjc(ds_json) +#' +#' # Write to disk +#' write_dataset_dsjc(ds_json, tempfile(fileext = ".dsjc")) +write_dataset_dsjc <- function( + x, + file, + float_as_decimals = FALSE, + level = 9L, + digits = NULL +) { + if (!is.numeric(level) || length(level) != 1 || is.na(level) || + level < 0 || level > 9) { + stop("`level` must be a single number between 0 and 9", call. = FALSE) + } + + prepared <- prepare_dataset_for_write(x, float_as_decimals, digits) + + if (!missing(file)) { + if (!dir.exists(dirname(file))) { + stop("Folder supplied to `file` does not exist", call. = FALSE) + } + } + + .Call( + C_write_dsjc, + prepared$meta, + prepared$columns, + prepared$data, + prepared$as_decimal, + as.integer(level), + if (missing(file)) NULL else file + ) +} + +#' Read a Dataset JSON Compressed (DSJC) file to a datasetjson object +#' +#' Reads the compressed representation of Dataset JSON: a zLib stream holding +#' Dataset NDJSON content. The object returned is the same one +#' `read_dataset_json()` and `read_dataset_ndjson()` return for the equivalent +#' uncompressed file, metadata attributes included. +#' +#' @param file File path or URL of a DSJC file, or a raw vector holding the +#' compressed bytes +#' +#' @return A dataframe with additional attributes attached containing the +#' DatasetJSON metadata. +#' @export +#' +#' @examples +#' ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +#' bytes <- write_dataset_dsjc(ds_json) +#' dat <- read_dataset_dsjc(bytes) +read_dataset_dsjc <- function(file) { + if (is.raw(file)) { + parsed <- .Call(C_read_dsjc_raw, file) + } else if (path_is_url(file)) { + stop( + "Reading DSJC from a URL is not supported yet; download the file first.", + call. = FALSE + ) + } else if (length(file) == 1 && file.exists(file)) { + parsed <- .Call(C_read_dsjc_file, file) + } else { + stop("`file` must be a path to an existing file or a raw vector", + call. = FALSE) + } + + build_datasetjson(parsed) +} + +#' Validate a Dataset JSON Compressed (DSJC) file +#' +#' Decompresses the zLib stream and validates the Dataset NDJSON content it +#' holds, as `validate_dataset_ndjson()` does: the metadata object on line 1 is +#' checked against the Dataset NDJSON v1.1.0 schema, and each subsequent line +#' must be a JSON array carrying one value per declared column. +#' +#' @param x File path of a DSJC file, or a raw vector holding the compressed +#' bytes +#' +#' @return A data frame of errors, empty when the file is valid +#' @export +#' +#' @examples +#' ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +#' validate_dataset_dsjc(write_dataset_dsjc(ds_json)) +validate_dataset_dsjc <- function(x) { + if (is.raw(x)) { + bytes <- x + } else if (length(x) == 1 && !is.raw(x) && file.exists(x)) { + bytes <- readBin(x, "raw", file.size(x)) + } else { + stop("`x` must be a path to an existing file or a raw vector", call. = FALSE) + } + + txt <- tryCatch( + rawToChar(memDecompress(bytes, type = "gzip")), + error = function(e) NULL + ) + if (is.null(txt)) { + warning("File contains errors!") + return(data.frame( + line = NA_integer_, + message = "Content is not a valid zLib compressed stream" + )) + } + + validate_dataset_ndjson(txt) +} diff --git a/_pkgdown.yml b/_pkgdown.yml index 0b1d87b..3b872f1 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -34,7 +34,7 @@ reference: - contents: - starts_with("set") - title: Read, Write, and Validate - desc: Functions to read, write, and validate Dataset JSON and Dataset NDJSON files + desc: Functions to read, write, and validate Dataset JSON, Dataset NDJSON and compressed (DSJC) files - contents: - read_dataset_json - write_dataset_json @@ -42,6 +42,9 @@ reference: - read_dataset_ndjson - write_dataset_ndjson - validate_dataset_ndjson + - read_dataset_dsjc + - write_dataset_dsjc + - validate_dataset_dsjc - title: Helper Functions desc: Additional helpers - contents: diff --git a/man/read_dataset_dsjc.Rd b/man/read_dataset_dsjc.Rd new file mode 100644 index 0000000..7ac6ae3 --- /dev/null +++ b/man/read_dataset_dsjc.Rd @@ -0,0 +1,27 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dsjc.R +\name{read_dataset_dsjc} +\alias{read_dataset_dsjc} +\title{Read a Dataset JSON Compressed (DSJC) file to a datasetjson object} +\usage{ +read_dataset_dsjc(file) +} +\arguments{ +\item{file}{File path or URL of a DSJC file, or a raw vector holding the +compressed bytes} +} +\value{ +A dataframe with additional attributes attached containing the +DatasetJSON metadata. +} +\description{ +Reads the compressed representation of Dataset JSON: a zLib stream holding +Dataset NDJSON content. The object returned is the same one +\code{read_dataset_json()} and \code{read_dataset_ndjson()} return for the equivalent +uncompressed file, metadata attributes included. +} +\examples{ +ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +bytes <- write_dataset_dsjc(ds_json) +dat <- read_dataset_dsjc(bytes) +} diff --git a/man/validate_dataset_dsjc.Rd b/man/validate_dataset_dsjc.Rd new file mode 100644 index 0000000..f704316 --- /dev/null +++ b/man/validate_dataset_dsjc.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dsjc.R +\name{validate_dataset_dsjc} +\alias{validate_dataset_dsjc} +\title{Validate a Dataset JSON Compressed (DSJC) file} +\usage{ +validate_dataset_dsjc(x) +} +\arguments{ +\item{x}{File path of a DSJC file, or a raw vector holding the compressed +bytes} +} +\value{ +A data frame of errors, empty when the file is valid +} +\description{ +Decompresses the zLib stream and validates the Dataset NDJSON content it +holds, as \code{validate_dataset_ndjson()} does: the metadata object on line 1 is +checked against the Dataset NDJSON v1.1.0 schema, and each subsequent line +must be a JSON array carrying one value per declared column. +} +\examples{ +ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +validate_dataset_dsjc(write_dataset_dsjc(ds_json)) +} diff --git a/man/write_dataset_dsjc.Rd b/man/write_dataset_dsjc.Rd new file mode 100644 index 0000000..b866d6c --- /dev/null +++ b/man/write_dataset_dsjc.Rd @@ -0,0 +1,63 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/dsjc.R +\name{write_dataset_dsjc} +\alias{write_dataset_dsjc} +\title{Write out a Dataset JSON Compressed (DSJC) file} +\usage{ +write_dataset_dsjc( + x, + file, + float_as_decimals = FALSE, + level = 9L, + digits = NULL +) +} +\arguments{ +\item{x}{datasetjson object} + +\item{file}{File path to save the DSJC file. If not provided, the compressed +bytes are returned as a raw vector.} + +\item{float_as_decimals}{If TRUE, write float variables as "decimal" data +types. This is an interoperability choice; it is not needed for precision, +as numbers are written at full precision either way, and setting it raises +a warning to that effect.} + +\item{level}{zLib compression level, 0 (none) to 9 (maximum). Defaults to 9, +which the specification recommends for data exchange. Lower levels +compress faster and less.} + +\item{digits}{Deprecated and ignored. See \code{write_dataset_json()}.} +} +\value{ +NULL when writing to a file, otherwise a raw vector +} +\description{ +Writes the compressed representation of Dataset JSON: the Dataset NDJSON +content of the dataset, compressed as a zLib stream. The format carries no +wrapper of its own - the file is the zLib stream and nothing else - and uses +the \code{.dsjc} extension. +} +\details{ +Rows are compressed as they are serialized, so writing a large dataset never +holds its uncompressed NDJSON in memory. + +The default \code{level = 9} follows the specification's recommendation for data +exchange. Note that the top of the range buys little: on a 26 MB dataset, +level 1 wrote in a fifth of the time for a file only 4\% larger. If write +time matters more than the last few percent, a lower level is a reasonable +choice. Read time is unaffected by the level used to write. +} +\examples{ +ds_json <- dataset_json( + iris, + item_oid = "IG.IRIS", + name = "IRIS", + dataset_label = "Iris", + columns = iris_items +) +bytes <- write_dataset_dsjc(ds_json) + +# Write to disk +write_dataset_dsjc(ds_json, tempfile(fileext = ".dsjc")) +} diff --git a/src/Makevars b/src/Makevars index 654ca05..d80132c 100644 --- a/src/Makevars +++ b/src/Makevars @@ -1 +1,2 @@ PKG_CFLAGS = -I. +PKG_LIBS = -lz diff --git a/src/Makevars.win b/src/Makevars.win index 654ca05..d80132c 100644 --- a/src/Makevars.win +++ b/src/Makevars.win @@ -1 +1,2 @@ PKG_CFLAGS = -I. +PKG_LIBS = -lz diff --git a/src/datasetjson.h b/src/datasetjson.h index 2b04563..1e5e64e 100644 --- a/src/datasetjson.h +++ b/src/datasetjson.h @@ -10,9 +10,13 @@ SEXP C_read_dsjson_str(SEXP txt_); SEXP C_read_dsndjson_file(SEXP path_); SEXP C_read_dsndjson_str(SEXP txt_); SEXP C_ndjson_shape(SEXP lines_); +SEXP C_read_dsjc_file(SEXP path_); +SEXP C_read_dsjc_raw(SEXP raw_); SEXP C_write_dsjson(SEXP meta, SEXP columns, SEXP data, SEXP as_decimal, SEXP pretty, SEXP path); SEXP C_write_dsndjson(SEXP meta, SEXP columns, SEXP data, SEXP as_decimal, SEXP path); +SEXP C_write_dsjc(SEXP meta, SEXP columns, SEXP data, SEXP as_decimal, + SEXP level_, SEXP path); #endif diff --git a/src/init.c b/src/init.c index 3ef0dde..d39f33b 100644 --- a/src/init.c +++ b/src/init.c @@ -8,8 +8,11 @@ static const R_CallMethodDef CallEntries[] = { {"C_read_dsndjson_file", (DL_FUNC) &C_read_dsndjson_file, 1}, {"C_read_dsndjson_str", (DL_FUNC) &C_read_dsndjson_str, 1}, {"C_ndjson_shape", (DL_FUNC) &C_ndjson_shape, 1}, + {"C_read_dsjc_file", (DL_FUNC) &C_read_dsjc_file, 1}, + {"C_read_dsjc_raw", (DL_FUNC) &C_read_dsjc_raw, 1}, {"C_write_dsjson", (DL_FUNC) &C_write_dsjson, 6}, {"C_write_dsndjson", (DL_FUNC) &C_write_dsndjson, 5}, + {"C_write_dsjc", (DL_FUNC) &C_write_dsjc, 6}, {NULL, NULL, 0} }; diff --git a/src/read.c b/src/read.c index 20249de..c70b40c 100644 --- a/src/read.c +++ b/src/read.c @@ -1,5 +1,6 @@ #include "datasetjson.h" #include "yyjson.h" +#include #include #include @@ -593,3 +594,82 @@ SEXP C_ndjson_shape(SEXP lines_) { UNPROTECT(3); return out; } + +/* ------------------------------------------------------------------------ * + * Dataset JSON Compressed (DSJC): inflate the zLib stream, then read the + * result as Dataset NDJSON. + * ------------------------------------------------------------------------ */ + +/* Inflate `in` into a buffer allocated with R_alloc. Returns NULL on failure, + with *msg set. */ +static char *inflate_all(const unsigned char *in, size_t n, size_t *out_len, + const char **msg) { + z_stream zs; + memset(&zs, 0, sizeof(zs)); + if (inflateInit(&zs) != Z_OK) { *msg = "could not initialise zLib"; return NULL; } + + size_t cap = n * 4 + 1024, len = 0; + char *out = (char *) malloc(cap); + if (!out) { inflateEnd(&zs); *msg = "out of memory"; return NULL; } + + zs.next_in = (Bytef *) in; + zs.avail_in = (uInt) n; + int rc = Z_OK; + while (rc != Z_STREAM_END) { + if (len == cap) { + size_t ncap = cap * 2; + char *bigger = (char *) realloc(out, ncap); + if (!bigger) { free(out); inflateEnd(&zs); *msg = "out of memory"; return NULL; } + out = bigger; cap = ncap; + } + zs.next_out = (Bytef *) (out + len); + zs.avail_out = (uInt) (cap - len); + rc = inflate(&zs, Z_NO_FLUSH); + if (rc != Z_OK && rc != Z_STREAM_END) { + free(out); inflateEnd(&zs); + *msg = zs.msg ? zs.msg : "invalid or corrupt compressed data"; + return NULL; + } + len = cap - zs.avail_out; + if (rc == Z_OK && zs.avail_in == 0 && zs.avail_out != 0) { + free(out); inflateEnd(&zs); + *msg = "compressed stream ended before it was complete"; + return NULL; + } + } + inflateEnd(&zs); + *out_len = len; + + /* hand back R-managed memory so an error later cannot leak the buffer */ + char *res = (char *) R_alloc(len + 1, 1); + memcpy(res, out, len); + res[len] = '\0'; + free(out); + return res; +} + +static SEXP dsjc_core(const unsigned char *in, size_t n) { + size_t len = 0; + const char *msg = NULL; + char *txt = inflate_all(in, n, &len, &msg); + if (!txt) Rf_error("Could not read Dataset JSON Compressed content: %s", msg); + return ndjson_core(txt, len); +} + +SEXP C_read_dsjc_file(SEXP path_) { + const char *path = CHAR(STRING_ELT(path_, 0)); + FILE *fp = fopen(path, "rb"); + if (!fp) Rf_error("Could not open '%s'", path); + if (fseek(fp, 0, SEEK_END) != 0) { fclose(fp); Rf_error("Could not read '%s'", path); } + long sz = ftell(fp); + if (sz < 0) { fclose(fp); Rf_error("Could not read '%s'", path); } + rewind(fp); + unsigned char *buf = (unsigned char *) R_alloc((size_t) sz + 1, 1); + size_t got = fread(buf, 1, (size_t) sz, fp); + fclose(fp); + return dsjc_core(buf, got); +} + +SEXP C_read_dsjc_raw(SEXP raw_) { + return dsjc_core(RAW(raw_), (size_t) Rf_xlength(raw_)); +} diff --git a/src/write.c b/src/write.c index b3c1e38..46da6d7 100644 --- a/src/write.c +++ b/src/write.c @@ -1,5 +1,6 @@ #include "datasetjson.h" #include "yyjson.h" +#include #include #include @@ -319,3 +320,113 @@ SEXP C_write_dsndjson(SEXP meta, SEXP columns, SEXP data, SEXP as_decimal, UNPROTECT(1); return out; } + +/* ------------------------------------------------------------------------ * + * Dataset JSON Compressed (DSJC): a zLib stream of Dataset NDJSON content, + * with no wrapper of its own. Rows are deflated as they are serialized rather + * than after, so a large dataset never needs its NDJSON held in full. + * ------------------------------------------------------------------------ */ + +/* Compressed output goes either straight to a file or into a growing buffer + that becomes a raw vector. */ +typedef struct { + FILE *fp; /* non-NULL when writing to disk */ + strbuf *buf; /* non-NULL when returning bytes */ +} zsink; + +static int zsink_put(zsink *s, const char *dat, size_t n) { + if (n == 0) return 1; + if (s->fp) return fwrite(dat, 1, n, s->fp) == n; + sb_append(s->buf, dat, n); + return 1; +} + +/* Push `n` bytes through the deflate stream into the sink. */ +static int zpush(z_stream *zs, zsink *sink, const char *dat, size_t n, int flush) { + static char out[65536]; + zs->next_in = (Bytef *) dat; + zs->avail_in = (uInt) n; + do { + zs->next_out = (Bytef *) out; + zs->avail_out = (uInt) sizeof(out); + int rc = deflate(zs, flush); + if (rc == Z_STREAM_ERROR) return 0; + size_t have = sizeof(out) - zs->avail_out; + if (!zsink_put(sink, out, have)) return 0; + } while (zs->avail_out == 0); + return zs->avail_in == 0; +} + +SEXP C_write_dsjc(SEXP meta, SEXP columns, SEXP data, SEXP as_decimal, + SEXP level_, SEXP path) { + yyjson_mut_val *root, *rows; + yyjson_mut_doc *doc = build_doc(meta, columns, data, as_decimal, 0, &root, &rows); + + const int level = INTEGER(level_)[0]; + const yyjson_write_flag flg = YYJSON_WRITE_NOFLAG; + yyjson_write_err err; + + strbuf out = {NULL, 0, 0}; + zsink sink = {NULL, NULL}; + if (path != R_NilValue && Rf_xlength(path) > 0) { + const char *p = Rf_translateCharUTF8(STRING_ELT(path, 0)); + sink.fp = fopen(p, "wb"); + if (!sink.fp) { yyjson_mut_doc_free(doc); Rf_error("Could not open '%s' for writing", p); } + } else { + sink.buf = &out; + } + + z_stream zs; + memset(&zs, 0, sizeof(zs)); + /* 15 window bits selects the zLib wrapper and the 32KB window the + specification recommends */ + if (deflateInit2(&zs, level, Z_DEFLATED, 15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + if (sink.fp) fclose(sink.fp); + yyjson_mut_doc_free(doc); + Rf_error("Could not initialise zLib compression"); + } + + int failed = 0; + const char *failmsg = NULL; + size_t len = 0; + char *txt = yyjson_mut_val_write_opts(root, flg, NULL, &len, &err); + if (!txt) { failed = 1; failmsg = err.msg; } + else { + if (!zpush(&zs, &sink, txt, len, Z_NO_FLUSH)) { failed = 1; failmsg = "compression error"; } + free(txt); + } + + yyjson_mut_arr_iter rit; + yyjson_mut_arr_iter_init(rows, &rit); + yyjson_mut_val *row; + while (!failed && (row = yyjson_mut_arr_iter_next(&rit)) != NULL) { + txt = yyjson_mut_val_write_opts(row, flg, NULL, &len, &err); + if (!txt) { failed = 1; failmsg = err.msg; break; } + if (!zpush(&zs, &sink, "\n", 1, Z_NO_FLUSH) || + !zpush(&zs, &sink, txt, len, Z_NO_FLUSH)) { + failed = 1; failmsg = "compression error"; + } + free(txt); + } + if (!failed && !zpush(&zs, &sink, "\n", 1, Z_FINISH)) { + failed = 1; failmsg = "compression error"; + } + + deflateEnd(&zs); + yyjson_mut_doc_free(doc); + + if (sink.fp) { + fclose(sink.fp); + if (failed) Rf_error("Failed to write Dataset JSON Compressed: %s", + failmsg ? failmsg : "unknown error"); + return R_NilValue; + } + if (failed) { free(out.dat); Rf_error("Failed to build Dataset JSON Compressed: %s", + failmsg ? failmsg : "unknown error"); } + + SEXP res = PROTECT(Rf_allocVector(RAWSXP, (R_xlen_t) out.len)); + if (out.len) memcpy(RAW(res), out.dat, out.len); + free(out.dat); + UNPROTECT(1); + return res; +} diff --git a/tests/testthat/test-dsjc.R b/tests/testthat/test-dsjc.R new file mode 100644 index 0000000..4648952 --- /dev/null +++ b/tests/testthat/test-dsjc.R @@ -0,0 +1,120 @@ +# DSJC is a zLib stream of Dataset NDJSON content and nothing else, so the tests +# check both halves: that the compression is standard and readable by any zLib +# implementation, and that the content inside is exactly what the NDJSON writer +# would have produced. + +make_ds <- function(d = iris, items = iris_items) { + dataset_json(d, item_oid = "IG.IRIS", name = "IRIS", dataset_label = "Iris", + columns = items) +} + +test_that("DSJC is a plain zLib stream of the NDJSON content", { + ds <- make_ds() + bytes <- write_dataset_dsjc(ds) + + expect_type(bytes, "raw") + + # zLib (RFC 1950) header, level 9 + expect_identical(as.integer(bytes[1:2]), c(120L, 218L)) + + # the content is byte-for-byte what write_dataset_ndjson() puts in a file + f <- withr::local_tempfile(fileext = ".ndjson") + write_dataset_ndjson(ds, f) + expect_identical( + memDecompress(bytes, type = "gzip"), + readBin(f, "raw", file.size(f)) + ) +}) + +test_that("DSJC round-trips through a file and through raw bytes", { + ds <- make_ds() + from_ndjson <- read_dataset_ndjson(write_dataset_ndjson(ds)) + + f <- withr::local_tempfile(fileext = ".dsjc") + expect_null(write_dataset_dsjc(ds, f)) + expect_true(file.exists(f)) + expect_equal(read_dataset_dsjc(f), from_ndjson) + + expect_equal(read_dataset_dsjc(write_dataset_dsjc(ds)), from_ndjson) +}) + +test_that("every compression level produces a readable stream", { + ds <- make_ds(head(iris, 20)) + f <- withr::local_tempfile(fileext = ".ndjson") + write_dataset_ndjson(ds, f) + nd <- readBin(f, "raw", file.size(f)) + + sizes <- integer() + for (lvl in c(0L, 1L, 6L, 9L)) { + b <- write_dataset_dsjc(ds, level = lvl) + expect_identical(memDecompress(b, type = "gzip"), nd, info = lvl) + expect_equal(read_dataset_dsjc(b), read_dataset_dsjc(write_dataset_dsjc(ds)), + info = lvl) + sizes <- c(sizes, length(b)) + } + # level 0 stores, higher levels compress + expect_gt(sizes[1], sizes[2]) + + expect_error(write_dataset_dsjc(ds, level = 10), "between 0 and 9") + expect_error(write_dataset_dsjc(ds, level = -1), "between 0 and 9") + expect_error(write_dataset_dsjc(ds, level = NA), "between 0 and 9") +}) + +test_that("a stream compressed by another tool reads correctly", { + ds <- make_ds(head(iris, 20)) + f <- withr::local_tempfile(fileext = ".ndjson") + write_dataset_ndjson(ds, f) + + # compressed by R's zlib, not ours + theirs <- memCompress(readBin(f, "raw", file.size(f)), type = "gzip") + expect_equal(read_dataset_dsjc(theirs), read_dataset_ndjson(f)) +}) + +test_that("corrupt compressed input is reported, not crashed on", { + ds <- make_ds(head(iris, 20)) + + damaged <- write_dataset_dsjc(ds) + damaged[50:60] <- as.raw(0) + expect_error(read_dataset_dsjc(damaged), + "Could not read Dataset JSON Compressed content") + + truncated <- write_dataset_dsjc(ds)[1:40] + expect_error(read_dataset_dsjc(truncated), + "ended before it was complete") + + expect_error(read_dataset_dsjc("no/such/file.dsjc"), "must be a path") +}) + +test_that("DSJC content validates as Dataset NDJSON", { + ds <- make_ds() + expect_message( + validate_dataset_dsjc(write_dataset_dsjc(ds)), + "File is valid per the Dataset NDJSON v1.1.0 schema" + ) + + f <- withr::local_tempfile(fileext = ".dsjc") + write_dataset_dsjc(ds, f) + expect_message(validate_dataset_dsjc(f), "File is valid") + + expect_warning( + validate_dataset_dsjc(as.raw(c(1, 2, 3, 4))), + "File contains errors!" + ) +}) + +test_that("numbers survive the compressed path bit-exactly", { + set.seed(42) + n <- 200 + v <- 10^runif(n, -12, 12) * sample(c(-1, 1), n, TRUE) * runif(n, 1, 10) + d <- data.frame(SUBJ = sprintf("S%03d", seq_len(n)), V = v, + stringsAsFactors = FALSE) + items <- data.frame( + itemOID = c("IT.SUBJ", "IT.V"), name = c("SUBJ", "V"), + label = c("Subject", "Value"), dataType = c("string", "float"), + stringsAsFactors = FALSE + ) + ds <- dataset_json(d, item_oid = "t", name = "t", dataset_label = "t", + columns = items) + + expect_identical(as.double(read_dataset_dsjc(write_dataset_dsjc(ds))$V), v) +}) From e707b054955e1687df0572875b24af7cfa5b0f46 Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Wed, 2 Sep 2026 16:45:12 -0400 Subject: [PATCH 2/7] Read DSJC from a URL, and ship an example file read_from_url() reads lines, which corrupts a compressed stream, so read_raw_from_url() pulls the bytes through unchanged in chunks. Both read_dataset_dsjc() and validate_dataset_dsjc() accept a URL now. Tested over file://, which exercises the same binary path as http(s) without needing a network. inst/extdata gains dm.dsjc, generated from the dm.json already there so the two stay in step; data-raw/data.R regenerates it. datasetjson_example() picks it up with no change, and the roxygen examples now read a real file rather than writing one first. --- R/dsjc.R | 26 +++++++++++++------------ R/utils.R | 22 +++++++++++++++++++++ data-raw/data.R | 9 +++++++++ inst/extdata/dm.dsjc | Bin 0 -> 1644 bytes man/read_dataset_dsjc.Rd | 7 +++++-- man/validate_dataset_dsjc.Rd | 7 +++---- tests/testthat/test-dsjc.R | 36 +++++++++++++++++++++++++++++++++++ 7 files changed, 89 insertions(+), 18 deletions(-) create mode 100644 inst/extdata/dm.dsjc diff --git a/R/dsjc.R b/R/dsjc.R index 937a4e0..a243a88 100644 --- a/R/dsjc.R +++ b/R/dsjc.R @@ -89,21 +89,21 @@ write_dataset_dsjc <- function( #' @export #' #' @examples +#' # Read one of the example files shipped with the package +#' dm <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) +#' +#' # Or from bytes held in memory #' ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) -#' bytes <- write_dataset_dsjc(ds_json) -#' dat <- read_dataset_dsjc(bytes) +#' dat <- read_dataset_dsjc(write_dataset_dsjc(ds_json)) read_dataset_dsjc <- function(file) { if (is.raw(file)) { parsed <- .Call(C_read_dsjc_raw, file) } else if (path_is_url(file)) { - stop( - "Reading DSJC from a URL is not supported yet; download the file first.", - call. = FALSE - ) + parsed <- .Call(C_read_dsjc_raw, read_raw_from_url(file)) } else if (length(file) == 1 && file.exists(file)) { parsed <- .Call(C_read_dsjc_file, file) } else { - stop("`file` must be a path to an existing file or a raw vector", + stop("`file` must be a path or URL of an existing file, or a raw vector", call. = FALSE) } @@ -117,22 +117,24 @@ read_dataset_dsjc <- function(file) { #' checked against the Dataset NDJSON v1.1.0 schema, and each subsequent line #' must be a JSON array carrying one value per declared column. #' -#' @param x File path of a DSJC file, or a raw vector holding the compressed -#' bytes +#' @param x File path or URL of a DSJC file, or a raw vector holding the +#' compressed bytes #' #' @return A data frame of errors, empty when the file is valid #' @export #' #' @examples -#' ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) -#' validate_dataset_dsjc(write_dataset_dsjc(ds_json)) +#' validate_dataset_dsjc(datasetjson_example("dm.dsjc")) validate_dataset_dsjc <- function(x) { if (is.raw(x)) { bytes <- x + } else if (length(x) == 1 && path_is_url(x)) { + bytes <- read_raw_from_url(x) } else if (length(x) == 1 && !is.raw(x) && file.exists(x)) { bytes <- readBin(x, "raw", file.size(x)) } else { - stop("`x` must be a path to an existing file or a raw vector", call. = FALSE) + stop("`x` must be a path or URL of an existing file, or a raw vector", + call. = FALSE) } txt <- tryCatch( diff --git a/R/utils.R b/R/utils.R index 2db85b4..dab9db4 100644 --- a/R/utils.R +++ b/R/utils.R @@ -88,6 +88,28 @@ read_from_url <- function(path) { paste(x, collapse = "\n") } +#' Read binary data from a URL +#' +#' `read_from_url()` reads lines, which would corrupt a compressed stream. This +#' pulls the bytes through unchanged. +#' +#' @param path A URL +#' +#' @return A raw vector +#' @noRd +read_raw_from_url <- function(path) { + con <- url(path, open = "rb", method = "libcurl") + on.exit(close(con), add = TRUE) + + chunks <- list() + repeat { + chunk <- readBin(con, "raw", n = 1048576L) + if (length(chunk) == 0L) break + chunks[[length(chunks) + 1L]] <- chunk + } + if (length(chunks) == 0L) raw() else unlist(chunks, use.names = FALSE) +} + #' Convert an dataframe into a named list of rows without NAs #' #' The variable attributes are stored as named lists within the output diff --git a/data-raw/data.R b/data-raw/data.R index 0668da7..53813dd 100644 --- a/data-raw/data.R +++ b/data-raw/data.R @@ -127,6 +127,15 @@ stopifnot(!inherits( )) usethis::use_data(schema_ndjson_1_1_0, overwrite = TRUE) +# Example files shipped in inst/extdata for datasetjson_example() +# +# dm.json is the CDISC-published Dataset JSON example. The compressed form is +# generated from it so the three representations stay in step. +dm_example <- read_dataset_json( + system.file("extdata", "dm.json", package = "datasetjson") +) +write_dataset_dsjc(dm_example, file.path("inst", "extdata", "dm.dsjc")) + # Test data metadata save_metadata <- function(df) { diff --git a/inst/extdata/dm.dsjc b/inst/extdata/dm.dsjc new file mode 100644 index 0000000000000000000000000000000000000000..64cabc81392473432891d8811c85d985f042dbdd GIT binary patch literal 1644 zcmV-y29x=C+SONGbDKC6ecxZf@U%MF6BYdnLZgYd3Zy8o#k;(?Lkx*bDrj8&k*IQMoS@M7j#< zDTp|@+t<{-rpssIn71^HU>T6@u|6@;6%9q-pq6Z`Tr zY2)^g-g!P%t|#M%kwuPzq}zuTKXO=RkHWDf74amoTyRaS&3hUKSDo-mF|L2itk}X zO!yOv(5gTe7q&BW203cpP=b3)go1-1^x-Jtdps|YeVlZn^!b%$|p)4OFWJlN1 zAn>E-z|4<0&A-cu?Grra5E$Odd#FglTCkyWRe;LuN}wSKcHeyFi*aFMZkD5sPbpn& z-OZ_bRhhPyGb*HQY8Q*54da>@47CdBd{I&=Z>W^&+zT5^WnPZ*c@ra@)7+F2&+}ut z0SYzqEDIAYi!(`3LgVNJ`%b0KqiaEAZe$A6FE81-g*gyNAFOoFg}b7s3Tox7Bj;*r z37eGJ>CB_}-iI8CPt|Kuc}kdz$v}v<)J!LzZ(ZMqd^f`#S=6lJy}SKyjQFf&lzLt{NQ{!} zuiJ2l3naZrn(nvTQxUDz6I~t&x*XP%UHT8{Ha4b>P!;-MwwyYPn;blENU;aO-jBnl zljZNjx8L5&rCT$Lexf{)BsOMA*@^S*AsvqG2}Bt zJCkolCf`7|f_GO?;o&)*IntG7&rL{gX||BYrejUncfnug^y7H;j=i})+)cM|O_#>O zqcjX6D*fKkq}}F^XY@<{OWTw`d|wp8)=V%K2&M{g>R7Awkj#UMa;0N(<3`X&?aF6* zH`nLlRqMkx8>@(9jNtRIml_}m>NN8l^wHpRSHbF0&~)I@#V-h&E9>&A$o9(H+#l!~ zSdgVz&&I-NTo^4QiP@)fMiOkJnFp6WC1F6guKLVqnUCRMKId;+IFoOKnzqJdJDjYM zXFgGoCTuV2Ht;~{AZ1UHtOgIAydX7xS`;u2Sj*#{ZDMX{VqK{B5QTKgo>EQi6>IA2 zt{FNPMH8!_Pz(d=BO1skk13KRpr+CNg-Rt4X{@Hg@C-+~km0D)z=K7V^u$w&0v7e^ zFLaaOH4m%0xZz06QXXeX@LdmFv9z+MD2xWmclDw$=Md?CG9}E5g*7mzk~taimmiuk zYEmCmVPlIog?djR7<$!X4hAocNqY8}Rz~k4i_zO9j7fBEHk&Kf5ODWur)#e}O)?Hj00Huo#V|(uHY_my1#3W%J7**3{}< z>x$7@+UdeIQ5E-75Fw@aOMO7{?ZKpjDh@jSF<3n4uu#K<`=23t)MQL5%AxFo->CU? qcwK@d7eS&6rybVCJFrpf!J?-SS?aMM6}O=mmHYp@68sB6F%e<90zTFN literal 0 HcmV?d00001 diff --git a/man/read_dataset_dsjc.Rd b/man/read_dataset_dsjc.Rd index 7ac6ae3..40f6815 100644 --- a/man/read_dataset_dsjc.Rd +++ b/man/read_dataset_dsjc.Rd @@ -21,7 +21,10 @@ Dataset NDJSON content. The object returned is the same one uncompressed file, metadata attributes included. } \examples{ +# Read one of the example files shipped with the package +dm <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) + +# Or from bytes held in memory ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) -bytes <- write_dataset_dsjc(ds_json) -dat <- read_dataset_dsjc(bytes) +dat <- read_dataset_dsjc(write_dataset_dsjc(ds_json)) } diff --git a/man/validate_dataset_dsjc.Rd b/man/validate_dataset_dsjc.Rd index f704316..9fe9b56 100644 --- a/man/validate_dataset_dsjc.Rd +++ b/man/validate_dataset_dsjc.Rd @@ -7,8 +7,8 @@ validate_dataset_dsjc(x) } \arguments{ -\item{x}{File path of a DSJC file, or a raw vector holding the compressed -bytes} +\item{x}{File path or URL of a DSJC file, or a raw vector holding the +compressed bytes} } \value{ A data frame of errors, empty when the file is valid @@ -20,6 +20,5 @@ checked against the Dataset NDJSON v1.1.0 schema, and each subsequent line must be a JSON array carrying one value per declared column. } \examples{ -ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) -validate_dataset_dsjc(write_dataset_dsjc(ds_json)) +validate_dataset_dsjc(datasetjson_example("dm.dsjc")) } diff --git a/tests/testthat/test-dsjc.R b/tests/testthat/test-dsjc.R index 4648952..f1c2fcb 100644 --- a/tests/testthat/test-dsjc.R +++ b/tests/testthat/test-dsjc.R @@ -118,3 +118,39 @@ test_that("numbers survive the compressed path bit-exactly", { expect_identical(as.double(read_dataset_dsjc(write_dataset_dsjc(ds))$V), v) }) + +test_that("the shipped example file reads back as the JSON it came from", { + from_json <- read_dataset_json(datasetjson_example("dm.json")) + from_dsjc <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) + + expect_equal(as.data.frame(from_json), as.data.frame(from_dsjc), + ignore_attr = "datasetJSONCreationDateTime") + + a <- attributes(from_json) + b <- attributes(from_dsjc) + a$datasetJSONCreationDateTime <- b$datasetJSONCreationDateTime <- NULL + expect_equal(a, b) + + expect_message(validate_dataset_dsjc(datasetjson_example("dm.dsjc")), + "File is valid") + + # it is meaningfully smaller than the JSON it came from + expect_lt(file.size(datasetjson_example("dm.dsjc")), + file.size(datasetjson_example("dm.json")) / 2) +}) + +test_that("DSJC can be read from a URL", { + ds <- make_ds(head(iris, 25)) + f <- withr::local_tempfile(fileext = ".dsjc") + write_dataset_dsjc(ds, f) + + # file:// exercises the same binary path as http(s) without needing a network + u <- paste0("file://", f) + expect_true(path_is_url(u)) + + expect_equal(read_dataset_dsjc(u), read_dataset_dsjc(f)) + expect_message(validate_dataset_dsjc(u), "File is valid") + + # the bytes must come through untouched - readLines() would corrupt them + expect_identical(read_raw_from_url(u), readBin(f, "raw", file.size(f))) +}) From 409c2a9048d03471c72c9a9b848bb2a2ecfd7e2c Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Wed, 2 Sep 2026 17:23:43 -0400 Subject: [PATCH 3/7] Ship dm.ndjson alongside the other example files The example set covered JSON and the compressed form but not the NDJSON in between, which is the format the compressed one is built from. All three are now generated from dm.json in data-raw, and a test asserts they read back to the same dataset - including that dm.dsjc really is dm.ndjson compressed, once the creation timestamp each file carries is normalised. read_dataset_ndjson() and validate_dataset_ndjson() examples now read a real file rather than writing one first. --- R/read_dataset_ndjson.R | 7 +++-- R/validate_dataset_json.R | 6 +++-- data-raw/data.R | 5 ++-- inst/extdata/dm.ndjson | 19 ++++++++++++++ man/validate_dataset_ndjson.Rd | 6 +++-- tests/testthat/test-dsjc.R | 48 ++++++++++++++++++++++++++-------- 6 files changed, 72 insertions(+), 19 deletions(-) create mode 100644 inst/extdata/dm.ndjson diff --git a/R/read_dataset_ndjson.R b/R/read_dataset_ndjson.R index f4faf1c..5faefcf 100644 --- a/R/read_dataset_ndjson.R +++ b/R/read_dataset_ndjson.R @@ -19,6 +19,10 @@ #' @export #' #' @examples +#' # Read one of the example files shipped with the package +#' dm <- read_dataset_ndjson(datasetjson_example("dm.ndjson")) +#' +#' # Or from NDJSON text held in memory #' ds_json <- dataset_json( #' iris, #' item_oid = "IG.IRIS", @@ -26,8 +30,7 @@ #' dataset_label = "Iris", #' columns = iris_items #' ) -#' nd <- write_dataset_ndjson(ds_json) -#' dat <- read_dataset_ndjson(nd) +#' dat <- read_dataset_ndjson(write_dataset_ndjson(ds_json)) read_dataset_ndjson <- function(file) { if (path_is_url(file)) { parsed <- .Call(C_read_dsndjson_str, read_from_url(file)) diff --git a/R/validate_dataset_json.R b/R/validate_dataset_json.R index 8d56b79..37d9be2 100644 --- a/R/validate_dataset_json.R +++ b/R/validate_dataset_json.R @@ -79,6 +79,9 @@ validate_dataset_json <- function(x) { #' @export #' #' @examples +#' # Validate a file on disk +#' validate_dataset_ndjson(datasetjson_example("dm.ndjson")) +#' #' ds_json <- dataset_json( #' iris, #' item_oid = "IG.IRIS", @@ -86,9 +89,8 @@ validate_dataset_json <- function(x) { #' dataset_label = "Iris", #' columns = iris_items #' ) -#' nd <- write_dataset_ndjson(ds_json) #' -#' validate_dataset_ndjson(nd) +#' validate_dataset_ndjson(write_dataset_ndjson(ds_json)) validate_dataset_ndjson <- function(x) { if (!requireNamespace("jsonvalidate", quietly = TRUE)) { stop( diff --git a/data-raw/data.R b/data-raw/data.R index 53813dd..da58c25 100644 --- a/data-raw/data.R +++ b/data-raw/data.R @@ -129,11 +129,12 @@ usethis::use_data(schema_ndjson_1_1_0, overwrite = TRUE) # Example files shipped in inst/extdata for datasetjson_example() # -# dm.json is the CDISC-published Dataset JSON example. The compressed form is -# generated from it so the three representations stay in step. +# dm.json is the CDISC-published Dataset JSON example. The NDJSON and +# compressed forms are generated from it so all three stay in step. dm_example <- read_dataset_json( system.file("extdata", "dm.json", package = "datasetjson") ) +write_dataset_ndjson(dm_example, file.path("inst", "extdata", "dm.ndjson")) write_dataset_dsjc(dm_example, file.path("inst", "extdata", "dm.dsjc")) # Test data metadata diff --git a/inst/extdata/dm.ndjson b/inst/extdata/dm.ndjson new file mode 100644 index 0000000..7b21bed --- /dev/null +++ b/inst/extdata/dm.ndjson @@ -0,0 +1,19 @@ +{"datasetJSONCreationDateTime":"2026-09-02T17:21:59","datasetJSONVersion":"1.1.0","fileOID":"www.cdisc.org/StudyMSGv2/1/Define-XML_2.1.0/2024-11-11/dm","dbLastModifiedDateTime":"2020-08-21T09:14:29","originator":"CDISC SDTM MSG Team","sourceSystem":{"name":"SAS on X64_10PRO","version":"9.0401M7"},"studyOID":"cdisc.com/CDISCPILOT01","metaDataVersionOID":"MDV.MSGv2.0.SDTMIG.3.3.SDTM.1.7","metaDataRef":"define.xml","itemGroupOID":"IG.DM","records":18,"name":"DM","label":"Demographics","columns":[{"itemOID":"IT.DM.STUDYID","name":"STUDYID","label":"Study Identifier","dataType":"string","length":12,"keySequence":1},{"itemOID":"IT.DM.DOMAIN","name":"DOMAIN","label":"Domain Abbreviation","dataType":"string","length":2},{"itemOID":"IT.DM.USUBJID","name":"USUBJID","label":"Unique Subject Identifier","dataType":"string","length":8,"keySequence":2},{"itemOID":"IT.DM.SUBJID","name":"SUBJID","label":"Subject Identifier for the Study","dataType":"string","length":4},{"itemOID":"IT.DM.RFSTDTC","name":"RFSTDTC","label":"Subject Reference Start Date/Time","dataType":"date"},{"itemOID":"IT.DM.RFENDTC","name":"RFENDTC","label":"Subject Reference End Date/Time","dataType":"date"},{"itemOID":"IT.DM.RFXSTDTC","name":"RFXSTDTC","label":"Date/Time of First Study Treatment","dataType":"date"},{"itemOID":"IT.DM.RFXENDTC","name":"RFXENDTC","label":"Date/Time of Last Study Treatment","dataType":"date"},{"itemOID":"IT.DM.RFICDTC","name":"RFICDTC","label":"Date/Time of Informed Consent","dataType":"date"},{"itemOID":"IT.DM.RFPENDTC","name":"RFPENDTC","label":"Date/Time of End of Participation","dataType":"date"},{"itemOID":"IT.DM.DTHDTC","name":"DTHDTC","label":"Date/Time of Death","dataType":"date"},{"itemOID":"IT.DM.DTHFL","name":"DTHFL","label":"Subject Death Flag","dataType":"string","length":1},{"itemOID":"IT.DM.SITEID","name":"SITEID","label":"Study Site Identifier","dataType":"string","length":3},{"itemOID":"IT.DM.BRTHDTC","name":"BRTHDTC","label":"Date/Time of Birth","dataType":"date"},{"itemOID":"IT.DM.AGE","name":"AGE","label":"Age","dataType":"integer"},{"itemOID":"IT.DM.AGEU","name":"AGEU","label":"Age Units","dataType":"string","length":5},{"itemOID":"IT.DM.SEX","name":"SEX","label":"Sex","dataType":"string","length":1},{"itemOID":"IT.DM.RACE","name":"RACE","label":"Race","dataType":"string","length":41},{"itemOID":"IT.DM.ETHNIC","name":"ETHNIC","label":"Ethnicity","dataType":"string","length":22},{"itemOID":"IT.DM.ARMCD","name":"ARMCD","label":"Planned Arm Code","dataType":"string","length":8},{"itemOID":"IT.DM.ARM","name":"ARM","label":"Description of Planned Arm","dataType":"string","length":28},{"itemOID":"IT.DM.ACTARMCD","name":"ACTARMCD","label":"Actual Arm Code","dataType":"string","length":8},{"itemOID":"IT.DM.ACTARM","name":"ACTARM","label":"Description of Actual Arm","dataType":"string","length":28},{"itemOID":"IT.DM.ARMNRS","name":"ARMNRS","label":"Reason Arm and/or Actual Arm is Null","dataType":"string","length":14},{"itemOID":"IT.DM.ACTARMUD","name":"ACTARMUD","label":"Description of Unplanned Actual Arm","dataType":"string","length":200},{"itemOID":"IT.DM.COUNTRY","name":"COUNTRY","label":"Country","dataType":"string","length":3}]} +["CDISCPILOT01","DM","CDISC001","1115","2012-11-30","2013-01-23","2012-11-30","2013-01-23","2012-11-23","2013-05-20","","","701","1928",84,"YEARS","M","WHITE","NOT HISPANIC OR LATINO","ZAN_LOW","Zanomaline Low Dose (54 mg)","ZAN_LOW","Zanomaline Low Dose (54 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC002","1211","2012-11-15","2013-01-14","2012-11-15","2013-01-12","2012-10-30","2013-01-14","2013-01-14","Y","701","1936",76,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_LOW","Zanomaline Low Dose (54 mg)","ZAN_LOW","Zanomaline Low Dose (54 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC003","1302","2013-08-29","2013-11-05","2013-08-29","2013-11-05","2013-08-20","2014-02-13","","","701","1951",61,"YEARS","M","WHITE","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC004","1345","2013-10-08","2014-03-18","2013-10-08","2014-03-18","2013-10-01","2014-03-18","","","701","1950",63,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","PLACEBO","Placebo","PLACEBO","Placebo","","","USA"] +["CDISCPILOT01","DM","CDISC005","1383","2013-02-04","2013-08-06","2013-02-04","2013-08-06","2013-01-22","2013-08-06","","","701","1941",72,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC006","1429","2013-03-19","2013-04-30","2013-03-19","2013-04-30","2013-02-25","2013-04-30","","","701","1929",84,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_LOW","Zanomaline Low Dose (54 mg)","ZAN_LOW","Zanomaline Low Dose (54 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC007","1444","2013-01-05","2013-02-13","2013-01-05","2013-02-12","2012-12-31","2013-06-20","","","701","1949",63,"YEARS","M","WHITE","HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC008","1445","2014-05-11","2014-11-01","2014-05-11","2014-11-01","2014-05-01","2014-11-01","2014-11-01","Y","704","1939",75,"YEARS","M","MULTIPLE","NOT HISPANIC OR LATINO","PLACEBO","Placebo","PLACEBO","Placebo","","","USA"] +["CDISCPILOT01","DM","CDISC009","1087","2012-10-22","2013-04-28","2012-10-22","2013-04-28","2012-10-06","2013-04-28","","","708","1938",74,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","PLACEBO","Placebo","PLACEBO","Placebo","","","USA"] +["CDISCPILOT01","DM","CDISC010","1236","2013-09-21","2013-09-26","2013-09-21","2013-09-21","2013-09-08","2013-09-26","","","708","1927",86,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC011","1336","2012-12-07","2013-06-05","2012-12-07","2013-06-05","2012-11-21","2013-07-05","","","708","1939",73,"YEARS","M","WHITE","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC012","1378","2013-09-03","2014-01-28","2013-09-03","2014-01-28","2013-08-24","2014-01-28","","","708","1946",67,"YEARS","M","BLACK OR AFRICAN AMERICAN","NOT HISPANIC OR LATINO","PLACEBO","Placebo","PLACEBO","Placebo","","","USA"] +["CDISCPILOT01","DM","CDISC013","1083","2013-07-22","2013-08-03","2013-07-22","2013-08-01","2013-07-09","2013-08-03","2013-08-03","Y","710","1924",89,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","PLACEBO","Placebo","PLACEBO","Placebo","","","USA"] +["CDISCPILOT01","DM","CDISC014","1012","2013-04-03","2013-05-02","2013-04-03","2013-04-29","2013-03-20","2013-09-18","","","711","1945",67,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC015","1022","","","","","2014-03-17","2014-03-17","","","711","1928",86,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","","","","","SCREEN FAILURE","","USA"] +["CDISCPILOT01","DM","CDISC016","1143","2013-04-03","2013-06-01","2013-04-03","2013-05-30","2013-03-30","2013-09-22","","","711","1936",76,"YEARS","F","WHITE","NOT HISPANIC OR LATINO","ZAN_LOW","Zanomaline Low Dose (54 mg)","ZAN_LOW","Zanomaline Low Dose (54 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC017","1250","2013-09-21","2014-02-08","2013-09-21","2014-01-31","2013-08-21","2014-03-08","","","718","1931",82,"YEARS","F","WHITE","HISPANIC OR LATINO","ZAN_LOW","Zanomaline Low Dose (54 mg)","ZAN_LOW","Zanomaline Low Dose (54 mg)","","","USA"] +["CDISCPILOT01","DM","CDISC018","1427","2012-12-17","2013-02-18","2012-12-17","2013-02-11","2012-12-13","2013-06-03","","","718","1938",74,"YEARS","F","BLACK OR AFRICAN AMERICAN","NOT HISPANIC OR LATINO","ZAN_HIGH","Zanomaline High Dose (81 mg)","ZAN_HIGH","Zanomaline High Dose (81 mg)","","","USA"] diff --git a/man/validate_dataset_ndjson.Rd b/man/validate_dataset_ndjson.Rd index ff6d589..2b04947 100644 --- a/man/validate_dataset_ndjson.Rd +++ b/man/validate_dataset_ndjson.Rd @@ -20,6 +20,9 @@ NDJSON v1.1.0 schema, and each subsequent line is checked to be a JSON array carrying one value per declared column. } \examples{ +# Validate a file on disk +validate_dataset_ndjson(datasetjson_example("dm.ndjson")) + ds_json <- dataset_json( iris, item_oid = "IG.IRIS", @@ -27,7 +30,6 @@ ds_json <- dataset_json( dataset_label = "Iris", columns = iris_items ) -nd <- write_dataset_ndjson(ds_json) -validate_dataset_ndjson(nd) +validate_dataset_ndjson(write_dataset_ndjson(ds_json)) } diff --git a/tests/testthat/test-dsjc.R b/tests/testthat/test-dsjc.R index f1c2fcb..0e0aa5d 100644 --- a/tests/testthat/test-dsjc.R +++ b/tests/testthat/test-dsjc.R @@ -119,24 +119,50 @@ test_that("numbers survive the compressed path bit-exactly", { expect_identical(as.double(read_dataset_dsjc(write_dataset_dsjc(ds))$V), v) }) -test_that("the shipped example file reads back as the JSON it came from", { - from_json <- read_dataset_json(datasetjson_example("dm.json")) - from_dsjc <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) - - expect_equal(as.data.frame(from_json), as.data.frame(from_dsjc), - ignore_attr = "datasetJSONCreationDateTime") +test_that("the three shipped example files agree with each other", { + # dm.ndjson and dm.dsjc are generated from dm.json in data-raw, so all three + # must read back to the same dataset + reads <- list( + json = read_dataset_json(datasetjson_example("dm.json")), + ndjson = read_dataset_ndjson(datasetjson_example("dm.ndjson")), + dsjc = read_dataset_dsjc(datasetjson_example("dm.dsjc")) + ) - a <- attributes(from_json) - b <- attributes(from_dsjc) - a$datasetJSONCreationDateTime <- b$datasetJSONCreationDateTime <- NULL - expect_equal(a, b) + for (nm in c("ndjson", "dsjc")) { + expect_equal(as.data.frame(reads[[nm]]), as.data.frame(reads$json), + ignore_attr = "datasetJSONCreationDateTime", info = nm) + a <- attributes(reads$json) + b <- attributes(reads[[nm]]) + a$datasetJSONCreationDateTime <- b$datasetJSONCreationDateTime <- NULL + expect_equal(b, a, info = nm) + } + expect_message(validate_dataset_ndjson(datasetjson_example("dm.ndjson")), + "File is valid") expect_message(validate_dataset_dsjc(datasetjson_example("dm.dsjc")), "File is valid") - # it is meaningfully smaller than the JSON it came from + # the compressed form is meaningfully smaller than either uncompressed one expect_lt(file.size(datasetjson_example("dm.dsjc")), file.size(datasetjson_example("dm.json")) / 2) + expect_lt(file.size(datasetjson_example("dm.dsjc")), + file.size(datasetjson_example("dm.ndjson")) / 2) + + # and dm.dsjc really is dm.ndjson compressed. Each file carries the moment it + # was written, so that one field is normalised before comparing. + blank_created <- function(x) { + sub('"datasetJSONCreationDateTime":"[^"]*"', + '"datasetJSONCreationDateTime":"X"', x) + } + dsjc_path <- datasetjson_example("dm.dsjc") + inflated <- rawToChar(memDecompress( + readBin(dsjc_path, "raw", file.size(dsjc_path)), type = "gzip")) + ndjson_text <- paste0( + paste(readLines(datasetjson_example("dm.ndjson"), warn = FALSE), + collapse = "\n"), + "\n") + + expect_identical(blank_created(inflated), blank_created(ndjson_text)) }) test_that("DSJC can be read from a URL", { From 3c1bc3739de93388fe6af1a68c2a9b61ce9340c1 Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Wed, 2 Sep 2026 17:32:05 -0400 Subject: [PATCH 4/7] Document the compressed format in the Getting Started vignette Covers what DSJC is, the three functions, the size difference using the shipped example files, the compression level trade-off, and the raw vector return. Sits after the NDJSON section, since DSJC is that format compressed. --- vignettes/datasetjson.Rmd | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/vignettes/datasetjson.Rmd b/vignettes/datasetjson.Rmd index 1597792..3814e02 100644 --- a/vignettes/datasetjson.Rmd +++ b/vignettes/datasetjson.Rmd @@ -122,6 +122,44 @@ cat(nd) The two formats are interchangeable - reading `iris.json` and `iris.ndjson` written from the same object gives you the same dataframe, with the same attributes. +## Compressed files + +Dataset JSON files are text, and text of this shape compresses very well. The standard defines a compressed representation, DSJC, which is simply the NDJSON content of a dataset as a zLib stream - no header, no wrapper, nothing but the compressed bytes. Files use the `.dsjc` extension. + +`write_dataset_dsjc()`, `read_dataset_dsjc()` and `validate_dataset_dsjc()` mirror the other two sets of functions: + +```{r dsjc, eval=FALSE} +write_dataset_dsjc(ds_json, file = "iris.dsjc") + +dat <- read_dataset_dsjc("iris.dsjc") +``` + +Because a DSJC file is a plain zLib stream, any zLib implementation can read it, and a stream produced by another tool reads here. The example files shipped with the package show the size difference on a small dataset: + +```{r dsjc_size} +sizes <- vapply( + c("dm.json", "dm.ndjson", "dm.dsjc"), + function(f) file.size(datasetjson_example(f)), + numeric(1) +) + +sizes +``` + +The saving grows with the dataset. On a 300,000 row file the NDJSON is 26 MB and the DSJC around 11 MB. + +`level` controls the compression, from 0 (none) to 9. It defaults to 9, which the standard recommends for data exchange, but the top of the range earns little: on that same 26 MB dataset, level 1 wrote in a fifth of the time for a file only 4% larger. Reading is unaffected by the level a file was written at. + +With no `file` argument, `write_dataset_dsjc()` returns a raw vector rather than a character string, since the format is binary: + +```{r dsjc_raw} +bytes <- write_dataset_dsjc(ds_json) + +head(bytes) +``` + +Rows are compressed as they are written, so producing a large file never holds the whole uncompressed dataset in memory. + The data frame that's read in is itself a `datasetjson` object and carries a number of attributes. For example, opening the dataframe within the RStudio IDE will present the variable labels. Additionally, the extra metadata provided in a Dataset JSON file is available. The attributes provided follow the naming convention of the Dataset JSON standard. ## Working with the metadata From f186f882018cf21c01ee3223b55564e6325d256d Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Wed, 2 Sep 2026 19:59:38 -0400 Subject: [PATCH 5/7] Inflate DSJC content straight into R memory The reader inflated into a malloc buffer and then copied the whole thing into R_alloc memory, the copy existing only so a later parse error could not leak the allocation. At scale that copy cost as much resident memory as the decompressed content itself: reading a 419MB file peaked at 2437MB, which is more than reading the 999MB uncompressed NDJSON it came from - the wrong way round for a format whose point is being smaller. Inflating into a PROTECTed raw vector, grown with Rf_xlengthgets, needs no copy and cannot leak if ndjson_core() raises. 1GB dataset: read peak 2437MB -> 2188MB. What remains above the NDJSON reader is the compressed input buffer, which is held whole while parsing; streaming the file through inflate would remove that too. --- src/read.c | 72 ++++++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/src/read.c b/src/read.c index c70b40c..ce77fe9 100644 --- a/src/read.c +++ b/src/read.c @@ -600,60 +600,56 @@ SEXP C_ndjson_shape(SEXP lines_) { * result as Dataset NDJSON. * ------------------------------------------------------------------------ */ -/* Inflate `in` into a buffer allocated with R_alloc. Returns NULL on failure, - with *msg set. */ -static char *inflate_all(const unsigned char *in, size_t n, size_t *out_len, - const char **msg) { +/* Inflate the whole stream into an R-managed raw vector. + Inflating into R memory rather than malloc means the buffer needs no copy + before parsing and cannot leak if parsing later raises an error - which + matters at scale, where the copy alone cost as much resident memory as the + decompressed content. */ +static SEXP dsjc_core(const unsigned char *in, size_t n) { z_stream zs; memset(&zs, 0, sizeof(zs)); - if (inflateInit(&zs) != Z_OK) { *msg = "could not initialise zLib"; return NULL; } + if (inflateInit(&zs) != Z_OK) Rf_error("Could not initialise zLib"); - size_t cap = n * 4 + 1024, len = 0; - char *out = (char *) malloc(cap); - if (!out) { inflateEnd(&zs); *msg = "out of memory"; return NULL; } + /* zLib streams carry no uncompressed size, so this is an estimate; text of + this shape typically compresses to under a third */ + R_xlen_t cap = (R_xlen_t) n * 4 + 1024; + PROTECT_INDEX pi; + SEXP buf; + PROTECT_WITH_INDEX(buf = Rf_allocVector(RAWSXP, cap), &pi); + + size_t len = 0; + int rc = Z_OK; + const char *msg = NULL; zs.next_in = (Bytef *) in; zs.avail_in = (uInt) n; - int rc = Z_OK; while (rc != Z_STREAM_END) { - if (len == cap) { - size_t ncap = cap * 2; - char *bigger = (char *) realloc(out, ncap); - if (!bigger) { free(out); inflateEnd(&zs); *msg = "out of memory"; return NULL; } - out = bigger; cap = ncap; + if ((R_xlen_t) len == cap) { + cap *= 2; + REPROTECT(buf = Rf_xlengthgets(buf, cap), pi); } - zs.next_out = (Bytef *) (out + len); - zs.avail_out = (uInt) (cap - len); + zs.next_out = (Bytef *) (RAW(buf) + len); + zs.avail_out = (uInt) ((size_t) cap - len); rc = inflate(&zs, Z_NO_FLUSH); if (rc != Z_OK && rc != Z_STREAM_END) { - free(out); inflateEnd(&zs); - *msg = zs.msg ? zs.msg : "invalid or corrupt compressed data"; - return NULL; + msg = zs.msg ? zs.msg : "invalid or corrupt compressed data"; + break; } - len = cap - zs.avail_out; + len = (size_t) cap - zs.avail_out; if (rc == Z_OK && zs.avail_in == 0 && zs.avail_out != 0) { - free(out); inflateEnd(&zs); - *msg = "compressed stream ended before it was complete"; - return NULL; + msg = "compressed stream ended before it was complete"; + break; } } inflateEnd(&zs); - *out_len = len; - - /* hand back R-managed memory so an error later cannot leak the buffer */ - char *res = (char *) R_alloc(len + 1, 1); - memcpy(res, out, len); - res[len] = '\0'; - free(out); - return res; -} + if (msg) { + UNPROTECT(1); + Rf_error("Could not read Dataset JSON Compressed content: %s", msg); + } -static SEXP dsjc_core(const unsigned char *in, size_t n) { - size_t len = 0; - const char *msg = NULL; - char *txt = inflate_all(in, n, &len, &msg); - if (!txt) Rf_error("Could not read Dataset JSON Compressed content: %s", msg); - return ndjson_core(txt, len); + SEXP out = PROTECT(ndjson_core((const char *) RAW(buf), len)); + UNPROTECT(2); + return out; } SEXP C_read_dsjc_file(SEXP path_) { From ff5ab808bf49c54f9c2e5989d809f9b813165919 Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Thu, 3 Sep 2026 16:05:35 -0400 Subject: [PATCH 6/7] Address review feedback on the DSJC tests - Note why memDecompress(type = "gzip") inflates a zLib stream: R's "gzip" covers both RFC 1952 and RFC 1950, and DSJC is the latter. - Say which zLib branch each malformed-input case targets, and cover truncation both ways - a heavy prefix cut and the loss of just the trailing checksum. Both reach the same branch: any prefix of a valid deflate stream is itself a valid partial stream, so inflate() asks for more input rather than reporting corruption. Corruption is covered separately by overwriting bytes mid-stream. - Add a float_as_decimals test for the compressed writer. It shares prepare_dataset_for_write() with the other two, but passes as_decimal to its own C entry point, so it needs its own coverage. Checks the metadata flips to decimal/decimal, the numbers are quoted inside the compressed content, and values still round-trip bit-exactly. --- tests/testthat/test-dsjc.R | 54 +++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test-dsjc.R b/tests/testthat/test-dsjc.R index 0e0aa5d..62a4c36 100644 --- a/tests/testthat/test-dsjc.R +++ b/tests/testthat/test-dsjc.R @@ -17,7 +17,10 @@ test_that("DSJC is a plain zLib stream of the NDJSON content", { # zLib (RFC 1950) header, level 9 expect_identical(as.integer(bytes[1:2]), c(120L, 218L)) - # the content is byte-for-byte what write_dataset_ndjson() puts in a file + # the content is byte-for-byte what write_dataset_ndjson() puts in a file. + # R's type = "gzip" covers both RFC 1952 (gzip, 0x1F 0x8B) and RFC 1950 + # (zLib, 0x78 0xDA); DSJC is the latter, so this inflates it correctly + # despite the argument's name. f <- withr::local_tempfile(fileext = ".ndjson") write_dataset_ndjson(ds, f) expect_identical( @@ -73,18 +76,63 @@ test_that("a stream compressed by another tool reads correctly", { test_that("corrupt compressed input is reported, not crashed on", { ds <- make_ds(head(iris, 20)) + # inflate() returns Z_DATA_ERROR: the stream is structurally broken damaged <- write_dataset_dsjc(ds) damaged[50:60] <- as.raw(0) expect_error(read_dataset_dsjc(damaged), "Could not read Dataset JSON Compressed content") - truncated <- write_dataset_dsjc(ds)[1:40] - expect_error(read_dataset_dsjc(truncated), + # inflate() returns Z_OK with no input left and no Z_STREAM_END: the stream is + # well formed as far as it goes but stops early. Any prefix of a valid deflate + # stream is a valid partial stream, so both a heavy truncation and the loss of + # just the trailing checksum land here rather than on the error above. + full <- write_dataset_dsjc(ds) + expect_error(read_dataset_dsjc(full[seq_len(40)]), + "ended before it was complete") + expect_error(read_dataset_dsjc(full[seq_len(length(full) - 4L)]), + "ended before it was complete") + expect_error(read_dataset_dsjc(full[seq_len(length(full) - 1L)]), "ended before it was complete") expect_error(read_dataset_dsjc("no/such/file.dsjc"), "must be a path") }) +test_that("float_as_decimals works through the compressed path", { + # prepare_dataset_for_write() is shared, but write_dataset_dsjc() passes + # as_decimal to its own C entry point, so the flag needs covering here too + ds <- make_ds(head(iris, 5)) + + expect_warning(write_dataset_dsjc(ds, float_as_decimals = TRUE), + "no longer needed to protect against") + bytes <- suppressWarnings(write_dataset_dsjc(ds, float_as_decimals = TRUE)) + + # the numbers really are quoted inside the compressed content, and the column + # metadata really did flip to decimal/decimal + content <- rawToChar(memDecompress(bytes, type = "gzip")) + expect_true(grepl('"dataType":"decimal","targetDataType":"decimal"', content, + fixed = TRUE)) + expect_true(grepl('["5.1","3.5"', content, fixed = TRUE)) + + # and it reads back to the same data as the plain compressed path + expect_equal(as.data.frame(read_dataset_dsjc(bytes)), + as.data.frame(read_dataset_dsjc(write_dataset_dsjc(ds))), + ignore_attr = TRUE) + + # values survive the decimal encoding through compression bit-exactly + v <- c(143.66666666666699825, 2 / 3, 1 / 3) + d <- data.frame(SUBJ = c("a", "b", "c"), V = v, stringsAsFactors = FALSE) + items <- data.frame( + itemOID = c("IT.SUBJ", "IT.V"), name = c("SUBJ", "V"), + label = c("Subject", "Value"), dataType = c("string", "float"), + stringsAsFactors = FALSE + ) + dsv <- dataset_json(d, item_oid = "t", name = "t", dataset_label = "t", + columns = items) + back <- read_dataset_dsjc( + suppressWarnings(write_dataset_dsjc(dsv, float_as_decimals = TRUE))) + expect_identical(as.double(back$V), v) +}) + test_that("DSJC content validates as Dataset NDJSON", { ds <- make_ds() expect_message( From f1b51eaff9605f5f32dafea24dad9cd4122ced8f Mon Sep 17 00:00:00 2001 From: "mike.stackhouse" Date: Thu, 3 Sep 2026 16:34:56 -0400 Subject: [PATCH 7/7] Use named arguments in the DSJC example Carries over the fix from #98's review; it lived in the merge commit the rebase onto the rewritten base dropped. --- R/dsjc.R | 8 +++++++- man/read_dataset_dsjc.Rd | 8 +++++++- man/read_dataset_ndjson.Rd | 7 +++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/R/dsjc.R b/R/dsjc.R index a243a88..727fe89 100644 --- a/R/dsjc.R +++ b/R/dsjc.R @@ -93,7 +93,13 @@ write_dataset_dsjc <- function( #' dm <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) #' #' # Or from bytes held in memory -#' ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +#' ds_json <- dataset_json( +#' iris, +#' item_oid = "IG.IRIS", +#' name = "IRIS", +#' dataset_label = "Iris", +#' columns = iris_items +#' ) #' dat <- read_dataset_dsjc(write_dataset_dsjc(ds_json)) read_dataset_dsjc <- function(file) { if (is.raw(file)) { diff --git a/man/read_dataset_dsjc.Rd b/man/read_dataset_dsjc.Rd index 40f6815..ef172c0 100644 --- a/man/read_dataset_dsjc.Rd +++ b/man/read_dataset_dsjc.Rd @@ -25,6 +25,12 @@ uncompressed file, metadata attributes included. dm <- read_dataset_dsjc(datasetjson_example("dm.dsjc")) # Or from bytes held in memory -ds_json <- dataset_json(iris, "IG.IRIS", "IRIS", "Iris", columns = iris_items) +ds_json <- dataset_json( + iris, + item_oid = "IG.IRIS", + name = "IRIS", + dataset_label = "Iris", + columns = iris_items +) dat <- read_dataset_dsjc(write_dataset_dsjc(ds_json)) } diff --git a/man/read_dataset_ndjson.Rd b/man/read_dataset_ndjson.Rd index 12b3858..b8b0bf1 100644 --- a/man/read_dataset_ndjson.Rd +++ b/man/read_dataset_ndjson.Rd @@ -26,6 +26,10 @@ returns for the equivalent \code{.json} file, including all of the metadata attached as attributes. See \code{read_dataset_json()} for the full list. } \examples{ +# Read one of the example files shipped with the package +dm <- read_dataset_ndjson(datasetjson_example("dm.ndjson")) + +# Or from NDJSON text held in memory ds_json <- dataset_json( iris, item_oid = "IG.IRIS", @@ -33,6 +37,5 @@ ds_json <- dataset_json( dataset_label = "Iris", columns = iris_items ) -nd <- write_dataset_ndjson(ds_json) -dat <- read_dataset_ndjson(nd) +dat <- read_dataset_ndjson(write_dataset_ndjson(ds_json)) }