CDM v5.4 documentation migration to quarto and bugfixes

.Rbuildignore

@@ -5,5 +5,6 @@
 extras/*
 docs/*
 rmd/*
+site/*
 ^CRAN-SUBMISSION$
 ^cran-comments\.md$

.github/workflows/R_CMD_check.yaml

@@ -67,7 +67,7 @@ jobs:
 
       - name: Cache R packages
         if: runner.os != 'Windows'
-        uses: actions/cache@v3.3.1
+        uses: actions/cache@v4
         with:
           path: ${{ env.R_LIBS_USER }}
           key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
@@ -111,14 +111,14 @@ jobs:
 
       - name: Upload check results
         if: failure()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ runner.os }}-r${{ matrix.config.r }}-results
           path: check
 
       - name: Upload source package
         if: success() && runner.os == 'macOS' && github.event_name != 'pull_request' && github.ref == 'refs/heads/master'
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: package_tarball
           path: check/*.tar.gz

.github/workflows/codespell.yml

@@ -17,6 +17,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2

DESCRIPTION

@@ -27,5 +27,5 @@ Suggests:
     RSQLite,
     withr
 NeedsCompilation: no
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.3
 Config/testthat/edition: 3

R/createDdl.R

@@ -49,24 +49,55 @@ createDdl <- function(cdmVersion){
 
   tableList <- tableSpecs$cdmTableName
 
+  # Define sort keys for major tables
+  sortKeyMap <- list(
+    visit = "visit_concept_id,person_id",
+    visit_detail = "visit_detail_concept_id,person_id",
+    condition_occurrence = "condition_concept_id,person_id",
+    drug_exposure = "drug_concept_id,person_id",
+    device_exposure = "device_concept_id,person_id",
+    procedure_occurrence = "procedure_concept_id,person_id",
+    observation = "observation_concept_id,person_id",
+    drug_era = "drug_concept_id,person_id"
+  )
+
   sql_result <- c()
   sql_result <- c(paste0("--@targetDialect CDM DDL Specification for OMOP Common Data Model ", cdmVersion))
   for (tableName in tableList){
     fields <- subset(cdmSpecs, cdmTableName == tableName)
     fieldNames <- fields$cdmFieldName
 
+    # Build HINT statement with DISTRIBUTE_ON_KEY (Redshift only)
     if ('person_id' %in% fieldNames){
-      query <- "\n\n--HINT DISTRIBUTE ON KEY (person_id)\n"
+      hintContent <- "--HINT DISTRIBUTE_ON_KEY(person_id)"
     } else {
-      query <- "\n\n--HINT DISTRIBUTE ON RANDOM\n"
+      hintContent <- "--HINT DISTRIBUTE_ON_KEY(RANDOM)"
+    }
+
+    # Add SORT_ON_KEY if table is in sortKeyMap and has all the sort fields
+    if (tableName %in% names(sortKeyMap)){
+      sortFieldStr <- sortKeyMap[[tableName]]
+      # Split by comma to get individual fields
+      sortFields <- trimws(strsplit(sortFieldStr, ",")[[1]])
+      # Check if all sort fields exist in the table
+      if (all(sortFields %in% fieldNames)){
+        hintContent <- paste0(hintContent, " SORT_ON_KEY(INTERLEAVED:", sortFieldStr, ")")
+      }
     }
 
+    # Use HINT as a single-line comment
+    # Redshift will recognize and apply --HINT directives
+    # Non-Redshift dialects will have these removed in writeDdl()
+    hint <- hintContent
+
+    query <- paste0("\n", hint, "\n")
     sql_result <- c(sql_result, query, paste0("CREATE TABLE @cdmDatabaseSchema.", tableName, " ("))
 
     n_fields <- length(fieldNames)
     for(fieldName in fieldNames) {
 
-      if (subset(fields, cdmFieldName == fieldName, isRequired) == "Yes") {
+      req_val <- subset(fields, cdmFieldName == fieldName, isRequired)
+      if (req_val == "Yes" || req_val == "true" || req_val == TRUE) {
         nullable_sql <- (" NOT NULL")
       } else {
         nullable_sql <- (" NULL")
@@ -109,7 +140,7 @@ createPrimaryKeys <- function(cdmVersion){
   cdmFieldCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Field_Level.csv")), package = "CommonDataModel", mustWork = TRUE)
   cdmSpecs <- read.csv(cdmFieldCsvLoc, stringsAsFactors = FALSE)
 
-  primaryKeys <- subset(cdmSpecs, isPrimaryKey == "Yes")
+  primaryKeys <- subset(cdmSpecs, isPrimaryKey == "true" | isPrimaryKey == "Yes" | isPrimaryKey == TRUE)
   pkFields <- primaryKeys$cdmFieldName
 
   sql_result <- c(paste0("--@targetDialect CDM Primary Key Constraints for OMOP Common Data Model ", cdmVersion, "\n"))
@@ -136,16 +167,20 @@ createForeignKeys <- function(cdmVersion){
   cdmFieldCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Field_Level.csv")), package = "CommonDataModel", mustWork = TRUE)
   cdmSpecs <- read.csv(cdmFieldCsvLoc, stringsAsFactors = FALSE)
 
-  foreignKeys <- subset(cdmSpecs, isForeignKey == "Yes")
-  foreignKeys$key <- paste0(foreignKeys$cdmTableName, "_", foreignKeys$cdmFieldName)
-
+  foreignKeys <- subset(cdmSpecs, isForeignKey == "true" | isForeignKey == "Yes" | isForeignKey == TRUE)
+
   sql_result <- c(paste0("--@targetDialect CDM Foreign Key Constraints for OMOP Common Data Model ", cdmVersion, "\n"))
-  for (foreignKey in foreignKeys$key){
-
-    subquery <- subset(foreignKeys, foreignKeys$key==foreignKey)
+
+  # Only process if there are foreign keys
+  if (nrow(foreignKeys) > 0) {
+    foreignKeys$key <- paste0(foreignKeys$cdmTableName, "_", foreignKeys$cdmFieldName)
+
+    for (foreignKey in foreignKeys$key){
+      subquery <- subset(foreignKeys, foreignKeys$key==foreignKey)
 
     sql_result <- c(sql_result, paste0("\nALTER TABLE @cdmDatabaseSchema.", subquery$cdmTableName, " ADD CONSTRAINT fpk_", subquery$cdmTableName, "_", subquery$cdmFieldName, " FOREIGN KEY (", subquery$cdmFieldName , ") REFERENCES @cdmDatabaseSchema.", subquery$fkTableName, " (", subquery$fkFieldName, ");\n"))
 
+    }
   }
   return(paste0(sql_result, collapse = ""))
 }

R/writeDDL.R

@@ -45,8 +45,22 @@ writeDdl <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
+  # Post-processing: remove conditional markers and handle dialects
+  lines <- strsplit(sql, "\n")[[1]]
+
+  if (tolower(targetDialect) == "redshift") {
+    # For Redshift: remove conditional markers but keep the HINT lines
+    lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)]
+  } else {
+    # For non-Redshift: remove HINT directives and conditional markers
+    lines <- lines[!grepl("--HINT|\\{#?if|\\{/if\\}", lines)]
+  }
+
+  sql <- paste(lines, collapse = "\n")
+
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "ddl.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
+  # Use writeLines instead of SqlRender::writeSql to avoid line wrapping that breaks SQL
+  writeLines(sql, con = file.path(outputfolder, filename))
   invisible(filename)
 }
 
@@ -71,8 +85,14 @@ writePrimaryKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabas
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
+  # Post-processing: remove any conditional markers that may have been added
+  lines <- strsplit(sql, "\n")[[1]]
+  lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)]
+  sql <- paste(lines, collapse = "\n")
+
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "primary", "keys.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
+  # Use writeLines instead of SqlRender::writeSql to avoid line wrapping
+  writeLines(sql, con = file.path(outputfolder, filename))
   invisible(filename)
 }
 
@@ -96,8 +116,14 @@ writeForeignKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabas
   sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect)
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
+  # Post-processing: remove any conditional markers that may have been added
+  lines <- strsplit(sql, "\n")[[1]]
+  lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)]
+  sql <- paste(lines, collapse = "\n")
+
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "constraints.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
+  # Use writeLines instead of SqlRender::writeSql to avoid line wrapping
+  writeLines(sql, con = file.path(outputfolder, filename))
   invisible(filename)
 }
 
@@ -123,6 +149,7 @@ writeIndex <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchem
   sql <- SqlRender::translate(sql, targetDialect = targetDialect)
 
   filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "indices.sql", sep = "_")
-  SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename))
+  # Use writeLines instead of SqlRender::writeSql to avoid line wrapping
+  writeLines(sql, con = file.path(outputfolder, filename))
   invisible(filename)
 }

docs/background.html

@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.8.27">
+<meta name="generator" content="quarto-1.7.29">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -33,19 +33,18 @@
 <link href="./man/favicon.ico" rel="icon">
 <script src="site_libs/quarto-html/quarto.js" type="module"></script>
 <script src="site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
-<script src="site_libs/quarto-html/axe/axe-check.js" type="module"></script>
 <script src="site_libs/quarto-html/popper.min.js"></script>
 <script src="site_libs/quarto-html/tippy.umd.min.js"></script>
 <script src="site_libs/quarto-html/anchor.min.js"></script>
 <link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-ed96de9b727972fe78a7b5d16c58bf87.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-dark-4d9afe2b8d18ee9fa5d0d57b5ed4214d.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-ed96de9b727972fe78a7b5d16c58bf87.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-0815c480559380816a4d1ea211a47e91.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-dark-748b535e376f14d4692bf2b2e5fd6380.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-0815c480559380816a4d1ea211a47e91.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
 <script src="site_libs/bootstrap/bootstrap.min.js"></script>
 <link href="site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
-<link href="site_libs/bootstrap/bootstrap-5b4ad623e5705c0698d39aec6f10cf02.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
-<link href="site_libs/bootstrap/bootstrap-dark-0d4a9d19931feb4fee1ddab4b2489000.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
-<link href="site_libs/bootstrap/bootstrap-5b4ad623e5705c0698d39aec6f10cf02.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
+<link href="site_libs/bootstrap/bootstrap-c1fac2584b48ed01fb6e278e36375074.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
+<link href="site_libs/bootstrap/bootstrap-dark-ccd81253fefb92e9cd3f9eb5b5d87b54.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
+<link href="site_libs/bootstrap/bootstrap-c1fac2584b48ed01fb6e278e36375074.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
 <script id="quarto-search-options" type="application/json">{
   "location": "navbar",
   "copy-button": false,
@@ -242,7 +241,6 @@
       toggleColorMode(toAlternate);
       setStyleSentinel(toAlternate);
       toggleGiscusIfUsed(toAlternate, darkModeDefault);
-      window.dispatchEvent(new Event('resize'));
     };
     // Switch to dark mode if need be
     if (hasAlternateSentinel()) {
@@ -258,8 +256,7 @@
       <div class="navbar-container container-fluid">
       <div class="navbar-brand-container mx-auto">
     <a href="./index.html" class="navbar-brand navbar-brand-logo">
-    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo light-content">
-    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo dark-content">
+    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo">
     </a>
     <a class="navbar-brand" href="./index.html">
     <span class="navbar-title">OMOP Common Data Model</span>
@@ -582,14 +579,13 @@ <h2 class="anchored" data-anchor-id="design-principles">Design Principles</h2>
       e.clearSelection();
     }
     const getTextToCopy = function(trigger) {
-      const outerScaffold = trigger.parentElement.cloneNode(true);
-      const codeEl = outerScaffold.querySelector('code');
-      for (const childEl of codeEl.children) {
-        if (isCodeAnnotation(childEl)) {
-          childEl.remove();
+        const codeEl = trigger.previousElementSibling.cloneNode(true);
+        for (const childEl of codeEl.children) {
+          if (isCodeAnnotation(childEl)) {
+            childEl.remove();
+          }
         }
-      }
-      return codeEl.innerText;
+        return codeEl.innerText;
     }
     const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
       text: getTextToCopy

docs/cdm30.html

@@ -2,7 +2,7 @@
 <html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>
 
 <meta charset="utf-8">
-<meta name="generator" content="quarto-1.8.27">
+<meta name="generator" content="quarto-1.7.29">
 
 <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">
 
@@ -33,19 +33,18 @@
 <link href="./man/favicon.ico" rel="icon">
 <script src="site_libs/quarto-html/quarto.js" type="module"></script>
 <script src="site_libs/quarto-html/tabsets/tabsets.js" type="module"></script>
-<script src="site_libs/quarto-html/axe/axe-check.js" type="module"></script>
 <script src="site_libs/quarto-html/popper.min.js"></script>
 <script src="site_libs/quarto-html/tippy.umd.min.js"></script>
 <script src="site_libs/quarto-html/anchor.min.js"></script>
 <link href="site_libs/quarto-html/tippy.css" rel="stylesheet">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-ed96de9b727972fe78a7b5d16c58bf87.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-dark-4d9afe2b8d18ee9fa5d0d57b5ed4214d.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
-<link href="site_libs/quarto-html/quarto-syntax-highlighting-ed96de9b727972fe78a7b5d16c58bf87.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-0815c480559380816a4d1ea211a47e91.css" rel="stylesheet" class="quarto-color-scheme" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-dark-748b535e376f14d4692bf2b2e5fd6380.css" rel="stylesheet" class="quarto-color-scheme quarto-color-alternate" id="quarto-text-highlighting-styles">
+<link href="site_libs/quarto-html/quarto-syntax-highlighting-0815c480559380816a4d1ea211a47e91.css" rel="stylesheet" class="quarto-color-scheme-extra" id="quarto-text-highlighting-styles">
 <script src="site_libs/bootstrap/bootstrap.min.js"></script>
 <link href="site_libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
-<link href="site_libs/bootstrap/bootstrap-5b4ad623e5705c0698d39aec6f10cf02.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
-<link href="site_libs/bootstrap/bootstrap-dark-0d4a9d19931feb4fee1ddab4b2489000.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
-<link href="site_libs/bootstrap/bootstrap-5b4ad623e5705c0698d39aec6f10cf02.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
+<link href="site_libs/bootstrap/bootstrap-c1fac2584b48ed01fb6e278e36375074.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme" id="quarto-bootstrap" data-mode="light">
+<link href="site_libs/bootstrap/bootstrap-dark-ccd81253fefb92e9cd3f9eb5b5d87b54.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme quarto-color-alternate" id="quarto-bootstrap" data-mode="dark">
+<link href="site_libs/bootstrap/bootstrap-c1fac2584b48ed01fb6e278e36375074.min.css" rel="stylesheet" append-hash="true" class="quarto-color-scheme-extra" id="quarto-bootstrap" data-mode="light">
 <script id="quarto-search-options" type="application/json">{
   "location": "navbar",
   "copy-button": false,
@@ -242,7 +241,6 @@
       toggleColorMode(toAlternate);
       setStyleSentinel(toAlternate);
       toggleGiscusIfUsed(toAlternate, darkModeDefault);
-      window.dispatchEvent(new Event('resize'));
     };
     // Switch to dark mode if need be
     if (hasAlternateSentinel()) {
@@ -258,8 +256,7 @@
       <div class="navbar-container container-fluid">
       <div class="navbar-brand-container mx-auto">
     <a href="./index.html" class="navbar-brand navbar-brand-logo">
-    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo light-content">
-    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo dark-content">
+    <img src="./man/ohdsi16x16.png" alt="" class="navbar-logo">
     </a>
     <a class="navbar-brand" href="./index.html">
     <span class="navbar-title">OMOP Common Data Model</span>
@@ -540,14 +537,13 @@ <h1 class="title">CDM v3</h1>
       e.clearSelection();
     }
     const getTextToCopy = function(trigger) {
-      const outerScaffold = trigger.parentElement.cloneNode(true);
-      const codeEl = outerScaffold.querySelector('code');
-      for (const childEl of codeEl.children) {
-        if (isCodeAnnotation(childEl)) {
-          childEl.remove();
+        const codeEl = trigger.previousElementSibling.cloneNode(true);
+        for (const childEl of codeEl.children) {
+          if (isCodeAnnotation(childEl)) {
+            childEl.remove();
+          }
         }
-      }
-      return codeEl.innerText;
+        return codeEl.innerText;
     }
     const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
       text: getTextToCopy