diff --git a/.Rbuildignore b/.Rbuildignore index 821228a9..079ec8c2 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -5,5 +5,6 @@ extras/* docs/* rmd/* +site/* ^CRAN-SUBMISSION$ ^cran-comments\.md$ diff --git a/.github/workflows/R_CMD_check.yaml b/.github/workflows/R_CMD_check.yaml index 47735116..eea366d6 100644 --- a/.github/workflows/R_CMD_check.yaml +++ b/.github/workflows/R_CMD_check.yaml @@ -67,7 +67,7 @@ jobs: - name: Cache R packages if: runner.os != 'Windows' - uses: actions/cache@v3.3.1 + uses: actions/cache@v4 with: path: ${{ env.R_LIBS_USER }} key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }} @@ -111,14 +111,14 @@ jobs: - name: Upload check results if: failure() - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: ${{ runner.os }}-r${{ matrix.config.r }}-results path: check - name: Upload source package if: success() && runner.os == 'macOS' && github.event_name != 'pull_request' && github.ref == 'refs/heads/master' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: package_tarball path: check/*.tar.gz diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 3ebbf550..22cdd283 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -17,6 +17,6 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Codespell uses: codespell-project/actions-codespell@v2 diff --git a/DESCRIPTION b/DESCRIPTION index 83740240..1a5c4f52 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -27,5 +27,5 @@ Suggests: RSQLite, withr NeedsCompilation: no -RoxygenNote: 7.2.3 +RoxygenNote: 7.3.3 Config/testthat/edition: 3 diff --git a/R/createDdl.R b/R/createDdl.R index 8ecb7111..3f17cb3b 100644 --- a/R/createDdl.R +++ b/R/createDdl.R @@ -49,24 +49,55 @@ createDdl <- function(cdmVersion){ tableList <- tableSpecs$cdmTableName + # Define sort keys for major tables + sortKeyMap <- list( + visit = "visit_concept_id,person_id", + visit_detail = "visit_detail_concept_id,person_id", + condition_occurrence = "condition_concept_id,person_id", + drug_exposure = "drug_concept_id,person_id", + device_exposure = "device_concept_id,person_id", + procedure_occurrence = "procedure_concept_id,person_id", + observation = "observation_concept_id,person_id", + drug_era = "drug_concept_id,person_id" + ) + sql_result <- c() sql_result <- c(paste0("--@targetDialect CDM DDL Specification for OMOP Common Data Model ", cdmVersion)) for (tableName in tableList){ fields <- subset(cdmSpecs, cdmTableName == tableName) fieldNames <- fields$cdmFieldName + # Build HINT statement with DISTRIBUTE_ON_KEY (Redshift only) if ('person_id' %in% fieldNames){ - query <- "\n\n--HINT DISTRIBUTE ON KEY (person_id)\n" + hintContent <- "--HINT DISTRIBUTE_ON_KEY(person_id)" } else { - query <- "\n\n--HINT DISTRIBUTE ON RANDOM\n" + hintContent <- "--HINT DISTRIBUTE_ON_KEY(RANDOM)" + } + + # Add SORT_ON_KEY if table is in sortKeyMap and has all the sort fields + if (tableName %in% names(sortKeyMap)){ + sortFieldStr <- sortKeyMap[[tableName]] + # Split by comma to get individual fields + sortFields <- trimws(strsplit(sortFieldStr, ",")[[1]]) + # Check if all sort fields exist in the table + if (all(sortFields %in% fieldNames)){ + hintContent <- paste0(hintContent, " SORT_ON_KEY(INTERLEAVED:", sortFieldStr, ")") + } } + # Use HINT as a single-line comment + # Redshift will recognize and apply --HINT directives + # Non-Redshift dialects will have these removed in writeDdl() + hint <- hintContent + + query <- paste0("\n", hint, "\n") sql_result <- c(sql_result, query, paste0("CREATE TABLE @cdmDatabaseSchema.", tableName, " (")) n_fields <- length(fieldNames) for(fieldName in fieldNames) { - if (subset(fields, cdmFieldName == fieldName, isRequired) == "Yes") { + req_val <- subset(fields, cdmFieldName == fieldName, isRequired) + if (req_val == "Yes" || req_val == "true" || req_val == TRUE) { nullable_sql <- (" NOT NULL") } else { nullable_sql <- (" NULL") @@ -109,7 +140,7 @@ createPrimaryKeys <- function(cdmVersion){ cdmFieldCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Field_Level.csv")), package = "CommonDataModel", mustWork = TRUE) cdmSpecs <- read.csv(cdmFieldCsvLoc, stringsAsFactors = FALSE) - primaryKeys <- subset(cdmSpecs, isPrimaryKey == "Yes") + primaryKeys <- subset(cdmSpecs, isPrimaryKey == "true" | isPrimaryKey == "Yes" | isPrimaryKey == TRUE) pkFields <- primaryKeys$cdmFieldName sql_result <- c(paste0("--@targetDialect CDM Primary Key Constraints for OMOP Common Data Model ", cdmVersion, "\n")) @@ -136,16 +167,20 @@ createForeignKeys <- function(cdmVersion){ cdmFieldCsvLoc <- system.file(file.path("csv", paste0("OMOP_CDMv", cdmVersion, "_Field_Level.csv")), package = "CommonDataModel", mustWork = TRUE) cdmSpecs <- read.csv(cdmFieldCsvLoc, stringsAsFactors = FALSE) - foreignKeys <- subset(cdmSpecs, isForeignKey == "Yes") - foreignKeys$key <- paste0(foreignKeys$cdmTableName, "_", foreignKeys$cdmFieldName) - + foreignKeys <- subset(cdmSpecs, isForeignKey == "true" | isForeignKey == "Yes" | isForeignKey == TRUE) + sql_result <- c(paste0("--@targetDialect CDM Foreign Key Constraints for OMOP Common Data Model ", cdmVersion, "\n")) - for (foreignKey in foreignKeys$key){ - - subquery <- subset(foreignKeys, foreignKeys$key==foreignKey) + + # Only process if there are foreign keys + if (nrow(foreignKeys) > 0) { + foreignKeys$key <- paste0(foreignKeys$cdmTableName, "_", foreignKeys$cdmFieldName) + + for (foreignKey in foreignKeys$key){ + subquery <- subset(foreignKeys, foreignKeys$key==foreignKey) sql_result <- c(sql_result, paste0("\nALTER TABLE @cdmDatabaseSchema.", subquery$cdmTableName, " ADD CONSTRAINT fpk_", subquery$cdmTableName, "_", subquery$cdmFieldName, " FOREIGN KEY (", subquery$cdmFieldName , ") REFERENCES @cdmDatabaseSchema.", subquery$fkTableName, " (", subquery$fkFieldName, ");\n")) + } } return(paste0(sql_result, collapse = "")) } diff --git a/R/writeDDL.R b/R/writeDDL.R index 35de110c..4524b670 100644 --- a/R/writeDDL.R +++ b/R/writeDDL.R @@ -45,8 +45,22 @@ writeDdl <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchema sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect) + # Post-processing: remove conditional markers and handle dialects + lines <- strsplit(sql, "\n")[[1]] + + if (tolower(targetDialect) == "redshift") { + # For Redshift: remove conditional markers but keep the HINT lines + lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)] + } else { + # For non-Redshift: remove HINT directives and conditional markers + lines <- lines[!grepl("--HINT|\\{#?if|\\{/if\\}", lines)] + } + + sql <- paste(lines, collapse = "\n") + filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "ddl.sql", sep = "_") - SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename)) + # Use writeLines instead of SqlRender::writeSql to avoid line wrapping that breaks SQL + writeLines(sql, con = file.path(outputfolder, filename)) invisible(filename) } @@ -71,8 +85,14 @@ writePrimaryKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabas sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect) + # Post-processing: remove any conditional markers that may have been added + lines <- strsplit(sql, "\n")[[1]] + lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)] + sql <- paste(lines, collapse = "\n") + filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "primary", "keys.sql", sep = "_") - SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename)) + # Use writeLines instead of SqlRender::writeSql to avoid line wrapping + writeLines(sql, con = file.path(outputfolder, filename)) invisible(filename) } @@ -96,8 +116,14 @@ writeForeignKeys <- function(targetDialect, cdmVersion, outputfolder, cdmDatabas sql <- SqlRender::render(sql = sql, cdmDatabaseSchema = cdmDatabaseSchema, targetDialect = targetDialect) sql <- SqlRender::translate(sql, targetDialect = targetDialect) + # Post-processing: remove any conditional markers that may have been added + lines <- strsplit(sql, "\n")[[1]] + lines <- lines[!grepl("\\{#?if|\\{/if\\}", lines)] + sql <- paste(lines, collapse = "\n") + filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "constraints.sql", sep = "_") - SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename)) + # Use writeLines instead of SqlRender::writeSql to avoid line wrapping + writeLines(sql, con = file.path(outputfolder, filename)) invisible(filename) } @@ -123,6 +149,7 @@ writeIndex <- function(targetDialect, cdmVersion, outputfolder, cdmDatabaseSchem sql <- SqlRender::translate(sql, targetDialect = targetDialect) filename <- paste("OMOPCDM", gsub(" ", "_", targetDialect), cdmVersion, "indices.sql", sep = "_") - SqlRender::writeSql(sql = sql, targetFile = file.path(outputfolder, filename)) + # Use writeLines instead of SqlRender::writeSql to avoid line wrapping + writeLines(sql, con = file.path(outputfolder, filename)) invisible(filename) } diff --git a/docs/background.html b/docs/background.html index e6ce9851..3dd85c31 100644 --- a/docs/background.html +++ b/docs/background.html @@ -2,7 +2,7 @@ - + @@ -33,19 +33,18 @@ - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + + - - - - + + + - - - + + +