diff --git a/DESCRIPTION b/DESCRIPTION index 68783fa..4846860 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: nat.python Type: Package -Title: Library-Agnostic Python Interoperability for R +Title: Python Interoperability and Environment Management for the 'natverse' Version: 0.0.0.9000 Authors@R: person(given = "Gregory", @@ -8,22 +8,26 @@ Authors@R: role = c("aut", "cre"), email = "jefferis@gmail.com", comment = c(ORCID = "0000-0002-0587-9355")) -Description: A small shared layer of library-agnostic Python interoperability - helpers built on 'reticulate'. Provides tools to introspect a Python - environment (which modules are installed and at what version), to convert - generic Python return values into idiomatic R (pandas 'DataFrame's to R - data frames, numpy 64-bit integer arrays to 'bit64' vectors, Python +Description: A small shared layer of Python interoperability helpers built on + 'reticulate' for the 'natverse' ecosystem. Provides tools to introspect a + Python environment (which modules are installed and at what version), to + convert generic Python return values into idiomatic R (pandas 'DataFrame's + to R data frames, numpy 64-bit integer arrays to 'bit64' vectors, Python datetimes to 'POSIXct'), and to bridge large integer identifiers between R, - numpy and raw bytes. It contains no knowledge of any specific Python - library or scientific domain; packages such as 'fafbseg', 'bancr' and - 'seatabler' are expected to depend on it and supply that specificity - themselves. + numpy and raw bytes. It also provisions and manages a shared 'miniconda' + Python environment, with opinionated defaults for the packages that natverse + tools expect (such as 'cloud-volume', 'CAVEclient' and 'navis'). Packages + such as 'fafbseg', 'bancr' and 'seatabler' depend on it for both this + interoperability and a single, consistent way to set Python up. License: GPL-3 Encoding: UTF-8 Language: en-GB Imports: reticulate, - bit64 + bit64, + cli, + jsonlite, + memoise Suggests: arrow, tibble, diff --git a/NAMESPACE b/NAMESPACE index 806e9c7..dc1394d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,7 @@ # Generated by roxygen2: do not edit by hand +export(check_module) +export(check_reticulate) export(forget_module_version) export(int64_overflows) export(module_available) @@ -7,8 +9,8 @@ export(module_version) export(null2na) export(pandas2df) export(py_module_info) -export(py_module_info2) export(pyids2bit64) export(rids2pyint) export(rids2raw) +export(simple_python) export(ts2pydatetime) diff --git a/R/env.R b/R/env.R new file mode 100644 index 0000000..9c3b1be --- /dev/null +++ b/R/env.R @@ -0,0 +1,259 @@ +# Python environment engine. +# +# The mechanics of environment provisioning, ported near-verbatim from fafbseg's +# simple_python family (utils.R). fafbseg keeps its exported simple_python() +# front-end and delegates the mechanics here; seatabler and others depend on +# nat.python and call simple_python() / check_module() directly. Messaging uses +# cli rather than fafbseg's usethis ui_* helpers; the fafbseg.condaenv option is +# generalised to nat.python.condaenv (default "r-reticulate"). + +# The conda environment nat.python manages by default. Consumers may override +# with options(nat.python.condaenv = ...). +np_condaenv <- function() getOption("nat.python.condaenv", "r-reticulate") + +#' Install a managed Python environment for R +#' +#' @description Sets up (and optionally populates) a dedicated miniconda Python +#' environment for use from R via reticulate. This is the ecosystem's shared +#' provisioning entry point: packages such as fafbseg, bancr and seatabler all +#' route their Python installation through it, so users have one command to +#' run and one environment to manage. +#' +#' @details With `miniconda = TRUE` (the default and recommendation) a private +#' miniconda install and `r-reticulate` conda environment are created/updated, +#' independent of any system Python. The `pyinstall` bundles install a curated +#' set of packages used across the FlyWire/connectomics ecosystem: +#' `"basic"` adds cloud-volume, seatable_api and CAVEclient; `"full"` adds +#' navis + fafbseg; `"extra"` additionally installs skeletonisation tooling +#' (skeletor, meshparty and friends). `"none"` provisions the environment but +#' installs no bundle, which is what you want when passing your own `pkgs`. +#' `"cleanenv"` and `"blast"` only print the (destructive) commands needed to +#' remove an environment; they never delete anything themselves. +#' +#' After any installation the cached module versions +#' ([forget_module_version()]) and the [check_module()] memoise cache are +#' cleared so that subsequent checks reflect the new environment. +#' +#' @param pyinstall Which package bundle to install. One of `"basic"`, `"full"`, +#' `"extra"`, `"cleanenv"`, `"blast"` or `"none"`. +#' @param pkgs Optional character vector of additional Python packages (pip +#' specifications) to install into the environment. +#' @param miniconda Whether to use the managed miniconda environment (strongly +#' recommended). When `FALSE` your current Python is used as-is. +#' +#' @return Invisibly `NULL`. Called for its side effect of provisioning Python. +#' @export +#' @examples +#' \dontrun{ +#' simple_python("basic") # the common case +#' simple_python("none", pkgs = "seatable_api") # just one package +#' } +simple_python <- function(pyinstall = c("basic", "full", "extra", "cleanenv", + "blast", "none"), + pkgs = NULL, miniconda = TRUE) { + + check_reticulate(check_python = FALSE) + check_python(initialize = FALSE) + ourpip <- function(...) + reticulate::py_install(..., pip = TRUE, + pip_options = "--upgrade --prefer-binary") + + # since we may well change installed modules, clear cached module versions and + # the check_module() memoise cache so later checks reflect the new environment + on.exit({ + forget_module_version() + forget_check_module() + }) + pyinstall <- match.arg(pyinstall) + if (pyinstall != "none") + simple_python_base(pyinstall, miniconda) + if (pyinstall %in% c("cleanenv", "blast")) return(invisible(NULL)) + + if (pyinstall %in% c("basic", "full", "extra")) { + cli::cli_inform("Installing cloudvolume") + ourpip("cloud-volume") + cli::cli_inform("Installing seatable_api (access flytable metadata service)") + # 2.6.3 had a problem, see + # https://github.com/seatable/seatable-api-python/issues/76 + ourpip("seatable_api!=2.6.3") + cli::cli_inform("Installing CAVEclient (access to extended FlyWire/FANC APIs)") + ourpip("caveclient") + } + if (pyinstall %in% c("full", "extra")) { + cli::cli_inform("Installing navis+fafbseg (python access to FlyWire/FANC data)") + ourpip("fafbseg") + } + if (pyinstall %in% c("extra")) { + cli::cli_inform("Installing skeletor (Philipp Schlegel mesh skeletonisation)") + ourpip("skeletor") + cli::cli_inform("Installing skeletor addons (for faster skeletonisation)") + ourpip(c("fastremap", "ncollpyde")) + cli::cli_inform("Installing meshparty (includes Seung lab mesh skeletonisation)") + ourpip("meshparty") + cli::cli_inform(paste("Installing pyembree (so meshparty can give skeletons", + "radius estimates)")) + # not sure this will always work, but definitely optional + tryCatch(reticulate::conda_install(packages = "pyembree"), + error = function(e) cli::cli_warn(conditionMessage(e))) + } + if (!is.null(pkgs)) { + cli::cli_inform("Installing user-specified packages") + ourpip(pkgs) + } + invisible(NULL) +} + +#' Check that a working Python is available via reticulate +#' +#' @description reticulate is a hard dependency of nat.python, so this really +#' just checks that a usable Python is set up, guiding the user to +#' [simple_python()] when it is not. The name is kept for continuity with the +#' ecosystem's `check_reticulate()` entry point. +#' +#' @param check_python Whether to check that a working Python is available. When +#' `FALSE` the function is a no-op returning `TRUE`. +#' @return Invisibly `TRUE` when the check passes, `FALSE` otherwise. +#' @export +check_reticulate <- function(check_python = TRUE) { + if (check_python) check_python() else invisible(TRUE) +} + +# Check for a usable Python, guiding the user to simple_python() when missing. +check_python <- function(initialize = TRUE) { + # if python is already running, then we're fine + if (reticulate::py_available()) + return(invisible(TRUE)) + + nopython <- c( + "!" = "You do not have Python set up for R.", + "i" = "We recommend installing it with {.run nat.python::simple_python()}.", + "i" = paste("Alternatively point the {.envvar RETICULATE_PYTHON} environment", + "variable at a Python you manage; see {.help simple_python}.")) + + if (!ownpythonrequested()) { + pyfound <- try(reticulate::use_miniconda(np_condaenv(), required = TRUE), + silent = TRUE) + if (inherits(pyfound, "try-error")) { + cli::cli_inform(nopython) + return(invisible(FALSE)) + } + } + + pyavail <- reticulate::py_available(initialize = initialize) + if (!initialize || pyavail) return(invisible(TRUE)) + cli::cli_inform(nopython) + invisible(FALSE) +} + +ownpythonrequested <- function() { + nzchar(Sys.getenv("RETICULATE_PYTHON")) +} + +checkownpython <- function(miniconda) { + if (ownpythonrequested() || !miniconda) + cli::cli_abort("You have specified a non-standard Python. Sorry you're on your own!") +} + +current_python <- function() { + conf <- reticulate::py_discover_config() + pypath <- conf$python + if (!isTRUE(nzchar(pypath)) || !isTRUE(try(file.exists(pypath)))) + structure(NA, .Names = "unknown_python") + else + structure(file.mtime(conf$python), .Names = conf$python) +} + +default_pyenv <- function() { + conf <- reticulate::py_discover_config() + sub(":.*", "", conf$pythonhome) +} + +# my own update function so that I can check if it actually updated anything +update_miniconda_base <- function() { + path <- reticulate::miniconda_path() + exe <- if (identical(.Platform$OS.type, "windows")) + "condabin/conda.bat" else "bin/conda" + conda <- file.path(path, exe) + + res <- system2(conda, c("update", "--yes", "--json", "--name", "base", "conda"), + stdout = TRUE) + if (!jsonlite::validate(res)) { + print(res) + cli::cli_abort("Unable to parse results of conda update") + } + js <- jsonlite::fromJSON(res) + # true when updated + length(js$actions) > 0 +} + +simple_python_base <- function(what, miniconda) { + if (what == "cleanenv") { + checkownpython(miniconda) + e <- default_pyenv() + cli::cli_inform(c( + paste("If you really want to clean the packages in your existing", + "miniconda for R virtual env at:"), + " " = "{e}", + "do:", + " " = "{.code reticulate::conda_remove(\"{e}\")}")) + return(invisible(NULL)) + } else if (what == "blast") { + checkownpython(miniconda) + mp <- reticulate::miniconda_path() + cli::cli_inform(c( + "If you really want to blast your whole existing miniconda for R install at:", + " " = "{mp}", + "do:", + " " = "{.code unlink(reticulate::miniconda_path(), recursive = TRUE)}", + "!" = paste("Don't do this without verifying that the path above correctly", + "identifies your installation!"))) + return(invisible(NULL)) + } + + py_was_running <- reticulate::py_available() + + pychanged <- FALSE + if (miniconda) { + if (nzchar(Sys.getenv("RETICULATE_PYTHON"))) + cli::cli_abort(c( + "You have chosen a specific Python via {.envvar RETICULATE_PYTHON}.", + "i" = paste("simple_python does not recommend this; unset it, e.g. with", + "{.run usethis::edit_r_environ()}."), + "i" = "If you are sure, use {.code simple_python(miniconda = FALSE)}.")) + + cli::cli_inform("Installing/updating a dedicated miniconda Python environment for R") + tryCatch({ + reticulate::install_miniconda() + pychanged <- TRUE + }, + error = function(e) { + if (grepl("already installed", conditionMessage(e))) + pychanged <<- update_miniconda_base() + }) + condaenv <- np_condaenv() + if (nzchar(condaenv) && condaenv != "r-reticulate") + reticulate::conda_create(envname = condaenv, + conda = reticulate::miniconda_path()) + if (py_was_running && pychanged) { + cli::cli_abort(c( + "You have just updated your version of Python on disk.", + "i" = "But there was already a different Python version attached to this R session.", + ">" = "{.strong Restart R} and run {.code simple_python()} again to use your new Python!")) + } + cli::cli_inform("Ensuring pip is available in conda environment {.val {condaenv}}") + reticulate::conda_install(envname = condaenv, packages = "pip") + reticulate::use_miniconda(condaenv) + } else { + cli::cli_inform(c( + "Using the following existing Python install. I hope you know what you're doing!")) + print(reticulate::py_config()) + if (!nzchar(Sys.getenv("RETICULATE_PYTHON"))) { + cli::cli_warn(c( + paste("When using a non-standard Python setup, we recommend telling R", + "exactly which install to use via {.envvar RETICULATE_PYTHON}."), + "i" = "Set it with {.run usethis::edit_r_environ()}, adding a line like:", + " " = "{.code RETICULATE_PYTHON=\"/opt/miniconda3/envs/r-reticulate/bin/python\"}")) + } + } + invisible(pychanged) +} diff --git a/R/int64.R b/R/int64.R index 4845288..8eb28bb 100644 --- a/R/int64.R +++ b/R/int64.R @@ -107,8 +107,6 @@ pyids2bit64 <- function(x, as_character = TRUE) { #' rids2pyint(c("720575940621039145", "720575940626877799")) #' } rids2pyint <- function(x, numpyarray = FALSE, usefile = NA) { - if (!requireNamespace("reticulate", quietly = TRUE)) - stop("Please install the 'reticulate' package.", call. = FALSE) np <- py_np(convert = FALSE) npa <- if (inherits(x, "np.ndarray")) x else if (!isTRUE(usefile) && (length(x) < 1e4 || isFALSE(usefile))) { diff --git a/R/modules.R b/R/modules.R index 655535c..4773d5d 100644 --- a/R/modules.R +++ b/R/modules.R @@ -2,72 +2,36 @@ # # Discover which modules are installed in the active Python environment and at # what version, without importing heavy modules just to find out. Ported from -# fafbseg's py_module_info / py_module_info2 / module_version / python_module_path -# (see nat.python-plan.md §3b). py_report is deliberately deferred to Phase 2, -# because it depends on the environment engine (check_reticulate / -# ownpythonrequested) and carries a consumer-curated package list. +# fafbseg's py_module_info / module_version (see nat.python-plan.md §3b), but +# using the metadata (no-import) approach as the single introspector. +# check_module() is the generic install/load gate abstracted from fafbseg's +# check_seatable / check_cloudvolume_reticulate; the environment engine it uses +# (check_reticulate / simple_python) lives in env.R. #' Report on installed Python modules #' -#' @description `py_module_info()` imports each module and reads its version and -#' filesystem path. `py_module_info2()` answers the same "is it installed and -#' what version" question *without* importing the modules, by reading Python -#' distribution metadata — much cheaper, and safe for heavy modules, at the -#' cost of no path column. +#' @description Reports which of the named Python modules are installed and at +#' what version *without importing* them — cheap, and safe for modules with +#' heavy or side-effecting imports. #' -#' @details `py_module_info()` imports each module (so a module with import side -#' effects will run them) and is the only variant that can report the on-disk -#' `path`. `py_module_info2()` uses `importlib.metadata` (or the +#' @details Reads Python distribution metadata via `importlib.metadata` (or the #' `importlib_metadata` backport) together with `packages_distributions()` to -#' map top-level import names onto installed distributions, so it never imports -#' the target module. +#' map top-level import names onto installed distributions, so the target +#' modules are never imported. Modules installed without standard distribution +#' metadata (namespace packages, some editable installs) can therefore report +#' as unavailable here even when they are importable; [module_version()] falls +#' back to importing in that case. #' #' @param modules Character vector of Python module (import) names. #' -#' @return A data frame with one row per unique module. `py_module_info()` has -#' columns `module`, `available`, `version`, `path`; `py_module_info2()` has -#' `module`, `available`, `version`. Returns `NULL` if reticulate is not -#' installed. +#' @return A data frame with one row per unique module and columns `module`, +#' `available`, `version`. #' @export #' @examples #' \dontrun{ #' py_module_info(c("numpy", "pandas")) -#' py_module_info2(c("numpy", "pandas")) # without importing them #' } py_module_info <- function(modules) { - if (!requireNamespace("reticulate", quietly = TRUE)) - return(NULL) - modules <- unique(modules) - paths <- character(length(modules)) - names(paths) <- modules - versions <- character(length(modules)) - names(versions) <- modules - available <- logical(length(modules)) - names(available) <- modules - - for (m in modules) { - mod <- tryCatch(reticulate::import(m), error = function(e) NULL) - available[m] <- !is.null(mod) - if (!available[m]) - next - paths[m] <- python_module_path(mod) - versions[m] <- tryCatch(mod$`__version__`, error = function(e) "") - } - df <- data.frame(module = modules, - available = available, - version = versions, - path = paths, - stringsAsFactors = FALSE) - row.names(df) <- NULL - df -} - -#' @rdname py_module_info -#' @export -py_module_info2 <- function(modules) { - if (!requireNamespace("reticulate", quietly = TRUE)) - return(NULL) - im <- tryCatch( reticulate::import("importlib.metadata", convert = FALSE), error = function(e) @@ -136,13 +100,26 @@ module_version <- function(module) { if (!is.null(cached)) return(cached) pmi <- try(py_module_info(module), silent = TRUE) - v <- if (inherits(pmi, "try-error") || is.null(pmi) || nrow(pmi) < 1L) - NA_character_ - else if (nzchar(pmi$version)) pmi$version else NA_character_ + v <- if (!inherits(pmi, "try-error") && !is.null(pmi) && nrow(pmi) >= 1L && + nzchar(pmi$version)) + pmi$version + else + # metadata gave nothing (namespace/editable install, or importlib.metadata + # unavailable): fall back to importing and reading __version__. + module_version_by_import(module) .nat_python_cache[[key]] <- v v } +# Import a module and read its __version__, for the cases where distribution +# metadata does not yield a version. Returns NA if it can't be imported. +module_version_by_import <- function(module) { + mod <- tryCatch(reticulate::import(module), error = function(e) NULL) + if (is.null(mod)) return(NA_character_) + v <- tryCatch(mod$`__version__`, error = function(e) "") + if (isTRUE(nzchar(v))) v else NA_character_ +} + #' @description `forget_module_version()` clears the [module_version()] cache. #' @rdname module_version #' @return `forget_module_version()` returns `NULL` invisibly. @@ -188,16 +165,149 @@ module_available <- function(module, action = c("none", "warning", "stop"), if (action == "none") available else invisible(available) } -# Extract a module's filesystem path, coping with namespace packages whose -# __path__ is a _NamespacePath repr rather than a plain character vector. -python_module_path <- function(mod) { - tryCatch({ - path <- mod$`__path__` - if (!is.character(path)) { - # e.g. "_NamespacePath(['/Users/paulbrooks/igneous', ''])" - path <- as.character(path) - path2 <- sub(".+?\\[(.+)\\].+?", "\\1", path) - scan(what = "", sep = ",", text = path2, quiet = TRUE) - } else path - }, error = function(e) "") +#' Ensure a Python module is installed and importable +#' +#' @description A generic gate for code that needs a particular Python module: +#' it checks the module is installed, imports it, optionally enforces a minimum +#' version, and — when it is missing — either installs it (interactively or on +#' request) or errors with actionable guidance. Package-specific checks (such +#' as fafbseg's `check_seatable()` / `check_cloudvolume_reticulate()`) become +#' thin wrappers over this. +#' +#' @details The check is deliberately ordered **installed-first, load-second**, +#' because the two failure modes need different advice. Installation is +#' detected from Python distribution metadata via [py_module_info()] (which +#' does not import the module), falling back to `reticulate::py_module_available()` +#' for namespace/local packages that lack metadata. Only once the module is +#' known to be installed is it actually imported, so an import that fails then +#' is reported as a *load* problem (a broken or mismatched environment), +#' distinct from the package simply being absent — with the underlying Python +#' error surfaced. +#' +#' The result is memoised for the session, so repeated calls for the same +#' module are cheap and hand back the already-imported module object. Any +#' installation via [simple_python()] clears this cache (as well as the +#' [module_version()] cache), so the next call re-checks and picks up the +#' newly installed package. Because of this, package-specific wrappers need +#' not memoise themselves. +#' +#' @param module A single Python module (import) name, e.g. `"seatable_api"`. +#' @param package The pip package (distribution) name to install if `module` is +#' missing. Defaults to `module`; supply it when they differ (e.g. import +#' `"cv2"` from package `"opencv-python"`). +#' @param min_version Optional minimum acceptable version (character or +#' [numeric_version]). Checked on every call via [module_version()]. +#' @param install How to handle a missing module: `"ask"` (the default) prompts +#' in an interactive session and otherwise errors; `"never"` always errors with +#' install instructions; `"always"` installs without prompting. +#' @param install_cmd The bundle argument passed to [simple_python()] when +#' installing (default `"none"`, i.e. install only `package`). Use e.g. +#' `"basic"` to pull a whole ecosystem bundle instead. +#' @param docs_url Optional documentation URL added to the failure message. +#' @param cache Whether to use (and populate) the session cache of results +#' (default `TRUE`). Pass `FALSE` to force a fresh check — e.g. after +#' installing or upgrading the module out of band. +#' +#' @return The imported module (a reticulate object), invisibly. +#' @export +#' @examples +#' \dontrun{ +#' seatable_api <- check_module("seatable_api") +#' cv <- check_module("cloudvolume", install_cmd = "basic", min_version = "5.0", +#' docs_url = "https://github.com/seung-lab/cloud-volume#setup") +#' } +check_module <- function(module, + package = module, + min_version = NULL, + install = c("ask", "never", "always"), + install_cmd = "none", + docs_url = NULL, + cache = TRUE) { + stopifnot(is.character(module), length(module) == 1L) + install <- match.arg(install) + # Route through the memoised copy or the raw guts. Keeping the guts in a plain + # function (check_module_impl) means cache = FALSE never touches the memoise + # store, and this wrapper keeps real formals for its documentation. + FUN <- if (isTRUE(cache)) check_module_memoised else check_module_impl + FUN(module, package, min_version, install, install_cmd, docs_url) +} + +# The guts of check_module(). Kept un-memoised so check_module(cache = FALSE) +# can call it directly; check_module_memoised is the cached copy used by +# default. +check_module_impl <- function(module, package, min_version, + install, install_cmd, docs_url) { + check_reticulate() + + installed <- module_installed(module) + if (!installed) { + do_install <- switch(install, + always = TRUE, + never = FALSE, + ask = interactive() && + tolower(readline(sprintf( + "Install the Python '%s' package now (y/n)? ", package))) == "y") + if (do_install) { + simple_python(pyinstall = install_cmd, pkgs = package) + installed <- module_installed(module) + } + if (!installed) module_missing_abort(module, package, install_cmd, docs_url) + } + + # Installed, so import it. A failure here is a load problem, not absence. + mod <- tryCatch(reticulate::import(module), + error = function(e) module_load_abort(module, e, docs_url)) + + if (!is.null(min_version)) { + v <- module_version(module) + if (is.na(v) || !isTRUE(numeric_version(v) >= min_version)) + cli::cli_abort(c( + "The Python module {.pkg {module}} is too old.", + "x" = "Need version {min_version} but found {if (is.na(v)) 'unknown' else v}.", + "i" = "Update with {.run nat.python::simple_python(pkgs = \"{package}\")}.")) + } + invisible(mod) +} + +# The cached copy of check_module_impl(). Caches the imported module for the +# session; simple_python() calls forget_check_module() after installing so a +# fresh check picks up newly installed packages. Only successful returns are +# cached (errors propagate uncached), so a failed check is retried. +check_module_memoised <- memoise::memoise(check_module_impl) + +# Clear the check_module() memoise cache. Called by simple_python() after an +# install, and by tests between cases. +forget_check_module <- function() { + memoise::forget(check_module_memoised) + invisible(NULL) +} + +# Is the module installed, per distribution metadata (no import), with a +# find_spec fallback for namespace/local packages that lack metadata? +module_installed <- function(module) { + info <- try(py_module_info(module), silent = TRUE) + if (!inherits(info, "try-error") && !is.null(info) && isTRUE(info$available[1])) + return(TRUE) + isTRUE(reticulate::py_module_available(module)) +} + +module_missing_abort <- function(module, package, install_cmd, docs_url) { + cli::cli_abort(c( + "The Python module {.pkg {module}} is required but not installed.", + "i" = "Install it with {.run nat.python::simple_python(pkgs = \"{package}\")}.", + if (!identical(install_cmd, "none")) + c("i" = "Or install the bundle: {.run nat.python::simple_python(\"{install_cmd}\")}.") + else NULL, + if (!is.null(docs_url)) c("i" = "See {.url {docs_url}}.") else NULL)) +} + +module_load_abort <- function(module, e, docs_url) { + cli::cli_abort(c( + "The Python module {.pkg {module}} is installed but failed to load.", + "x" = conditionMessage(e), + "i" = paste("This is usually an environment problem, not a missing package:", + "R may be pointing at the wrong Python."), + "i" = "Check/point {.envvar RETICULATE_PYTHON} via {.run usethis::edit_r_environ()}, e.g.", + " " = "{.code RETICULATE_PYTHON=\"/opt/miniconda3/envs/r-reticulate/bin/python\"}", + if (!is.null(docs_url)) c("i" = "See {.url {docs_url}}.") else NULL)) } diff --git a/man/check_module.Rd b/man/check_module.Rd new file mode 100644 index 0000000..c7177b0 --- /dev/null +++ b/man/check_module.Rd @@ -0,0 +1,76 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/modules.R +\name{check_module} +\alias{check_module} +\title{Ensure a Python module is installed and importable} +\usage{ +check_module( + module, + package = module, + min_version = NULL, + install = c("ask", "never", "always"), + install_cmd = "none", + docs_url = NULL, + cache = TRUE +) +} +\arguments{ +\item{module}{A single Python module (import) name, e.g. \code{"seatable_api"}.} + +\item{package}{The pip package (distribution) name to install if \code{module} is +missing. Defaults to \code{module}; supply it when they differ (e.g. import +\code{"cv2"} from package \code{"opencv-python"}).} + +\item{min_version}{Optional minimum acceptable version (character or +\link{numeric_version}). Checked on every call via \code{\link[=module_version]{module_version()}}.} + +\item{install}{How to handle a missing module: \code{"ask"} (the default) prompts +in an interactive session and otherwise errors; \code{"never"} always errors with +install instructions; \code{"always"} installs without prompting.} + +\item{install_cmd}{The bundle argument passed to \code{\link[=simple_python]{simple_python()}} when +installing (default \code{"none"}, i.e. install only \code{package}). Use e.g. +\code{"basic"} to pull a whole ecosystem bundle instead.} + +\item{docs_url}{Optional documentation URL added to the failure message.} + +\item{cache}{Whether to use (and populate) the session cache of results +(default \code{TRUE}). Pass \code{FALSE} to force a fresh check — e.g. after +installing or upgrading the module out of band.} +} +\value{ +The imported module (a reticulate object), invisibly. +} +\description{ +A generic gate for code that needs a particular Python module: +it checks the module is installed, imports it, optionally enforces a minimum +version, and — when it is missing — either installs it (interactively or on +request) or errors with actionable guidance. Package-specific checks (such +as fafbseg's \code{check_seatable()} / \code{check_cloudvolume_reticulate()}) become +thin wrappers over this. +} +\details{ +The check is deliberately ordered \strong{installed-first, load-second}, +because the two failure modes need different advice. Installation is +detected from Python distribution metadata via \code{\link[=py_module_info]{py_module_info()}} (which +does not import the module), falling back to \code{reticulate::py_module_available()} +for namespace/local packages that lack metadata. Only once the module is +known to be installed is it actually imported, so an import that fails then +is reported as a \emph{load} problem (a broken or mismatched environment), +distinct from the package simply being absent — with the underlying Python +error surfaced. + +The result is memoised for the session, so repeated calls for the same +module are cheap and hand back the already-imported module object. Any +installation via \code{\link[=simple_python]{simple_python()}} clears this cache (as well as the +\code{\link[=module_version]{module_version()}} cache), so the next call re-checks and picks up the +newly installed package. Because of this, package-specific wrappers need +not memoise themselves. +} +\examples{ +\dontrun{ +seatable_api <- check_module("seatable_api") +cv <- check_module("cloudvolume", install_cmd = "basic", min_version = "5.0", + docs_url = "https://github.com/seung-lab/cloud-volume#setup") +} +} diff --git a/man/check_reticulate.Rd b/man/check_reticulate.Rd new file mode 100644 index 0000000..79dea52 --- /dev/null +++ b/man/check_reticulate.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/env.R +\name{check_reticulate} +\alias{check_reticulate} +\title{Check that a working Python is available via reticulate} +\usage{ +check_reticulate(check_python = TRUE) +} +\arguments{ +\item{check_python}{Whether to check that a working Python is available. When +\code{FALSE} the function is a no-op returning \code{TRUE}.} +} +\value{ +Invisibly \code{TRUE} when the check passes, \code{FALSE} otherwise. +} +\description{ +reticulate is a hard dependency of nat.python, so this really +just checks that a usable Python is set up, guiding the user to +\code{\link[=simple_python]{simple_python()}} when it is not. The name is kept for continuity with the +ecosystem's \code{check_reticulate()} entry point. +} diff --git a/man/py_module_info.Rd b/man/py_module_info.Rd index f8b9806..6332e08 100644 --- a/man/py_module_info.Rd +++ b/man/py_module_info.Rd @@ -2,40 +2,33 @@ % Please edit documentation in R/modules.R \name{py_module_info} \alias{py_module_info} -\alias{py_module_info2} \title{Report on installed Python modules} \usage{ py_module_info(modules) - -py_module_info2(modules) } \arguments{ \item{modules}{Character vector of Python module (import) names.} } \value{ -A data frame with one row per unique module. \code{py_module_info()} has -columns \code{module}, \code{available}, \code{version}, \code{path}; \code{py_module_info2()} has -\code{module}, \code{available}, \code{version}. Returns \code{NULL} if reticulate is not -installed. +A data frame with one row per unique module and columns \code{module}, +\code{available}, \code{version}. } \description{ -\code{py_module_info()} imports each module and reads its version and -filesystem path. \code{py_module_info2()} answers the same "is it installed and -what version" question \emph{without} importing the modules, by reading Python -distribution metadata — much cheaper, and safe for heavy modules, at the -cost of no path column. +Reports which of the named Python modules are installed and at +what version \emph{without importing} them — cheap, and safe for modules with +heavy or side-effecting imports. } \details{ -\code{py_module_info()} imports each module (so a module with import side -effects will run them) and is the only variant that can report the on-disk -\code{path}. \code{py_module_info2()} uses \code{importlib.metadata} (or the +Reads Python distribution metadata via \code{importlib.metadata} (or the \code{importlib_metadata} backport) together with \code{packages_distributions()} to -map top-level import names onto installed distributions, so it never imports -the target module. +map top-level import names onto installed distributions, so the target +modules are never imported. Modules installed without standard distribution +metadata (namespace packages, some editable installs) can therefore report +as unavailable here even when they are importable; \code{\link[=module_version]{module_version()}} falls +back to importing in that case. } \examples{ \dontrun{ py_module_info(c("numpy", "pandas")) -py_module_info2(c("numpy", "pandas")) # without importing them } } diff --git a/man/simple_python.Rd b/man/simple_python.Rd new file mode 100644 index 0000000..041a99e --- /dev/null +++ b/man/simple_python.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/env.R +\name{simple_python} +\alias{simple_python} +\title{Install a managed Python environment for R} +\usage{ +simple_python( + pyinstall = c("basic", "full", "extra", "cleanenv", "blast", "none"), + pkgs = NULL, + miniconda = TRUE +) +} +\arguments{ +\item{pyinstall}{Which package bundle to install. One of \code{"basic"}, \code{"full"}, +\code{"extra"}, \code{"cleanenv"}, \code{"blast"} or \code{"none"}.} + +\item{pkgs}{Optional character vector of additional Python packages (pip +specifications) to install into the environment.} + +\item{miniconda}{Whether to use the managed miniconda environment (strongly +recommended). When \code{FALSE} your current Python is used as-is.} +} +\value{ +Invisibly \code{NULL}. Called for its side effect of provisioning Python. +} +\description{ +Sets up (and optionally populates) a dedicated miniconda Python +environment for use from R via reticulate. This is the ecosystem's shared +provisioning entry point: packages such as fafbseg, bancr and seatabler all +route their Python installation through it, so users have one command to +run and one environment to manage. +} +\details{ +With \code{miniconda = TRUE} (the default and recommendation) a private +miniconda install and \code{r-reticulate} conda environment are created/updated, +independent of any system Python. The \code{pyinstall} bundles install a curated +set of packages used across the FlyWire/connectomics ecosystem: +\code{"basic"} adds cloud-volume, seatable_api and CAVEclient; \code{"full"} adds +navis + fafbseg; \code{"extra"} additionally installs skeletonisation tooling +(skeletor, meshparty and friends). \code{"none"} provisions the environment but +installs no bundle, which is what you want when passing your own \code{pkgs}. +\code{"cleanenv"} and \code{"blast"} only print the (destructive) commands needed to +remove an environment; they never delete anything themselves. + +After any installation the cached module versions +(\code{\link[=forget_module_version]{forget_module_version()}}) and the \code{\link[=check_module]{check_module()}} memoise cache are +cleared so that subsequent checks reflect the new environment. +} +\examples{ +\dontrun{ +simple_python("basic") # the common case +simple_python("none", pkgs = "seatable_api") # just one package +} +} diff --git a/tests/testthat/test-check-module.R b/tests/testthat/test-check-module.R new file mode 100644 index 0000000..f98ca3d --- /dev/null +++ b/tests/testthat/test-check-module.R @@ -0,0 +1,84 @@ +# check_module() decision logic, exercised entirely offline by mocking the +# pieces that would otherwise touch Python (check_reticulate, module_installed, +# reticulate::import, module_version, simple_python). +# +# check_module() is memoised, so each test clears the cache first to keep cases +# independent (otherwise an earlier success for a module would be handed back). + +test_that("missing module with install='never' errors with instructions", { + forget_check_module() + local_mocked_bindings(check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) FALSE) + expect_error(check_module("nope", install = "never"), + "not installed") +}) + +test_that("missing module in non-interactive 'ask' errors (no prompt)", { + forget_check_module() + # tests run non-interactively, so 'ask' cannot prompt and must fall through + local_mocked_bindings(check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) FALSE) + expect_error(check_module("nope", install = "ask"), "not installed") +}) + +test_that("install='always' installs then errors if still absent", { + forget_check_module() + installed_called <- 0L + called <- FALSE + local_mocked_bindings( + check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) { installed_called <<- installed_called + 1L; FALSE }, + simple_python = function(...) { called <<- TRUE; invisible(NULL) }) + expect_error(check_module("nope", install = "always"), "not installed") + expect_true(called) # it tried to install + expect_equal(installed_called, 2L) # checked before and after install +}) + +test_that("installed + importable returns the module", { + forget_check_module() + stub <- structure(list(), class = "python.builtin.module") + local_mocked_bindings(check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) TRUE) + local_mocked_bindings(import = function(module, ...) stub, .package = "reticulate") + expect_identical(check_module("pandas"), stub) +}) + +test_that("installed but unloadable is reported as a load failure", { + forget_check_module() + local_mocked_bindings(check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) TRUE) + local_mocked_bindings(import = function(module, ...) stop("boom: bad build"), + .package = "reticulate") + expect_error(check_module("pandas"), "failed to load") + expect_error(check_module("pandas"), "boom: bad build") +}) + +test_that("min_version enforced", { + forget_check_module() + stub <- structure(list(), class = "python.builtin.module") + local_mocked_bindings(check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) TRUE, + module_version = function(module) "1.0.0") + local_mocked_bindings(import = function(module, ...) stub, .package = "reticulate") + expect_error(check_module("pandas", min_version = "2.0.0"), "too old") + expect_identical(check_module("pandas", min_version = "0.9.0"), stub) +}) + +test_that("memoisation returns the cached module and forget clears it", { + forget_check_module() + calls <- 0L + local_mocked_bindings( + check_reticulate = function(...) invisible(TRUE), + module_installed = function(module) { calls <<- calls + 1L; TRUE }) + local_mocked_bindings(import = function(module, ...) structure(list(), class = "python.builtin.module"), + .package = "reticulate") + m1 <- check_module("numpy") + m2 <- check_module("numpy") # served from cache, no re-check + expect_identical(m1, m2) + expect_equal(calls, 1L) + check_module("numpy", cache = FALSE) # bypasses cache, checks again + expect_equal(calls, 2L) + forget_check_module() + check_module("numpy") # cache cleared, checks again + expect_equal(calls, 3L) +})