databricks-setup-script.r

# This script creates a Python 3.12 environment for databricks-connect 17.3.x
# This matches your cluster running Databricks Runtime 17.3 LTS (Spark 3.5.x with geospatial support)

tryCatch({
  # --- STEP 1: Find or Install a Conda Installation ---
  
  user_conda_path <- file.path(Sys.getenv("LOCALAPPDATA"), "miniconda3", "Scripts", "conda.exe")
  cat("Looking for your existing conda executable at:", user_conda_path, "\n")
  
  if (file.exists(user_conda_path)) {
    cat("Existing miniconda3 found. Using it.\n")
    conda_to_use <- user_conda_path
  } else {
    cat("Existing miniconda3 not found. Installing a fresh copy for R/reticulate...\n")
    reticulate::install_miniconda(force = TRUE)
    conda_to_use <- file.path(reticulate::miniconda_path(), "Scripts", "conda.exe")
  }
  
  # --- STEP 2: Force reticulate to use the chosen conda executable ---
  cat("Forcing reticulate to use conda at:", conda_to_use, "\n")
  Sys.setenv(RETICULATE_CONDA = conda_to_use)
  
  # --- STEP 3: Create the conda environment with Python 3.12 ---
  env_name <- "r-sparklyr-databricks-17.3"
  python_version <- "3.12"
  
  # First, remove the environment if it exists
  if (env_name %in% reticulate::conda_list()$name) {
    cat("An old version of '", env_name, "' was found. Removing it first...\n", sep="")
    reticulate::conda_remove(envname = env_name)
  }
  
  # Now, create it fresh
  cat("\nCreating new conda environment '", env_name, "' with Python ", python_version, "...\n", sep = "")
  reticulate::conda_create(envname = env_name, python_version = python_version)
  
  # --- STEP 4: Install the Databricks Connect package for 17.3.x ---
  package_to_install <- "databricks-connect==17.3.*"
  cat("\nInstalling '", package_to_install, "' into the new environment...\n", sep = "")
  reticulate::conda_install(envname = env_name, packages = package_to_install, pip = TRUE)
  
  cat("\n-----------------------------------\n")
  cat("SUCCESS: Setup script completed successfully.\n")
  cat("Environment:", env_name, "\n")
  cat("Python version: 3.12\n")
  cat("Databricks-connect: 17.3.x\n")
  cat("\nNow update your connection script to use this environment:\n")
  cat("env_name <- 'r-sparklyr-databricks-17.3'\n")
  cat("-----------------------------------\n")
  
}, error = function(e) {
  cat("\n-----------------------------------\n")
  cat("ERROR: Setup script failed.\n")
  cat("The error message was:\n")
  cat(e$message, "\n")
  cat("-----------------------------------\n")
  stop("Setup failed.")
})