diff --git a/Cargo.toml b/Cargo.toml index 9c2ad29..dd70e44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,9 +14,10 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["alternate-registries"] +default = ["git-registries"] -alternate-registries = ["dep:git2"] +git-registries = ["dep:git2"] +alternate-registries = ["git-registries"] unstable = [] unstable-toolchain-ci = [] tracing = ["dep:tracing"] @@ -35,6 +36,7 @@ flate2 = "1" tar = "0.4.0" percent-encoding = "2.1.0" walkdir = "2.2" +url = "2.5" toml = "1.1.2" remove_dir_all = "1.0.0" base64 = "0.23.0" @@ -54,3 +56,4 @@ env_logger = "0.11.3" rand = "0.10.0" test-case = "3.3.1" tiny_http = "0.12.0" +mockito = "1" diff --git a/src/crates/mod.rs b/src/crates/mod.rs index fb3bb10..97815ab 100644 --- a/src/crates/mod.rs +++ b/src/crates/mod.rs @@ -2,12 +2,16 @@ mod git; mod local; mod registry; -use crate::Workspace; +use crate::{Workspace, crates::registry::CRATES_IO_SPARSE_INDEX}; +use anyhow::{Context as _, Result}; use log::info; use std::path::Path; +use url::Url; -#[cfg(feature = "alternate-registries")] -pub use registry::AlternativeRegistry; +#[cfg(feature = "git-registries")] +pub use registry::GitRegistry; +#[cfg(feature = "git-registries")] +pub use registry::GitRegistry as AlternativeRegistry; trait CrateTrait: std::fmt::Display { fn fetch(&self, workspace: &Workspace) -> anyhow::Result<()>; @@ -25,25 +29,48 @@ enum CrateType { pub struct Crate(CrateType); impl Crate { - /// Load a crate from specified registry. - #[cfg(feature = "alternate-registries")] - pub fn registry(registry: AlternativeRegistry, name: &str, version: &str) -> Self { - Crate(CrateType::Registry(registry::RegistryCrate::new( - registry::Registry::Alternative(registry), + /// Load a crate from a sparse registry index. + /// + /// `index` is the URL of the registry index, with or without Cargo's `sparse+` prefix. + /// Its `config.json` is cached in the workspace after the first fetch. + pub fn sparse_registry(index: U, name: &str, version: &str) -> Result + where + U: TryInto, + >::Error: std::error::Error + Send + Sync + 'static, + { + let index = index.try_into().context("invalid index url")?; + + Ok(Crate(CrateType::Registry(registry::RegistryCrate::new( + registry::Registry::Sparse(registry::normalize_sparse_index(index)?), name, version, - ))) + )))) } - /// Load a crate from the [crates.io registry](https://crates.io). - pub fn crates_io(name: &str, version: &str) -> Self { + /// Load a crate from a Git-indexed registry. + /// + /// For an HTTP sparse index, use [`Crate::sparse_registry`]. + #[cfg(feature = "git-registries")] + pub fn git_registry(registry: GitRegistry, name: &str, version: &str) -> Self { Crate(CrateType::Registry(registry::RegistryCrate::new( - registry::Registry::CratesIo, + registry::Registry::Git(registry), name, version, ))) } + /// Compatibility alias for [`Crate::git_registry`]. + #[cfg(feature = "git-registries")] + pub fn registry(registry: AlternativeRegistry, name: &str, version: &str) -> Self { + Self::git_registry(registry, name, version) + } + + /// Load a crate from the [crates.io registry](https://crates.io). + pub fn crates_io(name: &str, version: &str) -> Self { + Self::sparse_registry(CRATES_IO_SPARSE_INDEX.clone(), name, version) + .expect("we know crates.io index url is valid") + } + /// Load a crate from a git repository. The full URL needed to clone the repo has to be /// provided. pub fn git(url: &str) -> Self { diff --git a/src/crates/registry.rs b/src/crates/registry.rs index 4a58fd2..195ff54 100644 --- a/src/crates/registry.rs +++ b/src/crates/registry.rs @@ -1,28 +1,45 @@ use super::CrateTrait; use crate::Workspace; -#[cfg(feature = "alternate-registries")] use anyhow::Context as _; use flate2::read::GzDecoder; use log::info; -use std::fs::File; -use std::io::{BufReader, BufWriter, Read}; +use std::fs::{self, File}; +use std::io::{self, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; +use std::sync::LazyLock; use tar::Archive; +use url::Url; -static CRATES_ROOT: &str = "https://static.crates.io/crates"; +pub(crate) static CRATES_IO_SPARSE_INDEX: LazyLock = LazyLock::new(|| { + Url::parse("https://index.crates.io/").expect("crates.io sparse index URL is valid") +}); -/// A type for alternative registry as described in rust-lang/rfcs#2141 -#[cfg(feature = "alternate-registries")] -pub struct AlternativeRegistry { +pub(super) fn normalize_sparse_index(mut index: Url) -> anyhow::Result { + if let Some(index_url) = index.as_str().strip_prefix("sparse+") { + index = Url::parse(index_url).context("invalid sparse index URL")?; + } + + if !index.path().ends_with('/') { + let path = format!("{}/", index.path()); + index.set_path(&path); + } + Ok(index) +} + +/// A Git-indexed registry as described in rust-lang/rfcs#2141. +/// +/// For an HTTP sparse index, use [`Crate::sparse_registry`](super::Crate::sparse_registry). +#[cfg(feature = "git-registries")] +pub struct GitRegistry { registry_index: String, key: Option, } -#[cfg(feature = "alternate-registries")] -impl AlternativeRegistry { - /// Registry for specified registry index - pub fn new(registry_index: impl Into) -> AlternativeRegistry { - AlternativeRegistry { +#[cfg(feature = "git-registries")] +impl GitRegistry { + /// Create a Git-indexed registry for the specified registry index URL. + pub fn new(registry_index: impl Into) -> GitRegistry { + GitRegistry { registry_index: registry_index.into(), key: None, } @@ -43,25 +60,34 @@ impl AlternativeRegistry { } pub(crate) enum Registry { - CratesIo, - #[cfg(feature = "alternate-registries")] - Alternative(AlternativeRegistry), + Sparse(Url), + #[cfg(feature = "git-registries")] + Git(GitRegistry), } impl Registry { fn cache_folder(&self) -> String { match self { - Registry::CratesIo => "cratesio-sources".into(), - #[cfg(feature = "alternate-registries")] - Registry::Alternative(alt) => format!("{}-sources", alt.index_folder()), + Registry::Sparse(index) if index == &*CRATES_IO_SPARSE_INDEX => { + "cratesio-sources".into() + } + Registry::Sparse(index) => { + format!( + "{}-sources", + crate::utils::escape_path(index.as_str().as_bytes()) + ) + } + #[cfg(feature = "git-registries")] + Registry::Git(registry) => format!("{}-sources", registry.index_folder()), } } fn name(&self) -> String { match self { - Registry::CratesIo => "crates.io".into(), - #[cfg(feature = "alternate-registries")] - Registry::Alternative(alt) => alt.index().to_string(), + Registry::Sparse(index) if index == &*CRATES_IO_SPARSE_INDEX => "crates.io".into(), + Registry::Sparse(index) => index.as_str().into(), + #[cfg(feature = "git-registries")] + Registry::Git(registry) => registry.index().to_string(), } } } @@ -72,7 +98,6 @@ pub(super) struct RegistryCrate { version: String, } -#[cfg(feature = "alternate-registries")] #[derive(serde::Deserialize)] struct IndexConfig { dl: String, @@ -95,24 +120,74 @@ impl RegistryCrate { .join(format!("{}-{}.crate", self.name, self.version)) } + fn sparse_config(&self, workspace: &Workspace, index: &Url) -> anyhow::Result { + sparse_config(&workspace.cache_dir(), workspace.http_client(), index) + } +} + +/// Generate the path where we cache `config.json` from the given sparse index. +fn sparse_config_path(cache_dir: &Path, index: &Url) -> PathBuf { + cache_dir + .join("registry-index") + .join(crate::utils::escape_path(index.as_str().as_bytes())) + .join("config.json") +} + +/// Fetch & locally cache the `/config.json` file of the given sparse index. +fn sparse_config( + cache_dir: &Path, + http_client: &attohttpc::Session, + index: &Url, +) -> anyhow::Result { + let path = sparse_config_path(cache_dir, index); + match fs::read_to_string(&path) { + Ok(config) => serde_json::from_str(&config).context("registry has invalid config.json"), + Err(err) if err.kind() == io::ErrorKind::NotFound => { + let config_url = index.join("config.json")?; + let config = http_client + .get(config_url.as_str()) + .send()? + .error_for_status()? + .text() + .with_context(|| { + format!("unable to fetch sparse registry config at {config_url}") + })?; + + let parsed = serde_json::from_str::(&config) + .context("registry has invalid config.json")?; + + let parent = path.parent().expect("config path has a parent"); + fs::create_dir_all(parent)?; + + // Write config.json to a temporary path first, then atomically move it into place. + let mut temporary = tempfile::NamedTempFile::new_in(parent)?; + temporary.write_all(config.as_bytes())?; + temporary.persist(&path).map_err(|error| error.error)?; + Ok(parsed) + } + Err(err) => Err(err.into()), + } +} + +impl RegistryCrate { #[allow(unused_variables)] #[cfg_attr(feature = "tracing", tracing::instrument(skip_all, level = "debug"))] - fn fetch_url(&self, workspace: &Workspace) -> anyhow::Result { + fn fetch_url(&self, workspace: &Workspace) -> anyhow::Result { match &self.registry { - Registry::CratesIo => Ok(format!( - "{0}/{1}/{1}-{2}.crate", - CRATES_ROOT, self.name, self.version - )), - #[cfg(feature = "alternate-registries")] - Registry::Alternative(alt) => { + Registry::Sparse(index) => { + let config = self.sparse_config(workspace, index)?; + download_url(&config.dl, &self.name, &self.version) + } + #[cfg(feature = "git-registries")] + Registry::Git(registry) => { let index_path = workspace .cache_dir() .join("registry-index") - .join(alt.index_folder()); + .join(registry.index_folder()); if !index_path.exists() { - let url = alt.index(); + let url = registry.index(); let mut fo = git2::FetchOptions::new(); - if let Some(key) = alt.key.as_deref() { + if let Some(key) = registry.key.as_deref() { fo.remote_callbacks({ let mut callbacks = git2::RemoteCallbacks::new(); callbacks.credentials( @@ -136,30 +211,36 @@ impl RegistryCrate { info!("cloned registry index"); } let config = std::fs::read_to_string(index_path.join("config.json"))?; - let template_url = serde_json::from_str::(&config) - .context("registry has invalid config.json")? - .dl; - let replacements = [("{crate}", &self.name), ("{version}", &self.version)]; - - let url = if replacements - .iter() - .any(|(key, _)| template_url.contains(key)) - { - let mut url = template_url; - for (key, value) in &replacements { - url = url.replace(key, value); - } - url - } else { - format!("{}/{}/{}/download", template_url, self.name, self.version) - }; + let config = serde_json::from_str::(&config) + .context("registry has invalid config.json")?; - Ok(url) + download_url(&config.dl, &self.name, &self.version) } } } } +/// Generate a download url from a `dl` URL template. +/// +/// Replacements are incomplete for now and support only simple use-cases. +/// See https://doc.rust-lang.org/cargo/reference/registry-index.html +fn download_url(template: &str, name: &str, version: &str) -> anyhow::Result { + let replacements = [("{crate}", name), ("{version}", version)]; + if !replacements + .iter() + .any(|(marker, _)| template.contains(marker)) + { + Ok(format!("{}/{}/{}/download", template, name, version).parse()?) + } else { + Ok(replacements + .into_iter() + .fold(template.to_string(), |url, (marker, value)| { + url.replace(marker, value) + }) + .parse()?) + } +} + impl CrateTrait for RegistryCrate { #[cfg_attr( feature = "tracing", @@ -264,3 +345,128 @@ fn unpack_without_first_dir(archive: &mut Archive, path: &Path) -> a Ok(()) } + +#[cfg(test)] +mod tests { + use super::{download_url, normalize_sparse_index, sparse_config, sparse_config_path}; + use crate::crates::registry::CRATES_IO_SPARSE_INDEX; + use mockito::Server; + use std::fs; + use url::Url; + + #[test] + fn fetches_sparse_config_once_and_caches_it_by_normalized_index_url() { + let mut server = Server::new(); + let mock = server + .mock("GET", "/index/config.json") + .with_status(200) + .with_body(r#"{"dl":"https://downloads.example"}"#) + .expect(1) + .create(); + let cache = tempfile::tempdir().unwrap(); + let index = normalize_sparse_index(Url::parse(&format!("{}/index", server.url())).unwrap()) + .unwrap(); + let client = attohttpc::Session::new(); + let first = sparse_config(cache.path(), &client, &index).unwrap(); + let second = sparse_config(cache.path(), &client, &index).unwrap(); + + assert_eq!(first.dl, "https://downloads.example"); + assert_eq!(second.dl, "https://downloads.example"); + mock.assert(); + assert_eq!( + fs::read_to_string(sparse_config_path(cache.path(), &index)).unwrap(), + r#"{"dl":"https://downloads.example"}"# + ); + } + + #[test] + fn invalid_sparse_config_is_not_cached_and_can_be_retried() { + for invalid_body in ["not JSON", r#"{"api":"https://registry.example"}"#] { + let mut server = Server::new(); + let cache = tempfile::tempdir().unwrap(); + let index = + normalize_sparse_index(Url::parse(&format!("{}/index", server.url())).unwrap()) + .unwrap(); + let client = attohttpc::Session::new(); + let invalid = server + .mock("GET", "/index/config.json") + .with_status(200) + .with_body(invalid_body) + .expect(1) + .create(); + + assert!(sparse_config(cache.path(), &client, &index).is_err()); + assert!(!sparse_config_path(cache.path(), &index).exists()); + invalid.assert(); + invalid.remove(); + + let valid = server + .mock("GET", "/index/config.json") + .with_status(200) + .with_body(r#"{"dl":"https://downloads.example"}"#) + .expect(1) + .create(); + + assert_eq!( + sparse_config(cache.path(), &client, &index).unwrap().dl, + "https://downloads.example" + ); + assert!(sparse_config_path(cache.path(), &index).exists()); + valid.assert(); + } + } + + #[test] + fn sparse_registry_returns_error_for_invalid_stripped_url() { + let error = crate::Crate::sparse_registry("sparse+https://", "foo", "1.0.0") + .err() + .expect("invalid sparse URL should return an error"); + assert_eq!( + error.downcast_ref::(), + Some(&url::ParseError::EmptyHost) + ); + } + + #[test] + fn normalizes_sparse_index_urls_and_derives_config_url() { + let index = + normalize_sparse_index(Url::parse("sparse+https://registry.example/index").unwrap()) + .unwrap(); + + assert_eq!(index.as_str(), "https://registry.example/index/"); + assert_eq!( + index.join("config.json").unwrap().as_str(), + "https://registry.example/index/config.json" + ); + } + + #[test] + fn expands_supported_download_url_markers() { + let url = download_url( + "https://registry.example/{crate}/{version}", + "MyCrate", + "1.2.3", + ) + .unwrap(); + + assert_eq!(url.as_str(), "https://registry.example/MyCrate/1.2.3"); + } + + #[test] + fn appends_default_download_path_without_supported_markers() { + assert_eq!( + download_url("https://registry.example", "crate", "2.0.0") + .unwrap() + .as_str(), + "https://registry.example/crate/2.0.0/download" + ); + } + + #[test] + fn crates_io_sparse_url_is_normalized() { + assert_eq!( + normalize_sparse_index(CRATES_IO_SPARSE_INDEX.clone()).unwrap(), + *CRATES_IO_SPARSE_INDEX + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 8dcc4c1..b70c014 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,8 @@ //! //! Rustwide provides some optional features that can be enabled with Cargo: //! +//! * **git-registries**: support Git-indexed registries (enabled by default). +//! * **alternate-registries**: compatibility alias for **git-registries**. //! * **unstable**: allow Rustwide to use unstable Rust and Cargo features. While this feature also //! works on Rust stable it might cause Rustwide to break, and **no stability guarantee is //! present when using it!** @@ -35,9 +37,9 @@ mod workspace; pub use crate::build::{Build, BuildBuilder, BuildDirectory, BuildResult}; pub use crate::cmd::SandboxStatistics; -#[cfg(feature = "alternate-registries")] -pub use crate::crates::AlternativeRegistry; pub use crate::crates::Crate; +#[cfg(feature = "git-registries")] +pub use crate::crates::{AlternativeRegistry, GitRegistry}; pub use crate::prepare::PrepareError; pub use crate::toolchain::Toolchain; pub use crate::workspace::{Workspace, WorkspaceBuilder}; diff --git a/tests/integration/crates_alt.rs b/tests/integration/crates_alt.rs deleted file mode 100644 index 7baca32..0000000 --- a/tests/integration/crates_alt.rs +++ /dev/null @@ -1,14 +0,0 @@ -use rustwide::{AlternativeRegistry, Crate}; - -const INDEX_URL: &str = "https://github.com/rust-lang/staging.crates.io-index"; - -#[test] -fn test_fetch() -> anyhow::Result<()> { - let workspace = crate::utils::init_workspace()?; - - let alt = AlternativeRegistry::new(INDEX_URL); - let krate = Crate::registry(alt, "gcc", "0.3.38"); - krate.fetch(&workspace)?; - - Ok(()) -} diff --git a/tests/integration/crates_git_registry.rs b/tests/integration/crates_git_registry.rs new file mode 100644 index 0000000..5c7bc67 --- /dev/null +++ b/tests/integration/crates_git_registry.rs @@ -0,0 +1,24 @@ +use rustwide::{Crate, GitRegistry}; + +const INDEX_URL: &str = "https://github.com/rust-lang/staging.crates.io-index"; + +#[test] +fn legacy_registry_names_remain_compatible() { + let mut registry = rustwide::AlternativeRegistry::new(INDEX_URL); + registry.authenticate_with_ssh_key("unused test key"); + let registry: GitRegistry = registry; + let legacy = Crate::registry(registry, "gcc", "0.3.38"); + let current = Crate::git_registry(GitRegistry::new(INDEX_URL), "gcc", "0.3.38"); + assert_eq!(legacy.to_string(), current.to_string()); +} + +#[test] +fn test_fetch() -> anyhow::Result<()> { + let workspace = crate::utils::init_workspace()?; + + let registry = GitRegistry::new(INDEX_URL); + let krate = Crate::git_registry(registry, "gcc", "0.3.38"); + krate.fetch(&workspace)?; + + Ok(()) +} diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs index 53dd40b..7ddd5ff 100644 --- a/tests/integration/mod.rs +++ b/tests/integration/mod.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "alternate-registries")] -mod crates_alt; mod crates_git; +#[cfg(feature = "git-registries")] +mod crates_git_registry; mod purge_caches;