From 57950c3cb6e1ccfa9d5e627a671496292dff6f5f Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Wed, 9 Sep 2026 15:38:38 +0200 Subject: [PATCH 1/6] Report as you go --- README.md | 58 +++++++ src/lib.rs | 461 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/main.rs | 13 +- 3 files changed, 498 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 0b87e12..92674c0 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,64 @@ mod tests { In case of `Err` the result will contain a list of errors with detailed information about what went wrong. +#### Reporting progress + +The `Report` is only returned once every test case has been executed. To see the results while the suite is +still running, register a reporter. `StdoutReporter` prints every test case as soon as it has been executed, +followed by the details of the failures and the final result: + +```rust +use doc_assert::DocAssert; +use doc_assert::StdoutReporter; + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn test_docs() { + let result = DocAssert::new() + .with_url("http://localhost:8080") + .with_doc_path("README.md") + .with_reporter(StdoutReporter::new()) + .assert() + .await; + } +} +``` + +```text +2 tests +GET /blog (README.md:12) ✅ +POST /blog (README.md:30) ❌ + +failures: +------------- +POST /blog (README.md:30): expected response code 201, got 500 + +test result: FAILED. 1 passed; 1 failed +``` + +This is what the `doc-assert` binary does, so its output now appears as the test cases are executed instead of +all at once at the end of the run. + +Implement the `Reporter` trait to handle the events yourself, for instance to feed another test runner or to +collect metrics. All of its methods have an empty default implementation, so only the ones of interest need to +be implemented: + +```rust +use doc_assert::{Reporter, TestCaseId}; + +#[derive(Default)] +struct FailedTestCases(Vec); + +impl Reporter for FailedTestCases { + fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { + if result.is_err() { + self.0.push(id.to_string()); + } + } +} +``` + #### Variables In some case we may need to set some value which will be shared between requests. For instance test auth token. diff --git a/src/lib.rs b/src/lib.rs index 6393407..0abb446 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,6 +21,7 @@ use crate::{ use serde_json::Value; use std::collections::HashMap; use std::fmt::Display; +use std::io::Write; use std::vec; mod domain; @@ -55,6 +56,7 @@ pub struct DocAssert<'a> { url: Option<&'a str>, doc_paths: Vec<&'a str>, pub(crate) variables: Variables, + reporter: Option>, } impl<'a> DocAssert<'a> { @@ -74,6 +76,7 @@ impl<'a> DocAssert<'a> { url: None, doc_paths: vec![], variables: Variables::new(), + reporter: None, } } @@ -129,6 +132,26 @@ impl<'a> DocAssert<'a> { self } + /// Sets the reporter notified about every test case as soon as it is executed. + /// + /// Without a reporter the results are available only in the [`Report`] returned by + /// [`DocAssert::assert`], once the whole suite has been executed. Use + /// [`StdoutReporter`] to print the results while they are produced. + /// + /// # Examples + /// + /// ``` + /// # #![allow(unused_mut)] + /// use doc_assert::DocAssert; + /// use doc_assert::StdoutReporter; + /// + /// let mut doc_assert = DocAssert::new().with_reporter(StdoutReporter::new()); + /// ``` + pub fn with_reporter(mut self, reporter: impl Reporter + 'a) -> Self { + self.reporter = Some(Box::new(reporter)); + self + } + /// Execute the assertions /// /// The assertions will be executed and a report will be returned @@ -154,45 +177,61 @@ impl<'a> DocAssert<'a> { /// ``` pub async fn assert(mut self) -> Result { let url = self.url.take().expect("URL is required"); - let mut total_count = 0; + let mut reporter = self + .reporter + .take() + .unwrap_or_else(|| Box::new(NoopReporter)); + + // every documentation file is parsed upfront so that a parsing error is reported + // before any request is made and the number of test cases is known in advance + let mut test_cases = vec![]; + for doc_path in &self.doc_paths { + let parsed = parser::parse(doc_path.to_string()) + .map_err(|e| AssertionError::ParsingError(e.clone()))?; + for tc in parsed { + let id = TestCaseId { + http_method: tc.request.http_method.to_string(), + uri: tc.request.uri.clone(), + doc_path: doc_path.to_string(), + line_number: tc.request.line_number, + }; + test_cases.push((id, tc)); + } + } + + let total_count = test_cases.len(); let mut failed_count = 0; let mut summary = String::new(); let mut failures = String::new(); - for doc_path in self.doc_paths { - let test_cases = parser::parse(doc_path.to_string()) - .map_err(|e| AssertionError::ParsingError(e.clone()))?; - for tc in test_cases { - total_count += 1; - let id = format!( - "{} {} ({}:{})", - tc.request.http_method, tc.request.uri, doc_path, tc.request.line_number - ); - match executor::execute(url, tc, &mut self.variables).await { - Ok(_) => summary.push_str(format!("{} ✅\n", id).as_str()), - Err(err) => { - summary.push_str(format!("{} ❌\n", id).as_str()); - failures.push_str(format!("-------------\n{}: {}\n", id, err).as_str()); - failed_count += 1; - } + reporter.suite_started(total_count); + + for (id, tc) in test_cases { + reporter.test_case_started(&id); + let result = executor::execute(url, tc, &mut self.variables).await; + match &result { + Ok(_) => summary.push_str(format!("{} ✅\n", id).as_str()), + Err(err) => { + summary.push_str(format!("{} ❌\n", id).as_str()); + failures.push_str(format!("-------------\n{}: {}\n", id, err).as_str()); + failed_count += 1; } } + reporter.test_case_finished(&id, &result); } + let report = Report { + total_count, + failed_count, + summary, + failures: (failed_count > 0).then_some(failures), + }; + reporter.suite_finished(&report); + if failed_count == 0 { - Ok(Report { - total_count, - failed_count, - summary, - failures: None, - }) + Ok(report) } else { - Err(AssertionError::TestSuiteError(Report { - total_count, - failed_count, - summary, - failures: Some(failures), - })) + Err(AssertionError::TestSuiteError(report)) } } } @@ -261,6 +300,186 @@ impl Display for Report { } } +impl Report { + /// Total number of executed test cases + pub fn total_count(&self) -> usize { + self.total_count + } + + /// Number of failed test cases + pub fn failed_count(&self) -> usize { + self.failed_count + } + + /// Number of passed test cases + pub fn passed_count(&self) -> usize { + self.total_count - self.failed_count + } + + /// Summary of passed and failed test cases + pub fn summary(&self) -> &str { + &self.summary + } + + /// Detailed information about the failed assertions, `None` if all the test cases passed + pub fn failures(&self) -> Option<&str> { + self.failures.as_deref() + } +} + +/// Identifies a single test case defined in the documentation. +/// +/// It is displayed the same way it appears in the [`Report`] summary, +/// for example `GET /blog (README.md:12)`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestCaseId { + /// HTTP method of the request + pub http_method: String, + /// URI the request is sent to + pub uri: String, + /// Path to the documentation file the test case is defined in + pub doc_path: String, + /// Line number the request is defined at + pub line_number: usize, +} + +impl Display for TestCaseId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} {} ({}:{})", + self.http_method, self.uri, self.doc_path, self.line_number + ) + } +} + +/// Receives the progress of a run while the test cases are being executed. +/// +/// A reporter is registered with [`DocAssert::with_reporter`] and makes the results +/// available as soon as they are produced, instead of waiting for the whole suite to +/// finish. Every method has an empty default implementation, so only the events of +/// interest need to be implemented. +/// +/// See [`StdoutReporter`] for a ready to use implementation printing the progress +/// to the standard output. +/// +/// # Examples +/// +/// ``` +/// use doc_assert::{Reporter, TestCaseId}; +/// +/// #[derive(Default)] +/// struct FailedTestCases(Vec); +/// +/// impl Reporter for FailedTestCases { +/// fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { +/// if result.is_err() { +/// self.0.push(id.to_string()); +/// } +/// } +/// } +/// ``` +pub trait Reporter: Send { + /// Called once, before the first test case is executed. + /// + /// At this point every documentation file has been parsed successfully, + /// so `total_count` is the final number of test cases to be executed. + fn suite_started(&mut self, total_count: usize) { + let _ = total_count; + } + + /// Called just before the request of a test case is sent. + fn test_case_started(&mut self, id: &TestCaseId) { + let _ = id; + } + + /// Called as soon as a test case has been executed. + fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { + let _ = (id, result); + } + + /// Called once, after the last test case has been executed. + fn suite_finished(&mut self, report: &Report) { + let _ = report; + } +} + +/// [`Reporter`] printing the progress of a run to the standard output. +/// +/// Every test case is printed as soon as it has been executed, the details of the +/// failures and the final result follow once the whole suite is done: +/// +/// ```text +/// 2 tests +/// GET /blog (README.md:12) ✅ +/// POST /blog (README.md:30) ❌ +/// +/// failures: +/// ------------- +/// POST /blog (README.md:30): expected response code 201, got 500 +/// +/// test result: FAILED. 1 passed; 1 failed +/// ``` +/// +/// # Examples +/// +/// ``` +/// # #![allow(unused_mut)] +/// use doc_assert::DocAssert; +/// use doc_assert::StdoutReporter; +/// +/// let mut doc_assert = DocAssert::new().with_reporter(StdoutReporter::new()); +/// ``` +#[derive(Debug, Default)] +pub struct StdoutReporter; + +impl StdoutReporter { + /// Constructs a new `StdoutReporter`. + pub fn new() -> Self { + Self + } +} + +impl Reporter for StdoutReporter { + fn suite_started(&mut self, total_count: usize) { + println!("{} tests", total_count); + } + + fn test_case_started(&mut self, id: &TestCaseId) { + // the line is completed by `test_case_finished`, flushing it makes the test case + // currently being executed visible while the request is in flight + print!("{} ", id); + let _ = std::io::stdout().flush(); + } + + fn test_case_finished(&mut self, _id: &TestCaseId, result: &Result<(), String>) { + match result { + Ok(_) => println!("✅"), + Err(_) => println!("❌"), + } + } + + fn suite_finished(&mut self, report: &Report) { + match report.failures() { + Some(failures) => println!( + "\nfailures:\n{}\ntest result: FAILED. {} passed; {} failed", + failures, + report.passed_count(), + report.failed_count() + ), + None => println!( + "\ntest result: PASSED. {} passed; 0 failed", + report.total_count() + ), + } + } +} + +/// [`Reporter`] used when none was registered. +struct NoopReporter; + +impl Reporter for NoopReporter {} + /// Error type for DocAssert run pub enum AssertionError { /// Error parsing the documentation file @@ -516,3 +735,187 @@ fn extract_value(path: &Path, value: &Value) -> Option { } } } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use crate::{DocAssert, Report, Reporter, TestCaseId}; + + /// [`Reporter`] recording the events in the order they were received. + #[derive(Clone, Default)] + struct RecordingReporter { + events: Arc>>, + } + + impl RecordingReporter { + fn record(&self, event: String) { + self.events.lock().unwrap().push(event); + } + + fn events(&self) -> Vec { + self.events.lock().unwrap().clone() + } + } + + impl Reporter for RecordingReporter { + fn suite_started(&mut self, total_count: usize) { + self.record(format!("suite started: {}", total_count)); + } + + fn test_case_started(&mut self, id: &TestCaseId) { + self.record(format!( + "started: {} {}:{}", + id.http_method, id.uri, id.line_number + )); + } + + fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { + let outcome = if result.is_ok() { "passed" } else { "failed" }; + self.record(format!( + "finished: {} {}:{} {}", + id.http_method, id.uri, id.line_number, outcome + )); + } + + fn suite_finished(&mut self, report: &Report) { + self.record(format!( + "suite finished: {} passed, {} failed", + report.passed_count(), + report.failed_count() + )); + } + } + + /// Documentation file removing itself once it goes out of scope. + struct TempDoc { + path: std::path::PathBuf, + } + + impl TempDoc { + fn new(content: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "doc_assert_{}_{}.md", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, content).unwrap(); + Self { path } + } + + fn path(&self) -> &str { + self.path.to_str().unwrap() + } + } + + impl Drop for TempDoc { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } + } + + const DOC: &str = "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n\ + ```docassertrequest\n\ + GET /failing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n"; + + async fn server_with_passing_and_failing_endpoint() -> mockito::ServerGuard { + let mut server = mockito::Server::new_async().await; + server + .mock("GET", "/passing") + .with_status(200) + .create_async() + .await; + server + .mock("GET", "/failing") + .with_status(500) + .create_async() + .await; + server + } + + #[tokio::test] + async fn test_every_test_case_is_reported_as_soon_as_it_is_executed() { + let server = server_with_passing_and_failing_endpoint().await; + let doc = TempDoc::new(DOC); + let reporter = RecordingReporter::default(); + + let result = DocAssert::new() + .with_url(server.url().as_str()) + .with_doc_path(doc.path()) + .with_reporter(reporter.clone()) + .assert() + .await; + + assert!(result.is_err()); + // each test case is reported before the next one is started, which is what makes + // the results visible while the suite is still running + assert_eq!( + reporter.events(), + vec![ + "suite started: 2", + "started: GET /passing:1", + "finished: GET /passing:1 passed", + "started: GET /failing:7", + "finished: GET /failing:7 failed", + "suite finished: 1 passed, 1 failed", + ] + ); + } + + #[tokio::test] + async fn test_report_is_still_returned_when_no_reporter_is_registered() { + let server = server_with_passing_and_failing_endpoint().await; + let doc = TempDoc::new(DOC); + + let result = DocAssert::new() + .with_url(server.url().as_str()) + .with_doc_path(doc.path()) + .assert() + .await; + + match result { + Ok(_) => panic!("expected the suite to fail"), + Err(crate::AssertionError::TestSuiteError(report)) => { + assert_eq!(report.total_count(), 2); + assert_eq!(report.passed_count(), 1); + assert_eq!(report.failed_count(), 1); + assert!(report.summary().contains("GET /passing")); + assert!(report.failures().unwrap().contains("GET /failing")); + } + Err(crate::AssertionError::ParsingError(err)) => panic!("parsing error: {}", err), + } + } + + #[tokio::test] + async fn test_parsing_error_is_reported_before_any_test_case_is_executed() { + let server = server_with_passing_and_failing_endpoint().await; + let doc = TempDoc::new(DOC); + let reporter = RecordingReporter::default(); + + let result = DocAssert::new() + .with_url(server.url().as_str()) + .with_doc_path(doc.path()) + .with_doc_path("this/file/does/not/exist.md") + .with_reporter(reporter.clone()) + .assert() + .await; + + assert!(matches!( + result, + Err(crate::AssertionError::ParsingError(_)) + )); + assert!(reporter.events().is_empty()); + } +} diff --git a/src/main.rs b/src/main.rs index 7909d6f..53125f7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,7 @@ use serde_json::Value; use doc_assert::AssertionError; use doc_assert::DocAssert; +use doc_assert::StdoutReporter; use doc_assert::Variables; #[doc(hidden)] @@ -122,7 +123,8 @@ async fn main() { let mut doc_assert = DocAssert::new() .with_url(cli.url.as_str()) - .with_variables(variables); + .with_variables(variables) + .with_reporter(StdoutReporter::new()); for file in cli.files.iter() { let Some(file) = file.to_str() else { @@ -134,17 +136,18 @@ async fn main() { let result = doc_assert.assert().await; + // the report has already been printed by `StdoutReporter` while the test cases + // were being executed, only the exit code is left to be set here match result { - Ok(report) => { - println!("{}", report); + Ok(_) => { std::process::exit(Code::SUCCESS); } Err(err) => match err { AssertionError::ParsingError(err) => { handle_error!(Code::DOC_PARSING_ERROR, "Error parsing file: {}", err); } - AssertionError::TestSuiteError(report) => { - handle_error!(Code::DOC_ASSERTION_ERROR, "{}", report); + AssertionError::TestSuiteError(_) => { + std::process::exit(Code::DOC_ASSERTION_ERROR); } }, } From d9a7cd0b555c4f45f7c941f6db3e2fe82140925e Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Wed, 9 Sep 2026 17:28:27 +0200 Subject: [PATCH 2/6] refactor --- README.md | 130 ++++--- src/executor.rs | 171 +++++---- src/lib.rs | 906 +++++++++++++++++++----------------------------- src/main.rs | 43 ++- src/report.rs | 367 ++++++++++++++++++++ 5 files changed, 895 insertions(+), 722 deletions(-) create mode 100644 src/report.rs diff --git a/README.md b/README.md index 92674c0..4a94367 100644 --- a/README.md +++ b/README.md @@ -66,49 +66,66 @@ use doc_assert::DocAssert; mod tests { #[tokio::test] async fn test_docs() { - let result = DocAssert::new() + DocAssert::new() .with_url("http://localhost:8080") .with_doc_path("README.md") .assert() .await; - match result { - Ok(report) => { - // handle report - } - Err(err) => { - // handle error - } - } } } ``` -In case of `Err` the result will contain a list of errors with detailed information about what went wrong. +`assert` fails the test if anything went wrong, printing the whole report. If you would rather handle the +outcome yourself, `run` hands it back instead: -#### Reporting progress +```rust +# use doc_assert::DocAssert; +# async fn test() { +let report = DocAssert::new() + .with_url("http://localhost:8080") + .with_doc_path("README.md") + .run() + .await + .unwrap(); + +println!("{} of {} passed", report.passed_count(), report.total_count()); +for result in report.failures() { + println!("{} failed: {}", result.id(), result.failure().unwrap()); +} +# } +``` -The `Report` is only returned once every test case has been executed. To see the results while the suite is -still running, register a reporter. `StdoutReporter` prints every test case as soon as it has been executed, -followed by the details of the failures and the final result: +`run` returns `Err` only when the run could not be performed at all, for instance because a documentation file +could not be parsed. Test cases that failed are not an error: a run that executed its test cases always +returns a `Report`, and `Report::passed` is the verdict. -```rust -use doc_assert::DocAssert; -use doc_assert::StdoutReporter; +#### Seeing the results as they happen -#[cfg(test)] -mod tests { - #[tokio::test] - async fn test_docs() { - let result = DocAssert::new() - .with_url("http://localhost:8080") - .with_doc_path("README.md") - .with_reporter(StdoutReporter::new()) - .assert() - .await; - } +`assert` returns only once every test case has been executed. To handle the results while the run is still +going on, drive it yourself with `start`, which parses the documentation and hands back a `Run`: + +```rust +# use doc_assert::DocAssert; +# async fn test() { +let mut run = DocAssert::new() + .with_url("http://localhost:8080") + .with_doc_path("README.md") + .start() + .unwrap(); + +println!("{} tests", run.total_count()); +while let Some(result) = run.next().await { + println!("{}", result); } + +let report = run.finish(); +println!("{}", report.verdict()); +# } ``` +This is what the `doc-assert` binary does, so its output appears as the test cases are executed rather than +all at once at the end of the run: + ```text 2 tests GET /blog (README.md:12) ✅ @@ -116,30 +133,40 @@ POST /blog (README.md:30) ❌ failures: ------------- -POST /blog (README.md:30): expected response code 201, got 500 +POST /blog (README.md:30): error asserting response from POST /blog defined at line 36: expected response code 201, got 500 test result: FAILED. 1 passed; 1 failed ``` -This is what the `doc-assert` binary does, so its output now appears as the test cases are executed instead of -all at once at the end of the run. - -Implement the `Reporter` trait to handle the events yourself, for instance to feed another test runner or to -collect metrics. All of its methods have an empty default implementation, so only the ones of interest need to -be implemented: +Because you own the loop you can also stop early, time each test case, or race the run against a timeout: ```rust -use doc_assert::{Reporter, TestCaseId}; +# use doc_assert::DocAssert; +# async fn test() { +# let mut run = DocAssert::new().with_url("http://localhost:8080").with_doc_path("README.md").start().unwrap(); +while let Some(result) = run.next().await { + if !result.passed() { + break; // fail fast + } +} +let report = run.finish(); // the test cases executed so far +# } +``` -#[derive(Default)] -struct FailedTestCases(Vec); +A failure is a `Failure`, not a string, so you can render your own output — JUnit XML, TAP, JSON — by +matching on it: -impl Reporter for FailedTestCases { - fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { - if result.is_err() { - self.0.push(id.to_string()); +```rust +use doc_assert::{Failure, Mismatch}; + +fn is_server_error(failure: &Failure) -> bool { + matches!( + failure, + Failure::ResponseMismatch { + cause: Mismatch::StatusCode { actual: 500..=599, .. }, + .. } - } + ) } ``` @@ -150,26 +177,21 @@ In some case we may need to set some value which will be shared between requests We can define variable in the API before we run the tests: ```rust -use doc_assert::DocAssert; +use doc_assert::{DocAssert, Variables}; #[cfg(test)] mod tests { #[tokio::test] async fn test_docs() { - let result = DocAssert::new() + let mut variables = Variables::new(); + variables.insert("auth_token", "some_token"); + + DocAssert::new() .with_url("http://localhost:8080") .with_doc_path("README.md") - .with_variable("auth_token", "some_token") + .with_variables(variables) .assert() .await; - match result { - Ok(report) => { - // handle report - } - Err(err) => { - // handle error - } - } } } ``` diff --git a/src/executor.rs b/src/executor.rs index 0ff672c..3c61700 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -21,124 +21,125 @@ use reqwest::{Body, Client, Method, Response}; use crate::domain::{HttpMethod, Request, TestCase}; use crate::json_diff::path::Path; use crate::json_diff::{diff, CompareMode, Config}; +use crate::report::{Failure, Mismatch}; use crate::Variables; pub(crate) async fn execute( base_url: &str, test_case: TestCase, variables: &mut Variables, -) -> Result<(), String> { +) -> Result<(), Failure> { let mut test_request = test_case.request; variables.replace_request_placeholders(&mut test_request)?; - let test_request_line_number = test_request.line_number; - let http_method = &test_request.http_method; - let uri = &test_request.uri; - let mut test_response = test_case.response; variables.replace_response_placeholders(&mut test_response)?; - let test_response_line_number: usize = test_response.line_number; + + let http_method = test_request.http_method.to_string(); + let uri = test_request.uri.clone(); + let request_line_number = test_request.line_number; + let response_line_number = test_response.line_number; for i in 0..test_response.retries.max_retries { - let response = get_response(base_url, &test_request).await.map_err(|err| { - format!( - "error executing request {} {} defined at line {}: {}", - http_method, uri, test_request_line_number, err - ) - }); + let last_attempt = i == test_response.retries.max_retries - 1; + + let failure = match get_response(base_url, &test_request).await { + Err(reason) => Failure::RequestFailed { + http_method: http_method.clone(), + uri: uri.clone(), + line_number: request_line_number, + reason, + }, + Ok(response) => match assert_response(response, &test_response, variables).await { + Ok(_) => return Ok(()), + Err(cause) => Failure::ResponseMismatch { + http_method: http_method.clone(), + uri: uri.clone(), + line_number: response_line_number, + cause, + }, + }, + }; - match response { - Err(e) => { - if i == test_response.retries.max_retries - 1 { - return Err(e); - } - tokio::time::sleep(Duration::from_millis(test_response.retries.delay)).await; - continue; - } - Ok(response) => { - let assert_response = assert_response(response, &test_response, variables) - .await - .map_err(|err| { - format!( - "error asserting response from {} {} defined at line {}: {}", - http_method, uri, test_response_line_number, err - ) - }); - match assert_response { - Ok(_) => return Ok(()), - Err(e) => { - if i == test_response.retries.max_retries - 1 { - return Err(e); - } - tokio::time::sleep(Duration::from_millis(test_response.retries.delay)) - .await; - continue; - } - } - } + if last_attempt { + return Err(failure); } + tokio::time::sleep(Duration::from_millis(test_response.retries.delay)).await; } - Err("internal error executing request".to_string()) + Err(Failure::NotExecuted) } async fn assert_response( response: Response, test_response: &crate::domain::Response, variables: &mut Variables, -) -> Result<(), String> { +) -> Result<(), Mismatch> { if test_response.code != response.status().as_u16() { - return Err(format!( - "expected response code {}, got {}", - test_response.code, - response.status().as_u16() - )); + return Err(Mismatch::StatusCode { + expected: test_response.code, + actual: response.status().as_u16(), + }); } for (key, val) in test_response.headers.iter() { match response.headers().get(key.as_str()) { Some(test_val) => { if test_val != val.as_str() { - return Err(format!( - "expected header {} to be {}, got {}", - key, - val, - test_val.to_str().unwrap() - )); + return Err(Mismatch::Header { + name: key.clone(), + expected: val.clone(), + actual: test_val.to_str().unwrap().to_string(), + }); } } - None => return Err(format!("expected header {} not found", key)), + None => return Err(Mismatch::MissingHeader { name: key.clone() }), } } if let Some(test_body) = test_response.body.as_ref() { let mut diff_config = Config::new(CompareMode::Strict); for path in test_response.ignore_paths.iter() { - diff_config = diff_config.ignore_path( - Path::from_jsonpath(path.as_str()) - .map_err(|err| format!("invalid path {}: {}", path, err))?, - ); + diff_config = + diff_config.ignore_path(Path::from_jsonpath(path.as_str()).map_err(|err| { + Mismatch::InvalidIgnorePath { + path: path.clone(), + reason: err.to_string(), + } + })?); } for order in test_response.ignore_orders.iter() { - diff_config = diff_config.ignore_order( - Path::from_jsonpath(order.as_str()) - .map_err(|err| format!("invalid path {}: {}", order, err))?, - ); + diff_config = + diff_config.ignore_order(Path::from_jsonpath(order.as_str()).map_err(|err| { + Mismatch::InvalidIgnorePath { + path: order.clone(), + reason: err.to_string(), + } + })?); } - let response_body = response.text().await.map_err(|e| e.to_string())?; - let actual = &serde_json::from_str::(response_body.as_str()) - .map_err(|err| format!("error parsing JSON response from the server: {}", err))?; - let expected = &serde_json::from_str::(test_body.as_str()) - .map_err(|err| format!("error parsing JSON: {}", err))?; + let response_body = + response + .text() + .await + .map_err(|err| Mismatch::UnreadableResponseBody { + reason: err.to_string(), + })?; + let actual = + &serde_json::from_str::(response_body.as_str()).map_err(|err| { + Mismatch::MalformedResponseBody { + reason: err.to_string(), + } + })?; + let expected = + &serde_json::from_str::(test_body.as_str()).map_err(|err| { + Mismatch::MalformedExpectedBody { + reason: err.to_string(), + } + })?; let diff_result = diff(expected, actual, diff_config); if !diff_result.is_empty() { - return Err(format!( - "expected response differs from actual {}", - diff_result - .iter() - .map(|d| d.to_string()) - .collect::>() - .join("\n"), - )); + return Err(Mismatch::Body { + differences: diff_result.iter().map(|d| d.to_string()).collect(), + }); } if !test_response.variables.is_empty() { @@ -242,11 +243,7 @@ mod tests { let result = execute(server.url().as_str(), test_case, &mut variables).await; - match result { - Ok(_) => {} - Err(ref err) => assert_eq!("", err), - } - assert!(result.is_ok()); + assert_eq!(Ok(()), result); } #[tokio::test] @@ -309,14 +306,9 @@ mod tests { let mut variables = Variables::from_json(&json!({"name":"John"})).unwrap(); - let result: Result<(), String> = - execute(server.url().as_str(), test_case, &mut variables).await; + let result = execute(server.url().as_str(), test_case, &mut variables).await; - match result { - Ok(_) => {} - Err(ref err) => assert_eq!("", err), - } - assert!(result.is_ok()); + assert_eq!(Ok(()), result); let test_case = TestCase { request: Request { @@ -342,8 +334,7 @@ mod tests { }, }; - let result: Result<(), String> = - execute(server.url().as_str(), test_case, &mut variables).await; + let result = execute(server.url().as_str(), test_case, &mut variables).await; assert_eq!(Ok(()), result); } diff --git a/src/lib.rs b/src/lib.rs index 0abb446..62a27c7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,524 +13,349 @@ #![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))] #![allow(clippy::while_let_on_iterator)] +// `Failure` describes what went wrong in detail, which makes it larger than the lint likes. +// It is produced at most once per test case, next to an HTTP round trip, so its size is +// irrelevant here and worth the detail it carries. +#![allow(clippy::result_large_err)] use crate::{ - domain::{Request, Response}, + domain::{Request, Response, TestCase}, json_diff::path::{Key, Path}, }; use serde_json::Value; use std::collections::HashMap; use std::fmt::Display; -use std::io::Write; -use std::vec; mod domain; mod executor; mod json_diff; mod parser; +mod report; -/// Builder for the assertions. -/// -/// The builder is used to configure the assertions. +pub use report::{Failure, Mismatch, Report, TestCaseId, TestCaseResult, Verdict}; + +/// Builder for a documentation test run. /// /// # Examples /// /// ``` -/// # #![allow(unused_mut)] /// use doc_assert::DocAssert; -/// use doc_assert::Variables; /// /// async fn test() { -/// // Create Variables for values that will be shared between requests and responses -/// let mut variables = Variables::new(); -/// variables.insert_string("token".to_string(), "abcd".to_string()); -/// // Create a DocAssert builder with the base URL and the path to the documentation file -/// let mut doc_assert = DocAssert::new() +/// DocAssert::new() /// .with_url("http://localhost:8080") -/// .with_doc_path("path/to/README.md"); -/// // Execute the assertions -/// let report = doc_assert.assert().await; +/// .with_doc_path("README.md") +/// .assert() +/// .await; /// } /// ``` -pub struct DocAssert<'a> { - url: Option<&'a str>, - doc_paths: Vec<&'a str>, - pub(crate) variables: Variables, - reporter: Option>, +#[derive(Debug, Default)] +pub struct DocAssert { + url: Option, + doc_paths: Vec, + variables: Variables, } -impl<'a> DocAssert<'a> { +impl DocAssert { /// Constructs a new, empty `DocAssert` builder. /// - /// The builder is used to configure the assertions. + /// A URL and at least one documentation file are required before the run can start. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] /// use doc_assert::DocAssert; - /// let mut doc_assert = DocAssert::new(); + /// let doc_assert = DocAssert::new(); /// ``` pub fn new() -> Self { - Self { - url: None, - doc_paths: vec![], - variables: Variables::new(), - reporter: None, - } + Self::default() } /// Sets the base URL to test against. /// - /// The URL will be used to make the requests. + /// Required. The URL every request of the documentation is sent to. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] /// use doc_assert::DocAssert; - /// let mut doc_assert = DocAssert::new().with_url("http://localhost:8080"); + /// let doc_assert = DocAssert::new().with_url("http://localhost:8080"); /// ``` - pub fn with_url(mut self, url: &'a str) -> Self { - self.url = Some(url); + pub fn with_url(mut self, url: impl Into) -> Self { + self.url = Some(url.into()); self } - /// Sets the path to the documentation file. + /// Adds a documentation file to test. /// - /// The path will be used to parse the documentation. + /// At least one is required. The test cases of every file are executed in the order + /// the files were added. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] /// use doc_assert::DocAssert; - /// let mut doc_assert = DocAssert::new().with_doc_path("path/to/README.md"); + /// let doc_assert = DocAssert::new().with_doc_path("README.md"); /// ``` - pub fn with_doc_path(mut self, doc_path: &'a str) -> Self { - self.doc_paths.push(doc_path); + pub fn with_doc_path(mut self, doc_path: impl Into) -> Self { + self.doc_paths.push(doc_path.into()); self } /// Sets the variables to be used in the assertions. /// - /// The variables will be used to replace the placeholders in the documentation. + /// The variables replace the placeholders in the documentation. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] - /// use doc_assert::DocAssert; - /// use doc_assert::Variables; + /// use doc_assert::{DocAssert, Variables}; /// /// let mut variables = Variables::new(); - /// variables.insert_string("token".to_string(), "abcd".to_string()); - /// let mut doc_assert = DocAssert::new().with_variables(variables); + /// variables.insert("token", "abcd"); + /// let doc_assert = DocAssert::new().with_variables(variables); /// ``` pub fn with_variables(mut self, variables: Variables) -> Self { self.variables = variables; self } - /// Sets the reporter notified about every test case as soon as it is executed. + /// Executes every test case and panics unless all of them passed. + /// + /// This is how `DocAssert` is meant to be used inside a test: anything that goes + /// wrong fails the test, whether it is a documentation file that could not be parsed + /// or a test case that did not pass. Use [`DocAssert::run`] to get the [`Report`] and + /// handle it yourself, or [`DocAssert::start`] to handle every test case as soon as + /// it has been executed. /// - /// Without a reporter the results are available only in the [`Report`] returned by - /// [`DocAssert::assert`], once the whole suite has been executed. Use - /// [`StdoutReporter`] to print the results while they are produced. + /// # Panics + /// + /// Panics if the run could not be performed, or if any test case failed. The whole + /// [`Report`] is the panic message. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] /// use doc_assert::DocAssert; - /// use doc_assert::StdoutReporter; /// - /// let mut doc_assert = DocAssert::new().with_reporter(StdoutReporter::new()); + /// async fn test() { + /// DocAssert::new() + /// .with_url("http://localhost:8080") + /// .with_doc_path("README.md") + /// .assert() + /// .await; + /// } /// ``` - pub fn with_reporter(mut self, reporter: impl Reporter + 'a) -> Self { - self.reporter = Some(Box::new(reporter)); - self + pub async fn assert(self) -> Report { + let report = match self.run().await { + Ok(report) => report, + Err(err) => panic!("{}", err), + }; + report.assert_passed(); + report } - /// Execute the assertions + /// Executes every test case and returns the [`Report`]. /// - /// The assertions will be executed and a report will be returned + /// `Err` means the run could not be performed at all; test cases that failed are + /// reported by the [`Report`] itself, see [`Report::passed`]. /// /// # Examples /// /// ``` - /// # #![allow(unused_mut)] /// use doc_assert::DocAssert; + /// /// async fn test() { - /// let mut doc_assert = DocAssert::new() + /// let report = DocAssert::new() /// .with_url("http://localhost:8080") - /// .with_doc_path("path/to/README.md"); - /// match doc_assert.assert().await { - /// Ok(report) => { - /// // handle success - /// } - /// Err(err) => { - /// // handle error - /// } - /// }; + /// .with_doc_path("README.md") + /// .run() + /// .await + /// .unwrap(); + /// + /// println!("{} of {} passed", report.passed_count(), report.total_count()); /// } /// ``` - pub async fn assert(mut self) -> Result { - let url = self.url.take().expect("URL is required"); - let mut reporter = self - .reporter - .take() - .unwrap_or_else(|| Box::new(NoopReporter)); + pub async fn run(self) -> Result { + Ok(self.start()?.run_to_end().await) + } + + /// Parses the documentation and prepares the run without executing anything yet. + /// + /// Use this to drive the run yourself and handle every test case as soon as it has + /// been executed. + /// + /// # Examples + /// + /// ``` + /// use doc_assert::DocAssert; + /// + /// async fn test() { + /// let mut run = DocAssert::new() + /// .with_url("http://localhost:8080") + /// .with_doc_path("README.md") + /// .start() + /// .unwrap(); + /// + /// println!("{} tests", run.total_count()); + /// while let Some(result) = run.next().await { + /// println!("{}", result); + /// } + /// + /// let report = run.finish(); + /// println!("{}", report.verdict()); + /// } + /// ``` + pub fn start(self) -> Result { + let url = self.url.ok_or(Error::NoUrl)?; + if self.doc_paths.is_empty() { + return Err(Error::NoDocuments); + } // every documentation file is parsed upfront so that a parsing error is reported // before any request is made and the number of test cases is known in advance - let mut test_cases = vec![]; + let mut pending = vec![]; for doc_path in &self.doc_paths { - let parsed = parser::parse(doc_path.to_string()) - .map_err(|e| AssertionError::ParsingError(e.clone()))?; - for tc in parsed { + let test_cases = parser::parse(doc_path.clone()).map_err(|reason| Error::Parse { + doc_path: doc_path.clone(), + reason, + })?; + for test_case in test_cases { let id = TestCaseId { - http_method: tc.request.http_method.to_string(), - uri: tc.request.uri.clone(), - doc_path: doc_path.to_string(), - line_number: tc.request.line_number, + http_method: test_case.request.http_method.to_string(), + uri: test_case.request.uri.clone(), + doc_path: doc_path.clone(), + line_number: test_case.request.line_number, }; - test_cases.push((id, tc)); - } - } - - let total_count = test_cases.len(); - let mut failed_count = 0; - let mut summary = String::new(); - let mut failures = String::new(); - - reporter.suite_started(total_count); - - for (id, tc) in test_cases { - reporter.test_case_started(&id); - let result = executor::execute(url, tc, &mut self.variables).await; - match &result { - Ok(_) => summary.push_str(format!("{} ✅\n", id).as_str()), - Err(err) => { - summary.push_str(format!("{} ❌\n", id).as_str()); - failures.push_str(format!("-------------\n{}: {}\n", id, err).as_str()); - failed_count += 1; - } + pending.push((id, test_case)); } - reporter.test_case_finished(&id, &result); } - let report = Report { - total_count, - failed_count, - summary, - failures: (failed_count > 0).then_some(failures), - }; - reporter.suite_finished(&report); - - if failed_count == 0 { - Ok(report) - } else { - Err(AssertionError::TestSuiteError(report)) - } - } -} - -impl<'a> Default for DocAssert<'a> { - fn default() -> Self { - Self::new() + Ok(Run { + url, + variables: self.variables, + total_count: pending.len(), + pending: pending.into_iter(), + results: vec![], + }) } } -/// Report of the assertions +/// A run in progress, returned by [`DocAssert::start`]. /// -/// The report contains the total number of tests, the number of failed tests, -/// a summary of passed and failed tests, and detailed information about -/// the failed assertions. +/// The test cases are executed one by one, as [`Run::next`] is called, so their results +/// are available while the run is still going on. They are executed in the order they +/// appear in the documentation because a test case may use variables extracted from the +/// responses of the previous ones. /// -/// # Examples -/// -/// ``` -/// # #![allow(unused_mut)] -/// use doc_assert::DocAssert; -/// use doc_assert::Variables; -/// -/// async fn test() { -/// let mut doc_assert = DocAssert::new() -/// .with_url("http://localhost:8080") -/// .with_doc_path("path/to/README.md"); -/// match doc_assert.assert().await { -/// Ok(report) => { -/// println!("{}", report); -/// } -/// Err(err) => { -/// // handle error -/// } -/// }; -/// } -pub struct Report { - /// Total number of tests +/// Dropping a `Run` cancels it; the test cases that were already executed are lost with +/// it, so call [`Run::finish`] to get the [`Report`] of a partial run. +#[derive(Debug)] +pub struct Run { + url: String, + variables: Variables, + pending: std::vec::IntoIter<(TestCaseId, TestCase)>, + results: Vec, total_count: usize, - /// Number of failed tests - failed_count: usize, - /// Summary of passed and failed tests - summary: String, - /// Detailed information about the failed assertions - failures: Option, -} - -impl Display for Report { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.failures { - Some(failures) => write!( - f, - "{} tests\n{}\nfailures:\n{}\ntest result: FAILED. {} passed; {} failed", - self.total_count, - self.summary, - failures, - self.total_count - self.failed_count, - self.failed_count - ), - None => write!( - f, - "{} tests\n{}\ntest result: PASSED. {} passed; 0 failed", - self.total_count, self.summary, self.total_count - ), - } - } } -impl Report { - /// Total number of executed test cases +impl Run { + /// Total number of test cases in the run pub fn total_count(&self) -> usize { self.total_count } - /// Number of failed test cases - pub fn failed_count(&self) -> usize { - self.failed_count + /// Number of test cases executed so far + pub fn completed_count(&self) -> usize { + self.results.len() } - /// Number of passed test cases - pub fn passed_count(&self) -> usize { - self.total_count - self.failed_count + /// Executes the next test case, `None` once every one of them has been executed. + pub async fn next(&mut self) -> Option { + let (id, test_case) = self.pending.next()?; + let failure = executor::execute(&self.url, test_case, &mut self.variables) + .await + .err(); + let result = TestCaseResult::new(id, failure); + self.results.push(result.clone()); + Some(result) } - /// Summary of passed and failed test cases - pub fn summary(&self) -> &str { - &self.summary + /// Executes every remaining test case and returns the [`Report`]. + pub async fn run_to_end(mut self) -> Report { + while self.next().await.is_some() {} + self.finish() } - /// Detailed information about the failed assertions, `None` if all the test cases passed - pub fn failures(&self) -> Option<&str> { - self.failures.as_deref() + /// Returns the [`Report`] of the test cases executed so far. + pub fn finish(self) -> Report { + Report::new(self.results) } } -/// Identifies a single test case defined in the documentation. +/// The run could not be performed. /// -/// It is displayed the same way it appears in the [`Report`] summary, -/// for example `GET /blog (README.md:12)`. +/// This is not how a failed test case is reported; a run that executed its test cases +/// always produces a [`Report`], whether they passed or not. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct TestCaseId { - /// HTTP method of the request - pub http_method: String, - /// URI the request is sent to - pub uri: String, - /// Path to the documentation file the test case is defined in - pub doc_path: String, - /// Line number the request is defined at - pub line_number: usize, +pub enum Error { + /// No URL to test against was given + NoUrl, + /// No documentation file was given + NoDocuments, + /// A documentation file could not be read or parsed + Parse { + /// Path to the documentation file + doc_path: String, + /// Why it could not be read or parsed + reason: String, + }, } -impl Display for TestCaseId { +impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{} {} ({}:{})", - self.http_method, self.uri, self.doc_path, self.line_number - ) - } -} - -/// Receives the progress of a run while the test cases are being executed. -/// -/// A reporter is registered with [`DocAssert::with_reporter`] and makes the results -/// available as soon as they are produced, instead of waiting for the whole suite to -/// finish. Every method has an empty default implementation, so only the events of -/// interest need to be implemented. -/// -/// See [`StdoutReporter`] for a ready to use implementation printing the progress -/// to the standard output. -/// -/// # Examples -/// -/// ``` -/// use doc_assert::{Reporter, TestCaseId}; -/// -/// #[derive(Default)] -/// struct FailedTestCases(Vec); -/// -/// impl Reporter for FailedTestCases { -/// fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { -/// if result.is_err() { -/// self.0.push(id.to_string()); -/// } -/// } -/// } -/// ``` -pub trait Reporter: Send { - /// Called once, before the first test case is executed. - /// - /// At this point every documentation file has been parsed successfully, - /// so `total_count` is the final number of test cases to be executed. - fn suite_started(&mut self, total_count: usize) { - let _ = total_count; - } - - /// Called just before the request of a test case is sent. - fn test_case_started(&mut self, id: &TestCaseId) { - let _ = id; - } - - /// Called as soon as a test case has been executed. - fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { - let _ = (id, result); - } - - /// Called once, after the last test case has been executed. - fn suite_finished(&mut self, report: &Report) { - let _ = report; - } -} - -/// [`Reporter`] printing the progress of a run to the standard output. -/// -/// Every test case is printed as soon as it has been executed, the details of the -/// failures and the final result follow once the whole suite is done: -/// -/// ```text -/// 2 tests -/// GET /blog (README.md:12) ✅ -/// POST /blog (README.md:30) ❌ -/// -/// failures: -/// ------------- -/// POST /blog (README.md:30): expected response code 201, got 500 -/// -/// test result: FAILED. 1 passed; 1 failed -/// ``` -/// -/// # Examples -/// -/// ``` -/// # #![allow(unused_mut)] -/// use doc_assert::DocAssert; -/// use doc_assert::StdoutReporter; -/// -/// let mut doc_assert = DocAssert::new().with_reporter(StdoutReporter::new()); -/// ``` -#[derive(Debug, Default)] -pub struct StdoutReporter; - -impl StdoutReporter { - /// Constructs a new `StdoutReporter`. - pub fn new() -> Self { - Self - } -} - -impl Reporter for StdoutReporter { - fn suite_started(&mut self, total_count: usize) { - println!("{} tests", total_count); - } - - fn test_case_started(&mut self, id: &TestCaseId) { - // the line is completed by `test_case_finished`, flushing it makes the test case - // currently being executed visible while the request is in flight - print!("{} ", id); - let _ = std::io::stdout().flush(); - } - - fn test_case_finished(&mut self, _id: &TestCaseId, result: &Result<(), String>) { - match result { - Ok(_) => println!("✅"), - Err(_) => println!("❌"), - } - } - - fn suite_finished(&mut self, report: &Report) { - match report.failures() { - Some(failures) => println!( - "\nfailures:\n{}\ntest result: FAILED. {} passed; {} failed", - failures, - report.passed_count(), - report.failed_count() - ), - None => println!( - "\ntest result: PASSED. {} passed; 0 failed", - report.total_count() - ), + match self { + Error::NoUrl => write!(f, "no URL to test against"), + Error::NoDocuments => write!(f, "no documentation file to test"), + Error::Parse { doc_path, reason } => { + write!(f, "error parsing {}: {}", doc_path, reason) + } } } } -/// [`Reporter`] used when none was registered. -struct NoopReporter; - -impl Reporter for NoopReporter {} - -/// Error type for DocAssert run -pub enum AssertionError { - /// Error parsing the documentation file - ParsingError(String), - /// Error executing tests - TestSuiteError(Report), -} +impl std::error::Error for Error {} /// Variables to be used in the request and response bodies. /// -/// The variables are used to replace placeholders in the request -/// and response bodies in case some values need to be shared between requests and responses. +/// The variables replace placeholders in the request and response bodies, in case some +/// values need to be shared between requests. /// /// # Examples /// -/// Variables can be passed one by one with specified type: -/// -/// ``` -/// # use doc_assert::Variables; -/// # use serde_json::Value; -/// let mut variables = Variables::new(); -/// variables.insert_string("name".to_string(), "John".to_string()); -/// variables.insert_int("age".to_string(), 30); -/// ``` -/// -/// A `Value` can be passed directly: +/// Variables can be inserted one by one: /// /// ``` /// # use doc_assert::Variables; -/// # use serde_json::Value; /// let mut variables = Variables::new(); -/// variables.insert_value("name".to_string(), Value::String("John".to_string())); -/// variables.insert_value("age".to_string(), Value::Number(serde_json::Number::from(30))); +/// variables.insert("name", "John"); +/// variables.insert("age", 30); /// ``` /// /// Alternatively, they can be passed as a JSON object: /// /// ``` /// # use doc_assert::Variables; -/// # use serde_json::Value; /// let json = r#"{"name": "John", "age": 30}"#; /// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); /// ``` -/// #[derive(Debug, Clone, Default)] pub struct Variables { map: HashMap, } impl Variables { - /// Constructs a new `Variables`. + /// Constructs a new, empty `Variables`. /// /// # Examples /// @@ -550,29 +375,23 @@ impl Variables { /// /// ``` /// # use doc_assert::Variables; - /// # use serde_json::Value; /// let json = r#"{"name": "John", "age": 30}"#; /// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); /// ``` pub fn from_json(json: &Value) -> Result { - let mut map = HashMap::new(); - - if let Value::Object(obj) = json { - for (key, value) in obj { - map.insert(key.clone(), value.clone()); - } - } else { - return Err("variables must be an object".to_string()); + match json { + Value::Object(obj) => Ok(Self { + map: obj.clone().into_iter().collect(), + }), + _ => Err("variables must be an object".to_string()), } - - Ok(Self { map }) } - /// Inserts a `Value` into the `Variables`. + /// Inserts a variable, overwriting any variable of the same name. /// - /// This can be useful when more complex types are needed. - /// Since `Variables` is a wrapper around `HashMap` if you insert duplicate - /// keys the value will be overwritten. + /// Anything a `serde_json::Value` can be built from is accepted, which covers the + /// strings, numbers and booleans a documentation usually needs. Pass a `Value` + /// itself for the types it does not cover. /// /// # Examples /// @@ -580,90 +399,23 @@ impl Variables { /// # use doc_assert::Variables; /// # use serde_json::Value; /// let mut variables = Variables::new(); - /// variables.insert_value("name".to_string(), Value::String("John".to_string())); - /// ``` - pub fn insert_value(&mut self, name: String, value: Value) { - self.map.insert(name, value); - } - - /// Inserts a `String` into the `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let mut variables = Variables::new(); - /// variables.insert_string("name".to_string(), "John".to_string()); + /// variables.insert("name", "John"); + /// variables.insert("age", 30); + /// variables.insert("is_adult", true); + /// variables.insert("nickname", Value::Null); /// ``` - pub fn insert_string(&mut self, name: String, value: String) { - self.map.insert(name, Value::String(value)); - } - - /// Inserts an `i64` into the `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let mut variables = Variables::new(); - /// variables.insert_int("age".to_string(), 30); - /// ``` - pub fn insert_int(&mut self, name: String, value: i64) { - self.map - .insert(name, Value::Number(serde_json::Number::from(value))); - } - - /// Inserts an `f64` into the `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let mut variables = Variables::new(); - /// variables.insert_float("age".to_string(), 30.0); - /// ``` - pub fn insert_float(&mut self, name: String, value: f64) { - self.map.insert( - name, - Value::Number(serde_json::Number::from_f64(value).unwrap()), - ); - } - - /// Inserts a `bool` into the `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let mut variables = Variables::new(); - /// variables.insert_bool("is_adult".to_string(), true); - /// ``` - pub fn insert_bool(&mut self, name: String, value: bool) { - self.map.insert(name, Value::Bool(value)); - } - - /// Inserts a `null` into the `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let mut variables = Variables::new(); - /// variables.insert_null("name".to_string()); - /// ``` - pub fn insert_null(&mut self, name: String) { - self.map.insert(name, Value::Null); + pub fn insert(&mut self, name: impl Into, value: impl Into) { + self.map.insert(name.into(), value.into()); } pub(crate) fn obtain_from_response( &mut self, response: &Value, variable_templates: &HashMap, - ) -> Result<(), String> { + ) -> Result<(), Mismatch> { for (name, path) in variable_templates { - let value = extract_value(path, response).ok_or_else(|| { - format!("variable template {} not found in the response body", name) - })?; + let value = extract_value(path, response) + .ok_or_else(|| Mismatch::VariableNotFound { name: name.clone() })?; self.map.insert(name.clone(), value); } @@ -671,7 +423,7 @@ impl Variables { Ok(()) } - fn replace_placeholders(&self, input: &mut String, trim_quotes: bool) -> Result<(), String> { + fn replace_placeholders(&self, input: &mut String, trim_quotes: bool) -> Result<(), Failure> { for (name, value) in &self.map { let placeholder = format!("`{}`", name); let value_str = value.to_string(); @@ -686,13 +438,15 @@ impl Variables { } if input.contains('`') { - return Err(format!("unresolved variable placeholders in {}", input)); + return Err(Failure::UnresolvedVariables { + input: input.clone(), + }); } Ok(()) } - pub(crate) fn replace_request_placeholders(&self, input: &mut Request) -> Result<(), String> { + pub(crate) fn replace_request_placeholders(&self, input: &mut Request) -> Result<(), Failure> { self.replace_placeholders(&mut input.uri, true)?; if let Some(body) = &mut input.body { @@ -706,7 +460,10 @@ impl Variables { Ok(()) } - pub(crate) fn replace_response_placeholders(&self, input: &mut Response) -> Result<(), String> { + pub(crate) fn replace_response_placeholders( + &self, + input: &mut Response, + ) -> Result<(), Failure> { if let Some(body) = &mut input.body { self.replace_placeholders(body, false)?; } @@ -738,54 +495,7 @@ fn extract_value(path: &Path, value: &Value) -> Option { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; - - use crate::{DocAssert, Report, Reporter, TestCaseId}; - - /// [`Reporter`] recording the events in the order they were received. - #[derive(Clone, Default)] - struct RecordingReporter { - events: Arc>>, - } - - impl RecordingReporter { - fn record(&self, event: String) { - self.events.lock().unwrap().push(event); - } - - fn events(&self) -> Vec { - self.events.lock().unwrap().clone() - } - } - - impl Reporter for RecordingReporter { - fn suite_started(&mut self, total_count: usize) { - self.record(format!("suite started: {}", total_count)); - } - - fn test_case_started(&mut self, id: &TestCaseId) { - self.record(format!( - "started: {} {}:{}", - id.http_method, id.uri, id.line_number - )); - } - - fn test_case_finished(&mut self, id: &TestCaseId, result: &Result<(), String>) { - let outcome = if result.is_ok() { "passed" } else { "failed" }; - self.record(format!( - "finished: {} {}:{} {}", - id.http_method, id.uri, id.line_number, outcome - )); - } - - fn suite_finished(&mut self, report: &Report) { - self.record(format!( - "suite finished: {} passed, {} failed", - report.passed_count(), - report.failed_count() - )); - } - } + use crate::{DocAssert, Error, Failure, Mismatch}; /// Documentation file removing itself once it goes out of scope. struct TempDoc { @@ -817,6 +527,7 @@ mod tests { } } + /// One test case hitting `/passing`, then one hitting `/failing`. const DOC: &str = "```docassertrequest\n\ GET /passing\n\ ```\n\ @@ -830,7 +541,7 @@ mod tests { HTTP 200\n\ ```\n"; - async fn server_with_passing_and_failing_endpoint() -> mockito::ServerGuard { + async fn server() -> mockito::ServerGuard { let mut server = mockito::Server::new_async().await; server .mock("GET", "/passing") @@ -846,76 +557,153 @@ mod tests { } #[tokio::test] - async fn test_every_test_case_is_reported_as_soon_as_it_is_executed() { - let server = server_with_passing_and_failing_endpoint().await; + async fn test_every_test_case_is_available_as_soon_as_it_is_executed() { + let server = server().await; let doc = TempDoc::new(DOC); - let reporter = RecordingReporter::default(); - let result = DocAssert::new() - .with_url(server.url().as_str()) + let mut run = DocAssert::new() + .with_url(server.url()) .with_doc_path(doc.path()) - .with_reporter(reporter.clone()) - .assert() - .await; + .start() + .unwrap(); - assert!(result.is_err()); - // each test case is reported before the next one is started, which is what makes - // the results visible while the suite is still running - assert_eq!( - reporter.events(), - vec![ - "suite started: 2", - "started: GET /passing:1", - "finished: GET /passing:1 passed", - "started: GET /failing:7", - "finished: GET /failing:7 failed", - "suite finished: 1 passed, 1 failed", - ] - ); + // the count is known before anything has been executed + assert_eq!(2, run.total_count()); + assert_eq!(0, run.completed_count()); + + let first = run.next().await.unwrap(); + assert!(first.passed()); + assert_eq!("GET", first.id().http_method); + assert_eq!("/passing", first.id().uri); + // the second test case has not been executed at the point the first is returned + assert_eq!(1, run.completed_count()); + + let second = run.next().await.unwrap(); + assert!(!second.passed()); + assert_eq!("/failing", second.id().uri); + + assert!(run.next().await.is_none()); + + let report = run.finish(); + assert_eq!(2, report.total_count()); + assert_eq!(1, report.passed_count()); + assert_eq!(1, report.failed_count()); + assert!(!report.passed()); + assert_eq!(1, report.failures().count()); } #[tokio::test] - async fn test_report_is_still_returned_when_no_reporter_is_registered() { - let server = server_with_passing_and_failing_endpoint().await; + async fn test_failure_carries_the_reason_it_failed() { + let server = server().await; let doc = TempDoc::new(DOC); - let result = DocAssert::new() - .with_url(server.url().as_str()) + let report = DocAssert::new() + .with_url(server.url()) .with_doc_path(doc.path()) - .assert() - .await; - - match result { - Ok(_) => panic!("expected the suite to fail"), - Err(crate::AssertionError::TestSuiteError(report)) => { - assert_eq!(report.total_count(), 2); - assert_eq!(report.passed_count(), 1); - assert_eq!(report.failed_count(), 1); - assert!(report.summary().contains("GET /passing")); - assert!(report.failures().unwrap().contains("GET /failing")); - } - Err(crate::AssertionError::ParsingError(err)) => panic!("parsing error: {}", err), + .run() + .await + .unwrap(); + + let failure = report.failures().next().unwrap().failure().unwrap(); + match failure { + Failure::ResponseMismatch { cause, .. } => assert_eq!( + &Mismatch::StatusCode { + expected: 200, + actual: 500 + }, + cause + ), + other => panic!("unexpected failure: {:?}", other), } } #[tokio::test] - async fn test_parsing_error_is_reported_before_any_test_case_is_executed() { - let server = server_with_passing_and_failing_endpoint().await; + async fn test_run_can_be_stopped_early_and_still_report() { + let server = server().await; let doc = TempDoc::new(DOC); - let reporter = RecordingReporter::default(); - let result = DocAssert::new() - .with_url(server.url().as_str()) + let mut run = DocAssert::new() + .with_url(server.url()) .with_doc_path(doc.path()) + .start() + .unwrap(); + + run.next().await.unwrap(); + let report = run.finish(); + + // only the test cases that were executed are reported + assert_eq!(1, report.total_count()); + assert!(report.passed()); + } + + #[test] + fn test_a_run_without_documentation_is_an_error() { + assert_eq!( + Some(Error::NoDocuments), + DocAssert::new() + .with_url("http://localhost:8080") + .start() + .err() + ); + } + + #[test] + fn test_a_run_without_a_url_is_an_error() { + assert_eq!( + Some(Error::NoUrl), + DocAssert::new().with_doc_path("README.md").start().err() + ); + } + + #[test] + fn test_parsing_error_names_the_file_it_comes_from() { + let err = DocAssert::new() + .with_url("http://localhost:8080") .with_doc_path("this/file/does/not/exist.md") - .with_reporter(reporter.clone()) - .assert() - .await; + .start() + .unwrap_err(); - assert!(matches!( - result, - Err(crate::AssertionError::ParsingError(_)) - )); - assert!(reporter.events().is_empty()); + match err { + Error::Parse { doc_path, .. } => assert_eq!("this/file/does/not/exist.md", doc_path), + other => panic!("unexpected error: {:?}", other), + } + } + + #[test] + fn test_report_renders_the_same_thing_as_the_streamed_output() { + let report = crate::Report::new(vec![ + crate::TestCaseResult::new( + crate::TestCaseId { + http_method: "GET".to_string(), + uri: "/blog".to_string(), + doc_path: "README.md".to_string(), + line_number: 12, + }, + None, + ), + crate::TestCaseResult::new( + crate::TestCaseId { + http_method: "POST".to_string(), + uri: "/blog".to_string(), + doc_path: "README.md".to_string(), + line_number: 30, + }, + Some(Failure::NotExecuted), + ), + ]); + + // what the binary prints line by line has to add up to what `Report` displays + let streamed = format!( + "{} tests\n{}\n{}\n", + report.total_count(), + report + .results() + .iter() + .map(|r| r.to_string()) + .collect::>() + .join("\n"), + report.verdict() + ); + assert_eq!(format!("{}\n", report), streamed); } } diff --git a/src/main.rs b/src/main.rs index 53125f7..1a85ff3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,9 +18,8 @@ use std::str::FromStr; use clap::Parser; use serde_json::Value; -use doc_assert::AssertionError; use doc_assert::DocAssert; -use doc_assert::StdoutReporter; +use doc_assert::Error; use doc_assert::Variables; #[doc(hidden)] @@ -123,8 +122,7 @@ async fn main() { let mut doc_assert = DocAssert::new() .with_url(cli.url.as_str()) - .with_variables(variables) - .with_reporter(StdoutReporter::new()); + .with_variables(variables); for file in cli.files.iter() { let Some(file) = file.to_str() else { @@ -134,21 +132,28 @@ async fn main() { doc_assert = doc_assert.with_doc_path(file); } - let result = doc_assert.assert().await; - - // the report has already been printed by `StdoutReporter` while the test cases - // were being executed, only the exit code is left to be set here - match result { - Ok(_) => { - std::process::exit(Code::SUCCESS); + let mut run = match doc_assert.start() { + Ok(run) => run, + Err(err @ (Error::NoUrl | Error::NoDocuments)) => { + handle_error!(Code::INVALID_ARGUMENT, "Error: {}", err); } - Err(err) => match err { - AssertionError::ParsingError(err) => { - handle_error!(Code::DOC_PARSING_ERROR, "Error parsing file: {}", err); - } - AssertionError::TestSuiteError(_) => { - std::process::exit(Code::DOC_ASSERTION_ERROR); - } - }, + Err(err) => { + handle_error!(Code::DOC_PARSING_ERROR, "Error: {}", err); + } + }; + + // the test cases are printed as they are executed, only the failures and the final + // result are left to print once the run is over + println!("{} tests", run.total_count()); + while let Some(result) = run.next().await { + println!("{}", result); + } + + let report = run.finish(); + println!("{}", report.verdict()); + + if report.passed() { + std::process::exit(Code::SUCCESS); } + std::process::exit(Code::DOC_ASSERTION_ERROR); } diff --git a/src/report.rs b/src/report.rs new file mode 100644 index 0000000..793ebea --- /dev/null +++ b/src/report.rs @@ -0,0 +1,367 @@ +// Copyright 2024 The DocAssert Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! What a run produced: the outcome of every test case and the way it is rendered. + +use std::fmt::Display; + +/// Identifies a single test case defined in the documentation. +/// +/// Displayed the way it appears in a [`Report`], for example `GET /blog (README.md:12)`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestCaseId { + /// HTTP method of the request + pub http_method: String, + /// URI the request is sent to + pub uri: String, + /// Path to the documentation file the test case is defined in + pub doc_path: String, + /// Line number the request is defined at + pub line_number: usize, +} + +impl Display for TestCaseId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} {} ({}:{})", + self.http_method, self.uri, self.doc_path, self.line_number + ) + } +} + +/// Reason a test case did not pass. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Failure { + /// A placeholder in the request or the expected response could not be resolved, + /// usually because no variable of that name was defined or extracted earlier + UnresolvedVariables { + /// The URI, body or header value the placeholders were left in + input: String, + }, + /// The request could not be sent + RequestFailed { + /// HTTP method of the request + http_method: String, + /// URI the request was sent to + uri: String, + /// Line number the request is defined at + line_number: usize, + /// What went wrong while sending it + reason: String, + }, + /// The response did not match the one described in the documentation + ResponseMismatch { + /// HTTP method of the request + http_method: String, + /// URI the request was sent to + uri: String, + /// Line number the expected response is defined at + line_number: usize, + /// How the response differed + cause: Mismatch, + }, + /// The test case was never executed, which happens when its retry policy + /// allows no attempt at all + NotExecuted, +} + +impl Display for Failure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Failure::UnresolvedVariables { input } => { + write!(f, "unresolved variable placeholders in {}", input) + } + Failure::RequestFailed { + http_method, + uri, + line_number, + reason, + } => write!( + f, + "error executing request {} {} defined at line {}: {}", + http_method, uri, line_number, reason + ), + Failure::ResponseMismatch { + http_method, + uri, + line_number, + cause, + } => write!( + f, + "error asserting response from {} {} defined at line {}: {}", + http_method, uri, line_number, cause + ), + Failure::NotExecuted => write!(f, "internal error executing request"), + } + } +} + +impl std::error::Error for Failure {} + +/// How a response differed from the one described in the documentation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Mismatch { + /// The status code was not the expected one + StatusCode { + /// Status code the documentation describes + expected: u16, + /// Status code the server responded with + actual: u16, + }, + /// A header held a different value than the expected one + Header { + /// Name of the header + name: String, + /// Value the documentation describes + expected: String, + /// Value the server responded with + actual: String, + }, + /// A header described in the documentation was not present in the response + MissingHeader { + /// Name of the header + name: String, + }, + /// The body differed from the expected one + Body { + /// One entry per difference found, in the order they were found + differences: Vec, + }, + /// A variable the documentation extracts from the response was not found in it + VariableNotFound { + /// Name of the variable + name: String, + }, + /// An `[ignore]` or `[ignore-order]` JSONPath could not be parsed + InvalidIgnorePath { + /// The JSONPath as written in the documentation + path: String, + /// Why it could not be parsed + reason: String, + }, + /// The response body could not be read + UnreadableResponseBody { + /// Why it could not be read + reason: String, + }, + /// The response body was not valid JSON + MalformedResponseBody { + /// Why it could not be parsed + reason: String, + }, + /// The body described in the documentation was not valid JSON + MalformedExpectedBody { + /// Why it could not be parsed + reason: String, + }, +} + +impl Display for Mismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Mismatch::StatusCode { expected, actual } => { + write!(f, "expected response code {}, got {}", expected, actual) + } + Mismatch::Header { + name, + expected, + actual, + } => write!( + f, + "expected header {} to be {}, got {}", + name, expected, actual + ), + Mismatch::MissingHeader { name } => write!(f, "expected header {} not found", name), + Mismatch::Body { differences } => write!( + f, + "expected response differs from actual {}", + differences.join("\n") + ), + Mismatch::VariableNotFound { name } => write!( + f, + "variable template {} not found in the response body", + name + ), + Mismatch::InvalidIgnorePath { path, reason } => { + write!(f, "invalid path {}: {}", path, reason) + } + Mismatch::UnreadableResponseBody { reason } => write!(f, "{}", reason), + Mismatch::MalformedResponseBody { reason } => { + write!(f, "error parsing JSON response from the server: {}", reason) + } + Mismatch::MalformedExpectedBody { reason } => { + write!(f, "error parsing JSON: {}", reason) + } + } + } +} + +/// Outcome of a single executed test case. +/// +/// Displayed as the line it takes in a [`Report`], for example `GET /blog (README.md:12) ✅`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TestCaseResult { + id: TestCaseId, + failure: Option, +} + +impl TestCaseResult { + pub(crate) fn new(id: TestCaseId, failure: Option) -> Self { + Self { id, failure } + } + + /// The test case this is the outcome of + pub fn id(&self) -> &TestCaseId { + &self.id + } + + /// Whether the test case passed + pub fn passed(&self) -> bool { + self.failure.is_none() + } + + /// Why the test case failed, `None` if it passed + pub fn failure(&self) -> Option<&Failure> { + self.failure.as_ref() + } +} + +impl Display for TestCaseResult { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mark = if self.passed() { "✅" } else { "❌" }; + write!(f, "{} {}", self.id, mark) + } +} + +/// Outcome of a whole run. +/// +/// A report is produced whenever the run completed, whether the test cases passed or not, +/// so it always describes what happened. Use [`Report::passed`] for the verdict. +/// +/// Displaying a report renders the whole thing: the number of test cases, one line per +/// test case, the details of the failures and the verdict. When the test cases are +/// printed while they are executed, print [`Report::verdict`] instead so that the lines +/// are not repeated. +/// +/// # Examples +/// +/// ``` +/// # use doc_assert::DocAssert; +/// async fn test() { +/// let report = DocAssert::new() +/// .with_url("http://localhost:8080") +/// .with_doc_path("README.md") +/// .run() +/// .await +/// .unwrap(); +/// println!("{}", report); +/// assert!(report.passed()); +/// } +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Report { + results: Vec, +} + +impl Report { + pub(crate) fn new(results: Vec) -> Self { + Self { results } + } + + /// Outcome of every test case, in the order they were executed + pub fn results(&self) -> &[TestCaseResult] { + &self.results + } + + /// Outcome of every test case that failed + pub fn failures(&self) -> impl Iterator { + self.results.iter().filter(|r| !r.passed()) + } + + /// Total number of executed test cases + pub fn total_count(&self) -> usize { + self.results.len() + } + + /// Number of test cases that passed + pub fn passed_count(&self) -> usize { + self.results.iter().filter(|r| r.passed()).count() + } + + /// Number of test cases that failed + pub fn failed_count(&self) -> usize { + self.total_count() - self.passed_count() + } + + /// Whether every test case passed + pub fn passed(&self) -> bool { + self.failed_count() == 0 + } + + /// The details of the failures followed by the final result. + /// + /// This is everything a [`Report`] displays except the number of test cases and the + /// line of every test case, so it is what is left to print once the test cases have + /// been printed as they were executed. + pub fn verdict(&self) -> Verdict<'_> { + Verdict(self) + } + + /// Panics with the whole report if any test case failed. + /// + /// Meant for use inside a test, where a failed test case should fail the test. + /// + /// # Panics + /// + /// Panics if [`Report::passed`] is `false`. + pub fn assert_passed(&self) { + assert!(self.passed(), "{}", self); + } +} + +impl Display for Report { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "{} tests", self.total_count())?; + for result in &self.results { + writeln!(f, "{}", result)?; + } + write!(f, "{}", self.verdict()) + } +} + +/// The details of the failures followed by the final result, returned by [`Report::verdict`]. +#[derive(Debug)] +pub struct Verdict<'a>(&'a Report); + +impl Display for Verdict<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let report = self.0; + if !report.passed() { + writeln!(f, "\nfailures:")?; + for result in report.failures() { + // `failure` is always set on a result that did not pass + if let Some(failure) = result.failure() { + writeln!(f, "-------------\n{}: {}", result.id(), failure)?; + } + } + } + write!( + f, + "\ntest result: {}. {} passed; {} failed", + if report.passed() { "PASSED" } else { "FAILED" }, + report.passed_count(), + report.failed_count() + ) + } +} From 1dcdd2bbbd788b90968965ab276e89029b5ab6bf Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Thu, 10 Sep 2026 09:54:36 +0200 Subject: [PATCH 3/6] refactor --- README.md | 14 +++--- src/executor.rs | 23 +++------- src/lib.rs | 87 +++++++++++++++++++++--------------- src/main.rs | 34 +------------- src/parser.rs | 35 ++++++++++++++- src/report.rs | 116 ++++++++++++++++++++++++++---------------------- 6 files changed, 165 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 4a94367..f2d3754 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ let report = DocAssert::new() .await .unwrap(); -println!("{} of {} passed", report.passed_count(), report.total_count()); +println!("{} of {} passed", report.passed_count(), report.executed_count()); for result in report.failures() { println!("{} failed: {}", result.id(), result.failure().unwrap()); } @@ -119,7 +119,7 @@ while let Some(result) = run.next().await { } let report = run.finish(); -println!("{}", report.verdict()); +println!("{}", report.summary()); # } ``` @@ -133,12 +133,14 @@ POST /blog (README.md:30) ❌ failures: ------------- -POST /blog (README.md:30): error asserting response from POST /blog defined at line 36: expected response code 201, got 500 +POST /blog (README.md:30): response at line 36: expected response code 201, got 500 test result: FAILED. 1 passed; 1 failed ``` -Because you own the loop you can also stop early, time each test case, or race the run against a timeout: +Because you own the loop you can also stop early or time each test case. Stop it between calls, though: +`next` is not cancellation safe, so a future that is dropped part way through — raced against a timeout, or +selected on — loses the test case it had already taken off the queue. ```rust # use doc_assert::DocAssert; @@ -153,8 +155,8 @@ let report = run.finish(); // the test cases executed so far # } ``` -A failure is a `Failure`, not a string, so you can render your own output — JUnit XML, TAP, JSON — by -matching on it: +A failure is a `Failure`, not a string, and it carries only what its `TestCaseId` does not already say, so +you can render your own output — JUnit XML, TAP, JSON — by matching on it: ```rust use doc_assert::{Failure, Mismatch}; diff --git a/src/executor.rs b/src/executor.rs index 3c61700..5d572b7 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -35,39 +35,30 @@ pub(crate) async fn execute( let mut test_response = test_case.response; variables.replace_response_placeholders(&mut test_response)?; - let http_method = test_request.http_method.to_string(); - let uri = test_request.uri.clone(); - let request_line_number = test_request.line_number; let response_line_number = test_response.line_number; - for i in 0..test_response.retries.max_retries { - let last_attempt = i == test_response.retries.max_retries - 1; + let mut attempt = 0; + loop { + attempt += 1; let failure = match get_response(base_url, &test_request).await { - Err(reason) => Failure::RequestFailed { - http_method: http_method.clone(), - uri: uri.clone(), - line_number: request_line_number, - reason, - }, + Err(reason) => Failure::RequestFailed { reason }, Ok(response) => match assert_response(response, &test_response, variables).await { Ok(_) => return Ok(()), Err(cause) => Failure::ResponseMismatch { - http_method: http_method.clone(), - uri: uri.clone(), line_number: response_line_number, cause, }, }, }; - if last_attempt { + // the policy counts attempts rather than extra tries, and the parser rejects one + // that allows none, so the request is always attempted at least once + if attempt >= test_response.retries.max_retries { return Err(failure); } tokio::time::sleep(Duration::from_millis(test_response.retries.delay)).await; } - - Err(Failure::NotExecuted) } async fn assert_response( diff --git a/src/lib.rs b/src/lib.rs index 62a27c7..8a0e775 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,10 +13,6 @@ #![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))] #![allow(clippy::while_let_on_iterator)] -// `Failure` describes what went wrong in detail, which makes it larger than the lint likes. -// It is produced at most once per test case, next to an HTTP round trip, so its size is -// irrelevant here and worth the detail it carries. -#![allow(clippy::result_large_err)] use crate::{ domain::{Request, Response, TestCase}, @@ -32,7 +28,7 @@ mod json_diff; mod parser; mod report; -pub use report::{Failure, Mismatch, Report, TestCaseId, TestCaseResult, Verdict}; +pub use report::{Failure, Mismatch, Report, Summary, TestCaseId, TestCaseResult}; /// Builder for a documentation test run. /// @@ -173,7 +169,7 @@ impl DocAssert { /// .await /// .unwrap(); /// - /// println!("{} of {} passed", report.passed_count(), report.total_count()); + /// println!("{} of {} passed", report.passed_count(), report.executed_count()); /// } /// ``` pub async fn run(self) -> Result { @@ -203,7 +199,7 @@ impl DocAssert { /// } /// /// let report = run.finish(); - /// println!("{}", report.verdict()); + /// println!("{}", report.summary()); /// } /// ``` pub fn start(self) -> Result { @@ -221,12 +217,12 @@ impl DocAssert { reason, })?; for test_case in test_cases { - let id = TestCaseId { - http_method: test_case.request.http_method.to_string(), - uri: test_case.request.uri.clone(), - doc_path: doc_path.clone(), - line_number: test_case.request.line_number, - }; + let id = TestCaseId::new( + test_case.request.http_method.to_string(), + test_case.request.uri.clone(), + doc_path.clone(), + test_case.request.line_number, + ); pending.push((id, test_case)); } } @@ -260,7 +256,10 @@ pub struct Run { } impl Run { - /// Total number of test cases in the run + /// Total number of test cases the documentation defines. + /// + /// This is how many test cases the run would execute if it were driven to the end; + /// a run that is stopped early reports fewer, see [`Report::executed_count`]. pub fn total_count(&self) -> usize { self.total_count } @@ -271,6 +270,13 @@ impl Run { } /// Executes the next test case, `None` once every one of them has been executed. + /// + /// # Cancellation + /// + /// This is not cancellation safe. The test case is taken off the queue before the + /// request is sent, so dropping the returned future part way through — racing it + /// against a timeout, or selecting on it — loses that test case: it is neither + /// retried nor reported. Drive it to completion, and stop the run between calls. pub async fn next(&mut self) -> Option { let (id, test_case) = self.pending.next()?; let failure = executor::execute(&self.url, test_case, &mut self.variables) @@ -310,6 +316,8 @@ pub enum Error { /// Why it could not be read or parsed reason: String, }, + /// The variables were not given as a JSON object + VariablesNotAnObject, } impl Display for Error { @@ -320,6 +328,7 @@ impl Display for Error { Error::Parse { doc_path, reason } => { write!(f, "error parsing {}: {}", doc_path, reason) } + Error::VariablesNotAnObject => write!(f, "variables must be a JSON object"), } } } @@ -378,12 +387,12 @@ impl Variables { /// let json = r#"{"name": "John", "age": 30}"#; /// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); /// ``` - pub fn from_json(json: &Value) -> Result { + pub fn from_json(json: &Value) -> Result { match json { Value::Object(obj) => Ok(Self { map: obj.clone().into_iter().collect(), }), - _ => Err("variables must be an object".to_string()), + _ => Err(Error::VariablesNotAnObject), } } @@ -573,19 +582,19 @@ mod tests { let first = run.next().await.unwrap(); assert!(first.passed()); - assert_eq!("GET", first.id().http_method); - assert_eq!("/passing", first.id().uri); + assert_eq!("GET", first.id().http_method()); + assert_eq!("/passing", first.id().uri()); // the second test case has not been executed at the point the first is returned assert_eq!(1, run.completed_count()); let second = run.next().await.unwrap(); assert!(!second.passed()); - assert_eq!("/failing", second.id().uri); + assert_eq!("/failing", second.id().uri()); assert!(run.next().await.is_none()); let report = run.finish(); - assert_eq!(2, report.total_count()); + assert_eq!(2, report.executed_count()); assert_eq!(1, report.passed_count()); assert_eq!(1, report.failed_count()); assert!(!report.passed()); @@ -632,7 +641,7 @@ mod tests { let report = run.finish(); // only the test cases that were executed are reported - assert_eq!(1, report.total_count()); + assert_eq!(1, report.executed_count()); assert!(report.passed()); } @@ -673,36 +682,42 @@ mod tests { fn test_report_renders_the_same_thing_as_the_streamed_output() { let report = crate::Report::new(vec![ crate::TestCaseResult::new( - crate::TestCaseId { - http_method: "GET".to_string(), - uri: "/blog".to_string(), - doc_path: "README.md".to_string(), - line_number: 12, - }, + crate::TestCaseId::new( + "GET".to_string(), + "/blog".to_string(), + "README.md".to_string(), + 12, + ), None, ), crate::TestCaseResult::new( - crate::TestCaseId { - http_method: "POST".to_string(), - uri: "/blog".to_string(), - doc_path: "README.md".to_string(), - line_number: 30, - }, - Some(Failure::NotExecuted), + crate::TestCaseId::new( + "POST".to_string(), + "/blog".to_string(), + "README.md".to_string(), + 30, + ), + Some(Failure::ResponseMismatch { + line_number: 36, + cause: Mismatch::StatusCode { + expected: 201, + actual: 500, + }, + }), ), ]); // what the binary prints line by line has to add up to what `Report` displays let streamed = format!( "{} tests\n{}\n{}\n", - report.total_count(), + report.executed_count(), report .results() .iter() .map(|r| r.to_string()) .collect::>() .join("\n"), - report.verdict() + report.summary() ); assert_eq!(format!("{}\n", report), streamed); } diff --git a/src/main.rs b/src/main.rs index 1a85ff3..c983cdc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,7 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::convert::From; use std::path::PathBuf; use std::str::FromStr; @@ -22,24 +21,6 @@ use doc_assert::DocAssert; use doc_assert::Error; use doc_assert::Variables; -#[doc(hidden)] -#[macro_export] -macro_rules! write_to_file { - ($writer:expr, $msg:expr) => { - if let Err(err) = writeln!($writer, $msg) { - eprintln!("Error: {}", err); - std::process::exit(Code::INTERNAL_ERROR); - } - }; - - ($writer:expr, $msg:expr, $($arg:tt)*) => { - if let Err(err) = writeln!($writer, $msg, $($arg)*) { - eprintln!("Error: {}", err); - std::process::exit(Code::INTERNAL_ERROR); - } - }; -} - #[doc(hidden)] #[derive(Debug, Clone)] struct JSONVars(Value); @@ -54,7 +35,6 @@ impl FromStr for JSONVars { } #[doc(hidden)] -#[macro_export] macro_rules! handle_error { ($code:expr, $msg:expr, $($arg:tt)*) => { println!($msg, $($arg)*); @@ -98,18 +78,6 @@ struct Cli { async fn main() { let cli = Cli::parse(); - match &cli.variables { - Some(vars) => { - if let Value::String(_) = vars.0 { - handle_error!( - Code::INVALID_ARGUMENT, - "Error: Variables must be a JSON object" - ); - } - } - None => {} - } - let variables = match cli.variables { Some(vars) => match Variables::from_json(&vars.0) { Ok(vars) => vars, @@ -150,7 +118,7 @@ async fn main() { } let report = run.finish(); - println!("{}", report.verdict()); + println!("{}", report.summary()); if report.passed() { std::process::exit(Code::SUCCESS); diff --git a/src/parser.rs b/src/parser.rs index b6d057c..7b7d424 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -154,6 +154,15 @@ fn get_retry_policy(line: &str) -> Result { .parse::() .map_err(|e| format!("invalid max_retries: {}", e))?; + // a policy allowing no attempt at all would define a test case that can never be + // executed, so it is rejected here rather than reported as a failure later on + if max_retries == 0 { + return Err(format!( + "max_retries must be at least 1, got 0 in: {}", + line + )); + } + let delay = caps .name("delay") .ok_or(format!("invalid retry properties: {}", line))? @@ -267,7 +276,10 @@ fn get_headers_and_body( #[cfg(test)] mod tests { - use crate::{domain::RetryPolicy, parser::parse}; + use crate::{ + domain::RetryPolicy, + parser::{get_retry_policy, parse}, + }; #[test] fn test_parse() { @@ -319,4 +331,25 @@ mod tests { } ) } + + #[test] + fn test_a_retry_policy_allowing_no_attempt_is_rejected() { + let err = get_retry_policy("[retry]: # (0, 100)").unwrap_err(); + assert!( + err.contains("max_retries must be at least 1"), + "unexpected error: {}", + err + ); + } + + #[test] + fn test_a_retry_policy_of_one_attempt_is_accepted() { + assert_eq!( + Ok(RetryPolicy { + max_retries: 1, + delay: 100 + }), + get_retry_policy("[retry]: # (1, 100)") + ); + } } diff --git a/src/report.rs b/src/report.rs index 793ebea..c884ce7 100644 --- a/src/report.rs +++ b/src/report.rs @@ -20,14 +20,46 @@ use std::fmt::Display; /// Displayed the way it appears in a [`Report`], for example `GET /blog (README.md:12)`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TestCaseId { + http_method: String, + uri: String, + doc_path: String, + line_number: usize, +} + +impl TestCaseId { + pub(crate) fn new( + http_method: String, + uri: String, + doc_path: String, + line_number: usize, + ) -> Self { + Self { + http_method, + uri, + doc_path, + line_number, + } + } + /// HTTP method of the request - pub http_method: String, + pub fn http_method(&self) -> &str { + &self.http_method + } + /// URI the request is sent to - pub uri: String, + pub fn uri(&self) -> &str { + &self.uri + } + /// Path to the documentation file the test case is defined in - pub doc_path: String, + pub fn doc_path(&self) -> &str { + &self.doc_path + } + /// Line number the request is defined at - pub line_number: usize, + pub fn line_number(&self) -> usize { + self.line_number + } } impl Display for TestCaseId { @@ -41,7 +73,11 @@ impl Display for TestCaseId { } /// Reason a test case did not pass. +/// +/// A `Failure` only carries what the [`TestCaseId`] of the enclosing [`TestCaseResult`] +/// does not already say, so rendering the two together does not repeat the request. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum Failure { /// A placeholder in the request or the expected response could not be resolved, /// usually because no variable of that name was defined or extracted earlier @@ -51,29 +87,17 @@ pub enum Failure { }, /// The request could not be sent RequestFailed { - /// HTTP method of the request - http_method: String, - /// URI the request was sent to - uri: String, - /// Line number the request is defined at - line_number: usize, /// What went wrong while sending it reason: String, }, /// The response did not match the one described in the documentation ResponseMismatch { - /// HTTP method of the request - http_method: String, - /// URI the request was sent to - uri: String, - /// Line number the expected response is defined at + /// Line number the expected response is defined at, which is not the line the + /// request is defined at, see [`TestCaseId::line_number`] line_number: usize, /// How the response differed cause: Mismatch, }, - /// The test case was never executed, which happens when its retry policy - /// allows no attempt at all - NotExecuted, } impl Display for Failure { @@ -82,27 +106,10 @@ impl Display for Failure { Failure::UnresolvedVariables { input } => { write!(f, "unresolved variable placeholders in {}", input) } - Failure::RequestFailed { - http_method, - uri, - line_number, - reason, - } => write!( - f, - "error executing request {} {} defined at line {}: {}", - http_method, uri, line_number, reason - ), - Failure::ResponseMismatch { - http_method, - uri, - line_number, - cause, - } => write!( - f, - "error asserting response from {} {} defined at line {}: {}", - http_method, uri, line_number, cause - ), - Failure::NotExecuted => write!(f, "internal error executing request"), + Failure::RequestFailed { reason } => write!(f, "request failed: {}", reason), + Failure::ResponseMismatch { line_number, cause } => { + write!(f, "response at line {}: {}", line_number, cause) + } } } } @@ -111,6 +118,7 @@ impl std::error::Error for Failure {} /// How a response differed from the one described in the documentation. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum Mismatch { /// The status code was not the expected one StatusCode { @@ -250,8 +258,8 @@ impl Display for TestCaseResult { /// so it always describes what happened. Use [`Report::passed`] for the verdict. /// /// Displaying a report renders the whole thing: the number of test cases, one line per -/// test case, the details of the failures and the verdict. When the test cases are -/// printed while they are executed, print [`Report::verdict`] instead so that the lines +/// test case, the details of the failures and the final result. When the test cases are +/// printed while they are executed, print [`Report::summary`] instead so that the lines /// are not repeated. /// /// # Examples @@ -289,8 +297,12 @@ impl Report { self.results.iter().filter(|r| !r.passed()) } - /// Total number of executed test cases - pub fn total_count(&self) -> usize { + /// Number of test cases that were executed. + /// + /// A run that was stopped early only reports the test cases it got to, so this is + /// not necessarily the number of test cases the documentation defines, which is + /// what [`Run::total_count`](crate::Run::total_count) gives. + pub fn executed_count(&self) -> usize { self.results.len() } @@ -301,10 +313,10 @@ impl Report { /// Number of test cases that failed pub fn failed_count(&self) -> usize { - self.total_count() - self.passed_count() + self.executed_count() - self.passed_count() } - /// Whether every test case passed + /// Whether every executed test case passed pub fn passed(&self) -> bool { self.failed_count() == 0 } @@ -314,8 +326,8 @@ impl Report { /// This is everything a [`Report`] displays except the number of test cases and the /// line of every test case, so it is what is left to print once the test cases have /// been printed as they were executed. - pub fn verdict(&self) -> Verdict<'_> { - Verdict(self) + pub fn summary(&self) -> Summary<'_> { + Summary(self) } /// Panics with the whole report if any test case failed. @@ -332,19 +344,19 @@ impl Report { impl Display for Report { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "{} tests", self.total_count())?; + writeln!(f, "{} tests", self.executed_count())?; for result in &self.results { writeln!(f, "{}", result)?; } - write!(f, "{}", self.verdict()) + write!(f, "{}", self.summary()) } } -/// The details of the failures followed by the final result, returned by [`Report::verdict`]. +/// The details of the failures followed by the final result, returned by [`Report::summary`]. #[derive(Debug)] -pub struct Verdict<'a>(&'a Report); +pub struct Summary<'a>(&'a Report); -impl Display for Verdict<'_> { +impl Display for Summary<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let report = self.0; if !report.passed() { From 7b4a37129a9da44eb9d88c074b2f5cacd37aa23a Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Fri, 2 Oct 2026 16:01:43 +0200 Subject: [PATCH 4/6] Slim lib.rs down to the public entry point - Move Variables and the placeholder substitution into variables.rs - Take the URL in DocAssert::new instead of with_url, dropping Error::NoUrl - Return Option from Variables::from_json, dropping Error::VariablesNotAnObject - Fold Run::run_to_end into DocAssert::run - Move the public API tests to tests/api.rs, the render test to report.rs - Drop the crate-wide while_let_on_iterator allow --- README.md | 14 +- src/lib.rs | 468 +++-------------------------------------------- src/main.rs | 15 +- src/parser.rs | 2 +- src/report.rs | 52 +++++- src/variables.rs | 187 +++++++++++++++++++ tests/api.rs | 174 ++++++++++++++++++ 7 files changed, 451 insertions(+), 461 deletions(-) create mode 100644 src/variables.rs create mode 100644 tests/api.rs diff --git a/README.md b/README.md index f2d3754..cf0c738 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,7 @@ use doc_assert::DocAssert; mod tests { #[tokio::test] async fn test_docs() { - DocAssert::new() - .with_url("http://localhost:8080") + DocAssert::new("http://localhost:8080") .with_doc_path("README.md") .assert() .await; @@ -81,8 +80,7 @@ outcome yourself, `run` hands it back instead: ```rust # use doc_assert::DocAssert; # async fn test() { -let report = DocAssert::new() - .with_url("http://localhost:8080") +let report = DocAssert::new("http://localhost:8080") .with_doc_path("README.md") .run() .await @@ -107,8 +105,7 @@ going on, drive it yourself with `start`, which parses the documentation and han ```rust # use doc_assert::DocAssert; # async fn test() { -let mut run = DocAssert::new() - .with_url("http://localhost:8080") +let mut run = DocAssert::new("http://localhost:8080") .with_doc_path("README.md") .start() .unwrap(); @@ -145,7 +142,7 @@ selected on — loses the test case it had already taken off the queue. ```rust # use doc_assert::DocAssert; # async fn test() { -# let mut run = DocAssert::new().with_url("http://localhost:8080").with_doc_path("README.md").start().unwrap(); +# let mut run = DocAssert::new("http://localhost:8080").with_doc_path("README.md").start().unwrap(); while let Some(result) = run.next().await { if !result.passed() { break; // fail fast @@ -188,8 +185,7 @@ mod tests { let mut variables = Variables::new(); variables.insert("auth_token", "some_token"); - DocAssert::new() - .with_url("http://localhost:8080") + DocAssert::new("http://localhost:8080") .with_doc_path("README.md") .with_variables(variables) .assert() diff --git a/src/lib.rs b/src/lib.rs index 8a0e775..e8b43c4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,14 +12,8 @@ // limitations under the License. #![doc = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/README.md"))] -#![allow(clippy::while_let_on_iterator)] -use crate::{ - domain::{Request, Response, TestCase}, - json_diff::path::{Key, Path}, -}; -use serde_json::Value; -use std::collections::HashMap; +use crate::domain::TestCase; use std::fmt::Display; mod domain; @@ -27,8 +21,10 @@ mod executor; mod json_diff; mod parser; mod report; +mod variables; pub use report::{Failure, Mismatch, Report, Summary, TestCaseId, TestCaseResult}; +pub use variables::Variables; /// Builder for a documentation test run. /// @@ -38,48 +34,38 @@ pub use report::{Failure, Mismatch, Report, Summary, TestCaseId, TestCaseResult} /// use doc_assert::DocAssert; /// /// async fn test() { -/// DocAssert::new() -/// .with_url("http://localhost:8080") +/// DocAssert::new("http://localhost:8080") /// .with_doc_path("README.md") /// .assert() /// .await; /// } /// ``` -#[derive(Debug, Default)] +#[derive(Debug)] pub struct DocAssert { - url: Option, + url: String, doc_paths: Vec, variables: Variables, } impl DocAssert { - /// Constructs a new, empty `DocAssert` builder. + /// Constructs a `DocAssert` builder testing against `url`. /// - /// A URL and at least one documentation file are required before the run can start. + /// `url` is the base URL every request of the documentation is sent to. At least one + /// documentation file is also required before the run can start, see + /// [`DocAssert::with_doc_path`]. /// /// # Examples /// /// ``` /// use doc_assert::DocAssert; - /// let doc_assert = DocAssert::new(); - /// ``` - pub fn new() -> Self { - Self::default() - } - - /// Sets the base URL to test against. - /// - /// Required. The URL every request of the documentation is sent to. - /// - /// # Examples - /// + /// let doc_assert = DocAssert::new("http://localhost:8080"); /// ``` - /// use doc_assert::DocAssert; - /// let doc_assert = DocAssert::new().with_url("http://localhost:8080"); - /// ``` - pub fn with_url(mut self, url: impl Into) -> Self { - self.url = Some(url.into()); - self + pub fn new(url: impl Into) -> Self { + Self { + url: url.into(), + doc_paths: vec![], + variables: Variables::new(), + } } /// Adds a documentation file to test. @@ -91,7 +77,7 @@ impl DocAssert { /// /// ``` /// use doc_assert::DocAssert; - /// let doc_assert = DocAssert::new().with_doc_path("README.md"); + /// let doc_assert = DocAssert::new("http://localhost:8080").with_doc_path("README.md"); /// ``` pub fn with_doc_path(mut self, doc_path: impl Into) -> Self { self.doc_paths.push(doc_path.into()); @@ -109,7 +95,7 @@ impl DocAssert { /// /// let mut variables = Variables::new(); /// variables.insert("token", "abcd"); - /// let doc_assert = DocAssert::new().with_variables(variables); + /// let doc_assert = DocAssert::new("http://localhost:8080").with_variables(variables); /// ``` pub fn with_variables(mut self, variables: Variables) -> Self { self.variables = variables; @@ -135,8 +121,7 @@ impl DocAssert { /// use doc_assert::DocAssert; /// /// async fn test() { - /// DocAssert::new() - /// .with_url("http://localhost:8080") + /// DocAssert::new("http://localhost:8080") /// .with_doc_path("README.md") /// .assert() /// .await; @@ -162,8 +147,7 @@ impl DocAssert { /// use doc_assert::DocAssert; /// /// async fn test() { - /// let report = DocAssert::new() - /// .with_url("http://localhost:8080") + /// let report = DocAssert::new("http://localhost:8080") /// .with_doc_path("README.md") /// .run() /// .await @@ -173,7 +157,9 @@ impl DocAssert { /// } /// ``` pub async fn run(self) -> Result { - Ok(self.start()?.run_to_end().await) + let mut run = self.start()?; + while run.next().await.is_some() {} + Ok(run.finish()) } /// Parses the documentation and prepares the run without executing anything yet. @@ -187,8 +173,7 @@ impl DocAssert { /// use doc_assert::DocAssert; /// /// async fn test() { - /// let mut run = DocAssert::new() - /// .with_url("http://localhost:8080") + /// let mut run = DocAssert::new("http://localhost:8080") /// .with_doc_path("README.md") /// .start() /// .unwrap(); @@ -203,7 +188,6 @@ impl DocAssert { /// } /// ``` pub fn start(self) -> Result { - let url = self.url.ok_or(Error::NoUrl)?; if self.doc_paths.is_empty() { return Err(Error::NoDocuments); } @@ -228,7 +212,7 @@ impl DocAssert { } Ok(Run { - url, + url: self.url, variables: self.variables, total_count: pending.len(), pending: pending.into_iter(), @@ -287,12 +271,6 @@ impl Run { Some(result) } - /// Executes every remaining test case and returns the [`Report`]. - pub async fn run_to_end(mut self) -> Report { - while self.next().await.is_some() {} - self.finish() - } - /// Returns the [`Report`] of the test cases executed so far. pub fn finish(self) -> Report { Report::new(self.results) @@ -305,8 +283,6 @@ impl Run { /// always produces a [`Report`], whether they passed or not. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Error { - /// No URL to test against was given - NoUrl, /// No documentation file was given NoDocuments, /// A documentation file could not be read or parsed @@ -316,409 +292,17 @@ pub enum Error { /// Why it could not be read or parsed reason: String, }, - /// The variables were not given as a JSON object - VariablesNotAnObject, } impl Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Error::NoUrl => write!(f, "no URL to test against"), Error::NoDocuments => write!(f, "no documentation file to test"), Error::Parse { doc_path, reason } => { write!(f, "error parsing {}: {}", doc_path, reason) } - Error::VariablesNotAnObject => write!(f, "variables must be a JSON object"), } } } impl std::error::Error for Error {} - -/// Variables to be used in the request and response bodies. -/// -/// The variables replace placeholders in the request and response bodies, in case some -/// values need to be shared between requests. -/// -/// # Examples -/// -/// Variables can be inserted one by one: -/// -/// ``` -/// # use doc_assert::Variables; -/// let mut variables = Variables::new(); -/// variables.insert("name", "John"); -/// variables.insert("age", 30); -/// ``` -/// -/// Alternatively, they can be passed as a JSON object: -/// -/// ``` -/// # use doc_assert::Variables; -/// let json = r#"{"name": "John", "age": 30}"#; -/// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); -/// ``` -#[derive(Debug, Clone, Default)] -pub struct Variables { - map: HashMap, -} - -impl Variables { - /// Constructs a new, empty `Variables`. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let variables = Variables::new(); - /// ``` - pub fn new() -> Self { - Self { - map: HashMap::new(), - } - } - - /// Constructs a new `Variables` from a JSON object. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// let json = r#"{"name": "John", "age": 30}"#; - /// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); - /// ``` - pub fn from_json(json: &Value) -> Result { - match json { - Value::Object(obj) => Ok(Self { - map: obj.clone().into_iter().collect(), - }), - _ => Err(Error::VariablesNotAnObject), - } - } - - /// Inserts a variable, overwriting any variable of the same name. - /// - /// Anything a `serde_json::Value` can be built from is accepted, which covers the - /// strings, numbers and booleans a documentation usually needs. Pass a `Value` - /// itself for the types it does not cover. - /// - /// # Examples - /// - /// ``` - /// # use doc_assert::Variables; - /// # use serde_json::Value; - /// let mut variables = Variables::new(); - /// variables.insert("name", "John"); - /// variables.insert("age", 30); - /// variables.insert("is_adult", true); - /// variables.insert("nickname", Value::Null); - /// ``` - pub fn insert(&mut self, name: impl Into, value: impl Into) { - self.map.insert(name.into(), value.into()); - } - - pub(crate) fn obtain_from_response( - &mut self, - response: &Value, - variable_templates: &HashMap, - ) -> Result<(), Mismatch> { - for (name, path) in variable_templates { - let value = extract_value(path, response) - .ok_or_else(|| Mismatch::VariableNotFound { name: name.clone() })?; - - self.map.insert(name.clone(), value); - } - - Ok(()) - } - - fn replace_placeholders(&self, input: &mut String, trim_quotes: bool) -> Result<(), Failure> { - for (name, value) in &self.map { - let placeholder = format!("`{}`", name); - let value_str = value.to_string(); - - let value = if trim_quotes { - value_str.trim_matches('"') - } else { - value_str.as_str() - }; - - *input = input.replace(&placeholder, value); - } - - if input.contains('`') { - return Err(Failure::UnresolvedVariables { - input: input.clone(), - }); - } - - Ok(()) - } - - pub(crate) fn replace_request_placeholders(&self, input: &mut Request) -> Result<(), Failure> { - self.replace_placeholders(&mut input.uri, true)?; - - if let Some(body) = &mut input.body { - self.replace_placeholders(body, false)?; - } - - for (_, value) in &mut input.headers.iter_mut() { - self.replace_placeholders(value, true)?; - } - - Ok(()) - } - - pub(crate) fn replace_response_placeholders( - &self, - input: &mut Response, - ) -> Result<(), Failure> { - if let Some(body) = &mut input.body { - self.replace_placeholders(body, false)?; - } - - for (_, value) in &mut input.headers.iter_mut() { - self.replace_placeholders(value, true)?; - } - - Ok(()) - } -} - -fn extract_value(path: &Path, value: &Value) -> Option { - match path { - Path::Root => None, - Path::Keys(keys) => { - let mut current = value; - for key in keys { - match key { - Key::Field(field) => current = current.get(field)?, - Key::Idx(index) => current = current.get(index)?, - _ => return None, - } - } - Some(current.clone()) - } - } -} - -#[cfg(test)] -mod tests { - use crate::{DocAssert, Error, Failure, Mismatch}; - - /// Documentation file removing itself once it goes out of scope. - struct TempDoc { - path: std::path::PathBuf, - } - - impl TempDoc { - fn new(content: &str) -> Self { - let path = std::env::temp_dir().join(format!( - "doc_assert_{}_{}.md", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - std::fs::write(&path, content).unwrap(); - Self { path } - } - - fn path(&self) -> &str { - self.path.to_str().unwrap() - } - } - - impl Drop for TempDoc { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } - } - - /// One test case hitting `/passing`, then one hitting `/failing`. - const DOC: &str = "```docassertrequest\n\ - GET /passing\n\ - ```\n\ - ```docassertresponse\n\ - HTTP 200\n\ - ```\n\ - ```docassertrequest\n\ - GET /failing\n\ - ```\n\ - ```docassertresponse\n\ - HTTP 200\n\ - ```\n"; - - async fn server() -> mockito::ServerGuard { - let mut server = mockito::Server::new_async().await; - server - .mock("GET", "/passing") - .with_status(200) - .create_async() - .await; - server - .mock("GET", "/failing") - .with_status(500) - .create_async() - .await; - server - } - - #[tokio::test] - async fn test_every_test_case_is_available_as_soon_as_it_is_executed() { - let server = server().await; - let doc = TempDoc::new(DOC); - - let mut run = DocAssert::new() - .with_url(server.url()) - .with_doc_path(doc.path()) - .start() - .unwrap(); - - // the count is known before anything has been executed - assert_eq!(2, run.total_count()); - assert_eq!(0, run.completed_count()); - - let first = run.next().await.unwrap(); - assert!(first.passed()); - assert_eq!("GET", first.id().http_method()); - assert_eq!("/passing", first.id().uri()); - // the second test case has not been executed at the point the first is returned - assert_eq!(1, run.completed_count()); - - let second = run.next().await.unwrap(); - assert!(!second.passed()); - assert_eq!("/failing", second.id().uri()); - - assert!(run.next().await.is_none()); - - let report = run.finish(); - assert_eq!(2, report.executed_count()); - assert_eq!(1, report.passed_count()); - assert_eq!(1, report.failed_count()); - assert!(!report.passed()); - assert_eq!(1, report.failures().count()); - } - - #[tokio::test] - async fn test_failure_carries_the_reason_it_failed() { - let server = server().await; - let doc = TempDoc::new(DOC); - - let report = DocAssert::new() - .with_url(server.url()) - .with_doc_path(doc.path()) - .run() - .await - .unwrap(); - - let failure = report.failures().next().unwrap().failure().unwrap(); - match failure { - Failure::ResponseMismatch { cause, .. } => assert_eq!( - &Mismatch::StatusCode { - expected: 200, - actual: 500 - }, - cause - ), - other => panic!("unexpected failure: {:?}", other), - } - } - - #[tokio::test] - async fn test_run_can_be_stopped_early_and_still_report() { - let server = server().await; - let doc = TempDoc::new(DOC); - - let mut run = DocAssert::new() - .with_url(server.url()) - .with_doc_path(doc.path()) - .start() - .unwrap(); - - run.next().await.unwrap(); - let report = run.finish(); - - // only the test cases that were executed are reported - assert_eq!(1, report.executed_count()); - assert!(report.passed()); - } - - #[test] - fn test_a_run_without_documentation_is_an_error() { - assert_eq!( - Some(Error::NoDocuments), - DocAssert::new() - .with_url("http://localhost:8080") - .start() - .err() - ); - } - - #[test] - fn test_a_run_without_a_url_is_an_error() { - assert_eq!( - Some(Error::NoUrl), - DocAssert::new().with_doc_path("README.md").start().err() - ); - } - - #[test] - fn test_parsing_error_names_the_file_it_comes_from() { - let err = DocAssert::new() - .with_url("http://localhost:8080") - .with_doc_path("this/file/does/not/exist.md") - .start() - .unwrap_err(); - - match err { - Error::Parse { doc_path, .. } => assert_eq!("this/file/does/not/exist.md", doc_path), - other => panic!("unexpected error: {:?}", other), - } - } - - #[test] - fn test_report_renders_the_same_thing_as_the_streamed_output() { - let report = crate::Report::new(vec![ - crate::TestCaseResult::new( - crate::TestCaseId::new( - "GET".to_string(), - "/blog".to_string(), - "README.md".to_string(), - 12, - ), - None, - ), - crate::TestCaseResult::new( - crate::TestCaseId::new( - "POST".to_string(), - "/blog".to_string(), - "README.md".to_string(), - 30, - ), - Some(Failure::ResponseMismatch { - line_number: 36, - cause: Mismatch::StatusCode { - expected: 201, - actual: 500, - }, - }), - ), - ]); - - // what the binary prints line by line has to add up to what `Report` displays - let streamed = format!( - "{} tests\n{}\n{}\n", - report.executed_count(), - report - .results() - .iter() - .map(|r| r.to_string()) - .collect::>() - .join("\n"), - report.summary() - ); - assert_eq!(format!("{}\n", report), streamed); - } -} diff --git a/src/main.rs b/src/main.rs index c983cdc..4d29fb6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,17 +80,18 @@ async fn main() { let variables = match cli.variables { Some(vars) => match Variables::from_json(&vars.0) { - Ok(vars) => vars, - Err(e) => { - handle_error!(Code::INVALID_ARGUMENT, "Error: {}", e); + Some(vars) => vars, + None => { + handle_error!( + Code::INVALID_ARGUMENT, + "Error: Variables must be a JSON object" + ); } }, None => Variables::new(), }; - let mut doc_assert = DocAssert::new() - .with_url(cli.url.as_str()) - .with_variables(variables); + let mut doc_assert = DocAssert::new(cli.url).with_variables(variables); for file in cli.files.iter() { let Some(file) = file.to_str() else { @@ -102,7 +103,7 @@ async fn main() { let mut run = match doc_assert.start() { Ok(run) => run, - Err(err @ (Error::NoUrl | Error::NoDocuments)) => { + Err(err @ Error::NoDocuments) => { handle_error!(Code::INVALID_ARGUMENT, "Error: {}", err); } Err(err) => { diff --git a/src/parser.rs b/src/parser.rs index 7b7d424..cd8d67d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -115,7 +115,7 @@ pub(crate) fn parse(path: String) -> Result, String> { fn get_code(lines: &mut Enumerate) -> String { let mut buff = String::new(); - while let Some(line) = lines.next() { + for line in lines { if line.1.starts_with("```") { break; } diff --git a/src/report.rs b/src/report.rs index c884ce7..eedb362 100644 --- a/src/report.rs +++ b/src/report.rs @@ -267,8 +267,7 @@ impl Display for TestCaseResult { /// ``` /// # use doc_assert::DocAssert; /// async fn test() { -/// let report = DocAssert::new() -/// .with_url("http://localhost:8080") +/// let report = DocAssert::new("http://localhost:8080") /// .with_doc_path("README.md") /// .run() /// .await @@ -377,3 +376,52 @@ impl Display for Summary<'_> { ) } } + +#[cfg(test)] +mod tests { + use super::{Failure, Mismatch, Report, TestCaseId, TestCaseResult}; + + #[test] + fn test_report_renders_the_same_thing_as_the_streamed_output() { + let report = Report::new(vec![ + TestCaseResult::new( + TestCaseId::new( + "GET".to_string(), + "/blog".to_string(), + "README.md".to_string(), + 12, + ), + None, + ), + TestCaseResult::new( + TestCaseId::new( + "POST".to_string(), + "/blog".to_string(), + "README.md".to_string(), + 30, + ), + Some(Failure::ResponseMismatch { + line_number: 36, + cause: Mismatch::StatusCode { + expected: 201, + actual: 500, + }, + }), + ), + ]); + + // what the binary prints line by line has to add up to what `Report` displays + let streamed = format!( + "{} tests\n{}\n{}\n", + report.executed_count(), + report + .results() + .iter() + .map(|r| r.to_string()) + .collect::>() + .join("\n"), + report.summary() + ); + assert_eq!(format!("{}\n", report), streamed); + } +} diff --git a/src/variables.rs b/src/variables.rs new file mode 100644 index 0000000..c5aeb2d --- /dev/null +++ b/src/variables.rs @@ -0,0 +1,187 @@ +// Copyright 2024 The DocAssert Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Values shared between test cases, and the substitution of their placeholders. + +use crate::{ + domain::{Request, Response}, + json_diff::path::{Key, Path}, + report::{Failure, Mismatch}, +}; +use serde_json::Value; +use std::collections::HashMap; + +/// Variables to be used in the request and response bodies. +/// +/// The variables replace placeholders in the request and response bodies, in case some +/// values need to be shared between requests. +/// +/// # Examples +/// +/// Variables can be inserted one by one: +/// +/// ``` +/// # use doc_assert::Variables; +/// let mut variables = Variables::new(); +/// variables.insert("name", "John"); +/// variables.insert("age", 30); +/// ``` +/// +/// Alternatively, they can be passed as a JSON object: +/// +/// ``` +/// # use doc_assert::Variables; +/// let json = r#"{"name": "John", "age": 30}"#; +/// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); +/// ``` +#[derive(Debug, Clone, Default)] +pub struct Variables { + map: HashMap, +} + +impl Variables { + /// Constructs a new, empty `Variables`. + /// + /// # Examples + /// + /// ``` + /// # use doc_assert::Variables; + /// let variables = Variables::new(); + /// ``` + pub fn new() -> Self { + Self { + map: HashMap::new(), + } + } + + /// Constructs a new `Variables` from a JSON object, `None` if `json` is not an object. + /// + /// # Examples + /// + /// ``` + /// # use doc_assert::Variables; + /// let json = r#"{"name": "John", "age": 30}"#; + /// let variables = Variables::from_json(&serde_json::from_str(json).unwrap()).unwrap(); + /// ``` + pub fn from_json(json: &Value) -> Option { + let obj = json.as_object()?; + Some(Self { + map: obj.clone().into_iter().collect(), + }) + } + + /// Inserts a variable, overwriting any variable of the same name. + /// + /// Anything a `serde_json::Value` can be built from is accepted, which covers the + /// strings, numbers and booleans a documentation usually needs. Pass a `Value` + /// itself for the types it does not cover. + /// + /// # Examples + /// + /// ``` + /// # use doc_assert::Variables; + /// # use serde_json::Value; + /// let mut variables = Variables::new(); + /// variables.insert("name", "John"); + /// variables.insert("age", 30); + /// variables.insert("is_adult", true); + /// variables.insert("nickname", Value::Null); + /// ``` + pub fn insert(&mut self, name: impl Into, value: impl Into) { + self.map.insert(name.into(), value.into()); + } + + pub(crate) fn obtain_from_response( + &mut self, + response: &Value, + variable_templates: &HashMap, + ) -> Result<(), Mismatch> { + for (name, path) in variable_templates { + let value = extract_value(path, response) + .ok_or_else(|| Mismatch::VariableNotFound { name: name.clone() })?; + + self.map.insert(name.clone(), value); + } + + Ok(()) + } + + fn replace_placeholders(&self, input: &mut String, trim_quotes: bool) -> Result<(), Failure> { + for (name, value) in &self.map { + let placeholder = format!("`{}`", name); + let value_str = value.to_string(); + + let value = if trim_quotes { + value_str.trim_matches('"') + } else { + value_str.as_str() + }; + + *input = input.replace(&placeholder, value); + } + + if input.contains('`') { + return Err(Failure::UnresolvedVariables { + input: input.clone(), + }); + } + + Ok(()) + } + + pub(crate) fn replace_request_placeholders(&self, input: &mut Request) -> Result<(), Failure> { + self.replace_placeholders(&mut input.uri, true)?; + + if let Some(body) = &mut input.body { + self.replace_placeholders(body, false)?; + } + + for (_, value) in &mut input.headers.iter_mut() { + self.replace_placeholders(value, true)?; + } + + Ok(()) + } + + pub(crate) fn replace_response_placeholders( + &self, + input: &mut Response, + ) -> Result<(), Failure> { + if let Some(body) = &mut input.body { + self.replace_placeholders(body, false)?; + } + + for (_, value) in &mut input.headers.iter_mut() { + self.replace_placeholders(value, true)?; + } + + Ok(()) + } +} + +fn extract_value(path: &Path, value: &Value) -> Option { + match path { + Path::Root => None, + Path::Keys(keys) => { + let mut current = value; + for key in keys { + match key { + Key::Field(field) => current = current.get(field)?, + Key::Idx(index) => current = current.get(index)?, + _ => return None, + } + } + Some(current.clone()) + } + } +} diff --git a/tests/api.rs b/tests/api.rs new file mode 100644 index 0000000..33d4840 --- /dev/null +++ b/tests/api.rs @@ -0,0 +1,174 @@ +// Copyright 2024 The DocAssert Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Exercises the library the way a test suite depending on it would: through its public +//! API only, compiled as a separate crate. + +use doc_assert::{DocAssert, Error, Failure, Mismatch}; + +/// Documentation file removing itself once it goes out of scope. +struct TempDoc { + path: std::path::PathBuf, +} + +impl TempDoc { + fn new(content: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "doc_assert_{}_{}.md", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::write(&path, content).unwrap(); + Self { path } + } + + fn path(&self) -> &str { + self.path.to_str().unwrap() + } +} + +impl Drop for TempDoc { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +/// One test case hitting `/passing`, then one hitting `/failing`. +const DOC: &str = "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n\ + ```docassertrequest\n\ + GET /failing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n"; + +async fn server() -> mockito::ServerGuard { + let mut server = mockito::Server::new_async().await; + server + .mock("GET", "/passing") + .with_status(200) + .create_async() + .await; + server + .mock("GET", "/failing") + .with_status(500) + .create_async() + .await; + server +} + +#[tokio::test] +async fn test_every_test_case_is_available_as_soon_as_it_is_executed() { + let server = server().await; + let doc = TempDoc::new(DOC); + + let mut run = DocAssert::new(server.url()) + .with_doc_path(doc.path()) + .start() + .unwrap(); + + // the count is known before anything has been executed + assert_eq!(2, run.total_count()); + assert_eq!(0, run.completed_count()); + + let first = run.next().await.unwrap(); + assert!(first.passed()); + assert_eq!("GET", first.id().http_method()); + assert_eq!("/passing", first.id().uri()); + // the second test case has not been executed at the point the first is returned + assert_eq!(1, run.completed_count()); + + let second = run.next().await.unwrap(); + assert!(!second.passed()); + assert_eq!("/failing", second.id().uri()); + + assert!(run.next().await.is_none()); + + let report = run.finish(); + assert_eq!(2, report.executed_count()); + assert_eq!(1, report.passed_count()); + assert_eq!(1, report.failed_count()); + assert!(!report.passed()); + assert_eq!(1, report.failures().count()); +} + +#[tokio::test] +async fn test_failure_carries_the_reason_it_failed() { + let server = server().await; + let doc = TempDoc::new(DOC); + + let report = DocAssert::new(server.url()) + .with_doc_path(doc.path()) + .run() + .await + .unwrap(); + + let failure = report.failures().next().unwrap().failure().unwrap(); + match failure { + Failure::ResponseMismatch { cause, .. } => assert_eq!( + &Mismatch::StatusCode { + expected: 200, + actual: 500 + }, + cause + ), + other => panic!("unexpected failure: {:?}", other), + } +} + +#[tokio::test] +async fn test_run_can_be_stopped_early_and_still_report() { + let server = server().await; + let doc = TempDoc::new(DOC); + + let mut run = DocAssert::new(server.url()) + .with_doc_path(doc.path()) + .start() + .unwrap(); + + run.next().await.unwrap(); + let report = run.finish(); + + // only the test cases that were executed are reported + assert_eq!(1, report.executed_count()); + assert!(report.passed()); +} + +#[test] +fn test_a_run_without_documentation_is_an_error() { + assert_eq!( + Some(Error::NoDocuments), + DocAssert::new("http://localhost:8080").start().err() + ); +} + +#[test] +fn test_parsing_error_names_the_file_it_comes_from() { + let err = DocAssert::new("http://localhost:8080") + .with_doc_path("this/file/does/not/exist.md") + .start() + .unwrap_err(); + + match err { + Error::Parse { doc_path, .. } => assert_eq!("this/file/does/not/exist.md", doc_path), + other => panic!("unexpected error: {:?}", other), + } +} From 5e4506276894ee7f9bc6933c3dbf11956fb2658c Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Fri, 2 Oct 2026 17:12:43 +0200 Subject: [PATCH 5/6] more --- .github/workflows/doc-assert.yml | 4 +- Cargo.lock | 2 +- Cargo.toml | 6 +- README.md | 172 ++++++------ src/domain.rs | 9 +- src/executor.rs | 464 +++++++++++++++++++++++++------ src/lib.rs | 142 ++++++---- src/main.rs | 20 +- src/parser.rs | 44 +-- src/report.rs | 398 ++++++++++++++++++-------- src/variables.rs | 134 ++++++++- tests/api.rs | 233 +++++++++++----- tests/cli.rs | 301 ++++++++++++++++++++ tests/common/mod.rs | 60 ++++ 14 files changed, 1505 insertions(+), 484 deletions(-) create mode 100644 tests/cli.rs create mode 100644 tests/common/mod.rs diff --git a/.github/workflows/doc-assert.yml b/.github/workflows/doc-assert.yml index 0e41d38..a13a96d 100644 --- a/.github/workflows/doc-assert.yml +++ b/.github/workflows/doc-assert.yml @@ -31,11 +31,11 @@ jobs: - name: Format run: cargo fmt --all -- --check - name: Lint - run: cargo clippy -- -D warnings + run: cargo clippy --all-targets --features="binary" -- -D warnings - name: Build run: cargo build --verbose --features="binary" - name: Run tests - run: cargo test --verbose --workspace + run: cargo test --verbose --workspace --features="binary" - name: Start sample-api backend working-directory: ./tests/functional run: make backend-start diff --git a/Cargo.lock b/Cargo.lock index 7ee2459..983c775 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,7 +223,7 @@ dependencies = [ [[package]] name = "doc-assert" -version = "0.1.1" +version = "0.2.0" dependencies = [ "clap", "mockito", diff --git a/Cargo.toml b/Cargo.toml index ba38b1f..70b532f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ [package] name = "doc-assert" -version = "0.1.1" +version = "0.2.0" edition = "2021" license = "Apache-2.0" readme = "README.md" @@ -57,3 +57,7 @@ binary = ["clap"] name = "doc-assert" path = "src/main.rs" required-features = ["binary"] + +[[test]] +name = "cli" +required-features = ["binary"] diff --git a/README.md b/README.md index cf0c738..9684996 100644 --- a/README.md +++ b/README.md @@ -62,98 +62,70 @@ Once your documentation is prepared, you can run DocAssert from your tests like ```rust use doc_assert::DocAssert; -#[cfg(test)] -mod tests { - #[tokio::test] - async fn test_docs() { - DocAssert::new("http://localhost:8080") - .with_doc_path("README.md") - .assert() - .await; - } +// a test of your own, under `#[tokio::test]` +async fn test_docs() { + DocAssert::new("http://localhost:8080") + .with_doc_path("README.md") + .assert() + .await; } ``` -`assert` fails the test if anything went wrong, printing the whole report. If you would rather handle the -outcome yourself, `run` hands it back instead: - -```rust -# use doc_assert::DocAssert; -# async fn test() { -let report = DocAssert::new("http://localhost:8080") - .with_doc_path("README.md") - .run() - .await - .unwrap(); - -println!("{} of {} passed", report.passed_count(), report.executed_count()); -for result in report.failures() { - println!("{} failed: {}", result.id(), result.failure().unwrap()); -} -# } -``` +`assert` prints every test case as it is executed, the same way the [command line tool](#using-command-line-tool) +does, and fails the test unless all of them passed. Like any test output, it is shown when the test fails, or as it +is printed with `cargo test -- --nocapture`. -`run` returns `Err` only when the run could not be performed at all, for instance because a documentation file -could not be parsed. Test cases that failed are not an error: a run that executed its test cases always -returns a `Report`, and `Report::passed` is the verdict. +If you would rather handle the outcome yourself, `run` hands back a `Report` without printing anything: -#### Seeing the results as they happen +```rust +use doc_assert::DocAssert; -`assert` returns only once every test case has been executed. To handle the results while the run is still -going on, drive it yourself with `start`, which parses the documentation and hands back a `Run`: +async fn print_failures() { + let report = DocAssert::new("http://localhost:8080") + .with_doc_path("README.md") + .run() + .await + .unwrap(); -```rust -# use doc_assert::DocAssert; -# async fn test() { -let mut run = DocAssert::new("http://localhost:8080") - .with_doc_path("README.md") - .start() - .unwrap(); - -println!("{} tests", run.total_count()); -while let Some(result) = run.next().await { - println!("{}", result); + println!("{} of {} passed", report.passed_count(), report.total_count()); + for (id, failure) in report.failures() { + println!("{} failed: {}", id, failure); + } } - -let report = run.finish(); -println!("{}", report.summary()); -# } ``` -This is what the `doc-assert` binary does, so its output appears as the test cases are executed rather than -all at once at the end of the run: - -```text -2 tests -GET /blog (README.md:12) ✅ -POST /blog (README.md:30) ❌ +`run` returns `Err` only when the run could not be performed at all, for instance because a documentation file +could not be parsed. Test cases that failed are reported by the `Report`, and `Report::passed` tells whether every +one of them passed. -failures: -------------- -POST /blog (README.md:30): response at line 36: expected response code 201, got 500 +To handle every test case as soon as it has been executed, drive the run yourself with `start`, which parses the +documentation and hands back a `Run`. This one stops at the first failure: -test result: FAILED. 1 passed; 1 failed -``` +```rust +use doc_assert::DocAssert; -Because you own the loop you can also stop early or time each test case. Stop it between calls, though: -`next` is not cancellation safe, so a future that is dropped part way through — raced against a timeout, or -selected on — loses the test case it had already taken off the queue. +async fn fail_fast() { + let mut run = DocAssert::new("http://localhost:8080") + .with_doc_path("README.md") + .start() + .unwrap(); -```rust -# use doc_assert::DocAssert; -# async fn test() { -# let mut run = DocAssert::new("http://localhost:8080").with_doc_path("README.md").start().unwrap(); -while let Some(result) = run.next().await { - if !result.passed() { - break; // fail fast + while let Some(result) = run.next().await { + println!("{}", result); + if !result.passed() { + break; + } } + + // the test cases left are reported as not run + println!("{}", run.finish().summary()); } -let report = run.finish(); // the test cases executed so far -# } ``` -A failure is a `Failure`, not a string, and it carries only what its `TestCaseId` does not already say, so -you can render your own output — JUnit XML, TAP, JSON — by matching on it: +Stop the run between calls to `next` rather than racing it against a timeout or selecting on it: a test case +dropped part way through is lost, even though its request may already have reached the server. + +Failures are structured, so you can inspect them, or render them your own way, by matching on them: ```rust use doc_assert::{Failure, Mismatch}; @@ -178,19 +150,16 @@ We can define variable in the API before we run the tests: ```rust use doc_assert::{DocAssert, Variables}; -#[cfg(test)] -mod tests { - #[tokio::test] - async fn test_docs() { - let mut variables = Variables::new(); - variables.insert("auth_token", "some_token"); - - DocAssert::new("http://localhost:8080") - .with_doc_path("README.md") - .with_variables(variables) - .assert() - .await; - } +// a test of your own, under `#[tokio::test]` +async fn test_docs() { + let mut variables = Variables::new(); + variables.insert("auth_token", "some_token"); + + DocAssert::new("http://localhost:8080") + .with_doc_path("README.md") + .with_variables(variables) + .assert() + .await; } ``` @@ -273,7 +242,9 @@ Content-Type: application/json [retry]: # (3,4500) ~~~ -The first number in the retry policy is the number of retries, and the second number is the delay between retries in milliseconds. +The first number in the retry policy is the number of attempts, the first one included, and the second number is the +delay between attempts in milliseconds: `(3,4500)` sends the request up to 3 times, waiting 4.5 seconds after each +attempt that failed. The number of attempts must be at least 1. ### Using command line tool @@ -283,6 +254,31 @@ Instead of integrating DocAssert into your tests, you can also use it as a stand doc-assert --url http://localhost:8081 --variables '{"auth_token": "some_token"}' README.md ``` +Every documentation file is parsed before any request is sent. The test cases are then printed as they are executed, +the way `cargo test` prints its own: each one is named as soon as its request is sent and marked once it is done, so +a slow or retried request shows what is being waited for. The details of the failures follow at the end: + +```text +2 tests +GET /blog (README.md:12) ✅ +POST /blog (README.md:30) ❌ + +failures: +------------- +POST /blog (README.md:30): response at line 36: expected response code 201, got 500 + +test result: FAILED. 1 passed; 1 failed +``` + +The exit code tells how the run went, and errors preventing it are printed to stderr: + +| Code | Meaning | +|------|---------| +| 0 | every test case passed | +| 2 | invalid arguments, such as no documentation file, or variables that are not a JSON object | +| 3 | a documentation file could not be read or parsed, in which case no request was sent | +| 4 | at least one test case failed | + ### Using DocAssert for AI-assisted development DocAssert is a good fit for AI-assisted workflows where an agent updates API docs and code in the same task. diff --git a/src/domain.rs b/src/domain.rs index 087ce99..e09cce4 100644 --- a/src/domain.rs +++ b/src/domain.rs @@ -69,14 +69,15 @@ pub(crate) struct Request { #[derive(Debug, Clone, PartialEq)] pub(crate) struct RetryPolicy { - pub max_retries: u64, + /// Number of times the request is sent before giving up, the first time included + pub max_attempts: u64, pub delay: u64, } impl Default for RetryPolicy { fn default() -> Self { RetryPolicy { - max_retries: 1, + max_attempts: 1, delay: 0, } } @@ -86,8 +87,8 @@ impl Default for RetryPolicy { pub(crate) struct Response { pub code: u16, pub headers: HashMap, - pub ignore_paths: Vec, - pub ignore_orders: Vec, + pub ignore_paths: Vec, + pub ignore_orders: Vec, pub body: Option, pub line_number: usize, pub variables: HashMap, diff --git a/src/executor.rs b/src/executor.rs index 5d572b7..1ebc4b4 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -17,9 +17,9 @@ use std::time::Duration; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest::{Body, Client, Method, Response}; +use serde_json::Value; use crate::domain::{HttpMethod, Request, TestCase}; -use crate::json_diff::path::Path; use crate::json_diff::{diff, CompareMode, Config}; use crate::report::{Failure, Mismatch}; use crate::Variables; @@ -29,136 +29,149 @@ pub(crate) async fn execute( test_case: TestCase, variables: &mut Variables, ) -> Result<(), Failure> { - let mut test_request = test_case.request; - variables.replace_request_placeholders(&mut test_request)?; - - let mut test_response = test_case.response; - variables.replace_response_placeholders(&mut test_response)?; - - let response_line_number = test_response.line_number; - + let mut request = test_case.request; + variables.replace_request_placeholders(&mut request)?; + + let mut expected = test_case.response; + variables.replace_response_placeholders(&mut expected)?; + + // what the documentation describes is checked before anything is sent, as sending the + // request again would not make it any more valid + let headers = + map_headers(&request.headers).map_err(|reason| Failure::InvalidDocumentation { + line_number: request.line_number, + reason, + })?; + let expected_body = expected + .body + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(|err| Failure::InvalidDocumentation { + line_number: expected.line_number, + reason: format!("expected body is not valid JSON: {}", err), + })?; + let diff_config = diff_config(&expected); + + let client = Client::new(); let mut attempt = 0; loop { attempt += 1; - let failure = match get_response(base_url, &test_request).await { + let failure = match send(&client, base_url, &request, headers.clone()).await { Err(reason) => Failure::RequestFailed { reason }, - Ok(response) => match assert_response(response, &test_response, variables).await { - Ok(_) => return Ok(()), - Err(cause) => Failure::ResponseMismatch { - line_number: response_line_number, - cause, - }, - }, + Ok(response) => { + match assert_response(response, &expected, expected_body.as_ref(), &diff_config) + .await + { + Ok(extracted) => { + variables.extend(extracted); + return Ok(()); + } + Err(cause) => Failure::ResponseMismatch { + line_number: expected.line_number, + cause, + }, + } + } }; - // the policy counts attempts rather than extra tries, and the parser rejects one - // that allows none, so the request is always attempted at least once - if attempt >= test_response.retries.max_retries { + if attempt >= expected.retries.max_attempts { return Err(failure); } - tokio::time::sleep(Duration::from_millis(test_response.retries.delay)).await; + tokio::time::sleep(Duration::from_millis(expected.retries.delay)).await; } } +fn diff_config(expected: &crate::domain::Response) -> Config { + let mut config = Config::new(CompareMode::Strict); + for path in &expected.ignore_paths { + config = config.ignore_path(path.clone()); + } + for path in &expected.ignore_orders { + config = config.ignore_order(path.clone()); + } + config +} + +/// Checks a response against the expected one, returning the variables extracted from it. async fn assert_response( response: Response, - test_response: &crate::domain::Response, - variables: &mut Variables, -) -> Result<(), Mismatch> { - if test_response.code != response.status().as_u16() { + expected: &crate::domain::Response, + expected_body: Option<&Value>, + diff_config: &Config, +) -> Result, Mismatch> { + if expected.code != response.status().as_u16() { return Err(Mismatch::StatusCode { - expected: test_response.code, + expected: expected.code, actual: response.status().as_u16(), }); } - for (key, val) in test_response.headers.iter() { + for (key, val) in expected.headers.iter() { match response.headers().get(key.as_str()) { - Some(test_val) => { - if test_val != val.as_str() { + Some(actual) => { + if actual != val.as_str() { return Err(Mismatch::Header { name: key.clone(), expected: val.clone(), - actual: test_val.to_str().unwrap().to_string(), + actual: String::from_utf8_lossy(actual.as_bytes()).into_owned(), }); } } None => return Err(Mismatch::MissingHeader { name: key.clone() }), } } - if let Some(test_body) = test_response.body.as_ref() { - let mut diff_config = Config::new(CompareMode::Strict); - for path in test_response.ignore_paths.iter() { - diff_config = - diff_config.ignore_path(Path::from_jsonpath(path.as_str()).map_err(|err| { - Mismatch::InvalidIgnorePath { - path: path.clone(), - reason: err.to_string(), - } - })?); - } - for order in test_response.ignore_orders.iter() { - diff_config = - diff_config.ignore_order(Path::from_jsonpath(order.as_str()).map_err(|err| { - Mismatch::InvalidIgnorePath { - path: order.clone(), - reason: err.to_string(), - } - })?); - } - let response_body = - response - .text() - .await - .map_err(|err| Mismatch::UnreadableResponseBody { - reason: err.to_string(), - })?; - let actual = - &serde_json::from_str::(response_body.as_str()).map_err(|err| { - Mismatch::MalformedResponseBody { - reason: err.to_string(), - } - })?; - let expected = - &serde_json::from_str::(test_body.as_str()).map_err(|err| { - Mismatch::MalformedExpectedBody { - reason: err.to_string(), - } - })?; - let diff_result = diff(expected, actual, diff_config); - if !diff_result.is_empty() { - return Err(Mismatch::Body { - differences: diff_result.iter().map(|d| d.to_string()).collect(), - }); - } - - if !test_response.variables.is_empty() { - variables.obtain_from_response(actual, &test_response.variables)?; + let Some(expected_body) = expected_body else { + return Ok(HashMap::new()); + }; + + let response_body = response + .text() + .await + .map_err(|err| Mismatch::UnreadableResponseBody { + reason: err.to_string(), + })?; + let actual = serde_json::from_str::(response_body.as_str()).map_err(|err| { + Mismatch::MalformedResponseBody { + reason: err.to_string(), } + })?; + let diff_result = diff(expected_body, &actual, diff_config.clone()); + if !diff_result.is_empty() { + return Err(Mismatch::Body { + differences: diff_result.iter().map(|d| d.to_string()).collect(), + }); } - Ok(()) + + Variables::extract_from_response(&actual, &expected.variables) } -async fn get_response(base_url: &str, test_request: &Request) -> Result { - let mut request_builder = Client::new() +async fn send( + client: &Client, + base_url: &str, + request: &Request, + headers: HeaderMap, +) -> Result { + let mut request_builder = client .request( - map_method(&test_request.http_method), - format!("{}{}", base_url, test_request.uri), + map_method(&request.http_method), + format!("{}{}", base_url, request.uri), ) - .headers(map_headers(&test_request.headers)?); - if let Some(body) = &test_request.body { + .headers(headers); + if let Some(body) = &request.body { request_builder = request_builder.body(Body::from(body.clone())); } - let response = request_builder.send().await.map_err(|e| e.to_string())?; - Ok(response) + request_builder.send().await.map_err(|e| e.to_string()) } fn map_headers(headers: &HashMap) -> Result { let mut header_map = HeaderMap::new(); for (key, value) in headers { - let header_name = HeaderName::from_str(key.clone().as_str()).map_err(|e| e.to_string())?; - let header_value = HeaderValue::from_str(value.as_str()).map_err(|e| e.to_string())?; + let header_name = HeaderName::from_str(key.as_str()) + .map_err(|e| format!("invalid header name {}: {}", key, e))?; + let header_value = HeaderValue::from_str(value.as_str()) + .map_err(|e| format!("invalid value of header {}: {}", key, e))?; header_map.insert(header_name, header_value); } Ok(header_map) @@ -179,11 +192,64 @@ mod tests { use serde_json::json; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use std::time::{Duration, Instant}; + use crate::domain::{HttpMethod, Request, Response, RetryPolicy, TestCase}; use crate::executor::execute; use crate::json_diff::path::JSONPath; + use crate::report::{Failure, Mismatch}; use crate::Variables; + /// `GET /resource` expecting a 200 with `body`, sent as many times as `retries` allows. + fn get_resource(body: Option<&str>, retries: RetryPolicy) -> TestCase { + TestCase { + request: Request { + http_method: HttpMethod::Get, + headers: HashMap::new(), + uri: "/resource".to_string(), + body: None, + line_number: 1, + }, + response: Response { + code: 200, + headers: HashMap::new(), + ignore_paths: vec![], + ignore_orders: vec![], + body: body.map(str::to_string), + line_number: 4, + variables: HashMap::new(), + retries, + }, + } + } + + fn retries(max_attempts: u64, delay: u64) -> RetryPolicy { + RetryPolicy { + max_attempts, + delay, + } + } + + /// Mocks `GET /resource` answering with the bodies one after the other, the last one + /// repeated once they are exhausted. + async fn serve_in_turn( + server: &mut mockito::ServerGuard, + bodies: &'static [&'static str], + ) -> mockito::Mock { + let served = Arc::new(AtomicUsize::new(0)); + server + .mock("GET", "/resource") + .with_status(200) + .with_body_from_request(move |_| { + let i = served.fetch_add(1, Ordering::SeqCst); + bodies[i.min(bodies.len() - 1)].into() + }) + .create_async() + .await + } + #[tokio::test] async fn test_execute() { let users_endpoint = "/users"; @@ -221,7 +287,7 @@ mod tests { headers: vec![(header_name.to_string(), header_value.to_string())] .into_iter() .collect(), - ignore_paths: vec!["$.id".to_string()], + ignore_paths: vec!["$.id".jsonpath().unwrap()], ignore_orders: vec![], body: Some(response_body.to_string()), line_number: 2, @@ -286,7 +352,7 @@ mod tests { headers: vec![(header_name.to_string(), header_value.to_string())] .into_iter() .collect(), - ignore_paths: vec!["$.id".to_string()], + ignore_paths: vec!["$.id".jsonpath().unwrap()], ignore_orders: vec![], body: Some(response_body.to_string()), line_number: 2, @@ -329,4 +395,220 @@ mod tests { assert_eq!(Ok(()), result); } + + #[tokio::test] + async fn test_a_test_case_passes_as_soon_as_an_attempt_does() { + let mut server = mockito::Server::new_async().await; + let mock = serve_in_turn( + &mut server, + &[ + r#"{"ready":false}"#, + r#"{"ready":false}"#, + r#"{"ready":true}"#, + ], + ) + .await + .expect(3); + let test_case = get_resource(Some(r#"{"ready":true}"#), retries(5, 50)); + + let started = Instant::now(); + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + assert_eq!(Ok(()), result); + mock.assert_async().await; + assert!(started.elapsed() >= Duration::from_millis(100)); + } + + #[tokio::test] + async fn test_the_failure_of_the_last_attempt_is_reported() { + let mut server = mockito::Server::new_async().await; + let mock = serve_in_turn(&mut server, &[r#"{"v":1}"#, r#"{"v":2}"#, r#"{"v":3}"#]) + .await + .expect(3); + let test_case = get_resource(Some(r#"{"v":0}"#), retries(3, 0)); + + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + // the policy counts attempts, so it is sent 3 times rather than retried 3 times + mock.assert_async().await; + match result { + Err(Failure::ResponseMismatch { + line_number: 4, + cause: Mismatch::Body { differences }, + }) => { + let differences = differences.join("\n"); + assert!(differences.contains('3'), "{}", differences); + assert!(!differences.contains('2'), "{}", differences); + } + other => panic!("unexpected result: {:?}", other), + } + } + + #[tokio::test] + async fn test_there_is_no_delay_after_the_last_attempt() { + let mut server = mockito::Server::new_async().await; + server + .mock("GET", "/resource") + .with_status(500) + .create_async() + .await; + let test_case = get_resource(None, retries(2, 500)); + + let started = Instant::now(); + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + let elapsed = started.elapsed(); + + assert!(result.is_err()); + assert!(elapsed >= Duration::from_millis(500), "{:?}", elapsed); + assert!(elapsed < Duration::from_millis(1000), "{:?}", elapsed); + } + + #[tokio::test] + async fn test_an_unreachable_server_fails_the_request() { + let test_case = get_resource(None, retries(2, 50)); + + let started = Instant::now(); + let result = execute("http://127.0.0.1:1", test_case, &mut Variables::new()).await; + + assert!( + matches!(result, Err(Failure::RequestFailed { .. })), + "{:?}", + result + ); + assert!(started.elapsed() >= Duration::from_millis(50)); + } + + #[tokio::test] + async fn test_an_invalid_expected_body_fails_without_sending_the_request() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", "/resource") + .expect(0) + .create_async() + .await; + let test_case = get_resource(Some("{not json"), retries(3, 1000)); + + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + assert!( + matches!( + result, + Err(Failure::InvalidDocumentation { line_number: 4, .. }) + ), + "{:?}", + result + ); + mock.assert_async().await; + } + + #[tokio::test] + async fn test_an_invalid_request_header_fails_without_sending_the_request() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", "/resource") + .expect(0) + .create_async() + .await; + let mut test_case = get_resource(None, retries(3, 1000)); + test_case + .request + .headers + .insert("Bad Name".to_string(), "value".to_string()); + + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + assert!( + matches!( + result, + Err(Failure::InvalidDocumentation { line_number: 1, .. }) + ), + "{:?}", + result + ); + mock.assert_async().await; + } + + #[tokio::test] + async fn test_an_unresolved_variable_fails_without_sending_the_request() { + let mut server = mockito::Server::new_async().await; + let mock = server + .mock("GET", mockito::Matcher::Any) + .expect(0) + .create_async() + .await; + let mut test_case = get_resource(None, RetryPolicy::default()); + test_case.request.uri = "/users/`id`".to_string(); + + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + assert_eq!( + Err(Failure::UnresolvedVariables { + names: vec!["id".to_string()] + }), + result + ); + mock.assert_async().await; + } + + #[tokio::test] + async fn test_a_non_ascii_header_value_is_reported_rather_than_panicking() { + let mut server = mockito::Server::new_async().await; + server + .mock("GET", "/resource") + .with_status(200) + .with_header("X-Name", "café") + .create_async() + .await; + let mut test_case = get_resource(None, RetryPolicy::default()); + test_case + .response + .headers + .insert("X-Name".to_string(), "cafe".to_string()); + + let result = execute(server.url().as_str(), test_case, &mut Variables::new()).await; + + assert_eq!( + Err(Failure::ResponseMismatch { + line_number: 4, + cause: Mismatch::Header { + name: "X-Name".to_string(), + expected: "cafe".to_string(), + actual: "café".to_string(), + } + }), + result + ); + } + + #[tokio::test] + async fn test_variables_are_only_kept_from_a_passing_attempt() { + let mut server = mockito::Server::new_async().await; + serve_in_turn(&mut server, &[r#"{"id":1}"#]).await; + let mut test_case = get_resource(Some(r#"{"id":1}"#), RetryPolicy::default()); + test_case.response.variables = [ + ("id".to_string(), "$.id".jsonpath().unwrap()), + ("missing".to_string(), "$.missing".jsonpath().unwrap()), + ] + .into_iter() + .collect(); + let mut variables = Variables::new(); + + let result = execute(server.url().as_str(), test_case, &mut variables).await; + + assert_eq!( + Err(Failure::ResponseMismatch { + line_number: 4, + cause: Mismatch::VariableNotFound { + name: "missing".to_string() + } + }), + result + ); + // `id` was found, but not kept since the test case did not pass + let mut request = get_resource(None, RetryPolicy::default()).request; + request.uri = "/users/`id`".to_string(); + assert!(variables + .replace_request_placeholders(&mut request) + .is_err()); + } } diff --git a/src/lib.rs b/src/lib.rs index e8b43c4..96a1468 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ use crate::domain::TestCase; use std::fmt::Display; +use std::io::Write; mod domain; mod executor; @@ -23,23 +24,17 @@ mod parser; mod report; mod variables; -pub use report::{Failure, Mismatch, Report, Summary, TestCaseId, TestCaseResult}; +pub use report::{Failure, Mismatch, Report, TestCaseId, TestCaseResult}; pub use variables::Variables; /// Builder for a documentation test run. /// -/// # Examples +/// Once configured, the run is performed in one of three ways: /// -/// ``` -/// use doc_assert::DocAssert; -/// -/// async fn test() { -/// DocAssert::new("http://localhost:8080") -/// .with_doc_path("README.md") -/// .assert() -/// .await; -/// } -/// ``` +/// - [`DocAssert::assert`] prints every test case as it is executed and fails the test +/// unless all of them passed, which is what a test suite usually wants; +/// - [`DocAssert::run`] prints nothing and returns the [`Report`]; +/// - [`DocAssert::start`] returns a [`Run`] to execute the test cases one by one. #[derive(Debug)] pub struct DocAssert { url: String, @@ -102,18 +97,17 @@ impl DocAssert { self } - /// Executes every test case and panics unless all of them passed. + /// Executes every test case, printing each one as it is executed, and panics unless + /// all of them passed. /// - /// This is how `DocAssert` is meant to be used inside a test: anything that goes - /// wrong fails the test, whether it is a documentation file that could not be parsed - /// or a test case that did not pass. Use [`DocAssert::run`] to get the [`Report`] and - /// handle it yourself, or [`DocAssert::start`] to handle every test case as soon as - /// it has been executed. + /// The output is the one of the `doc-assert` binary, see [`Run::print_progress`]. It + /// is printed the way `cargo test` expects test output to be, so it is shown when the + /// test fails, or as it is produced with `cargo test -- --nocapture`. /// /// # Panics /// - /// Panics if the run could not be performed, or if any test case failed. The whole - /// [`Report`] is the panic message. + /// Panics with the [`Error`] if the run could not be performed, and when any test case + /// failed. /// /// # Examples /// @@ -127,16 +121,21 @@ impl DocAssert { /// .await; /// } /// ``` - pub async fn assert(self) -> Report { - let report = match self.run().await { - Ok(report) => report, + pub async fn assert(self) { + let run = match self.start() { + Ok(run) => run, Err(err) => panic!("{}", err), }; - report.assert_passed(); - report + let report = run.print_progress().await; + assert!( + report.passed(), + "{} of {} documentation test cases failed", + report.failed_count(), + report.total_count() + ); } - /// Executes every test case and returns the [`Report`]. + /// Executes every test case and returns the [`Report`], printing nothing. /// /// `Err` means the run could not be performed at all; test cases that failed are /// reported by the [`Report`] itself, see [`Report::passed`]. @@ -153,7 +152,7 @@ impl DocAssert { /// .await /// .unwrap(); /// - /// println!("{} of {} passed", report.passed_count(), report.executed_count()); + /// println!("{} of {} passed", report.passed_count(), report.total_count()); /// } /// ``` pub async fn run(self) -> Result { @@ -164,9 +163,6 @@ impl DocAssert { /// Parses the documentation and prepares the run without executing anything yet. /// - /// Use this to drive the run yourself and handle every test case as soon as it has - /// been executed. - /// /// # Examples /// /// ``` @@ -178,13 +174,13 @@ impl DocAssert { /// .start() /// .unwrap(); /// - /// println!("{} tests", run.total_count()); /// while let Some(result) = run.next().await { - /// println!("{}", result); + /// if !result.passed() { + /// break; // fail fast + /// } /// } /// /// let report = run.finish(); - /// println!("{}", report.summary()); /// } /// ``` pub fn start(self) -> Result { @@ -223,13 +219,12 @@ impl DocAssert { /// A run in progress, returned by [`DocAssert::start`]. /// -/// The test cases are executed one by one, as [`Run::next`] is called, so their results -/// are available while the run is still going on. They are executed in the order they -/// appear in the documentation because a test case may use variables extracted from the -/// responses of the previous ones. +/// The test cases are executed one by one, in the order they appear in the documentation, +/// because a test case may use variables extracted from the responses of the previous +/// ones. /// -/// Dropping a `Run` cancels it; the test cases that were already executed are lost with -/// it, so call [`Run::finish`] to get the [`Report`] of a partial run. +/// Dropping a `Run` discards the results of the test cases it executed; call +/// [`Run::finish`] to get the [`Report`] of the test cases executed so far. #[derive(Debug)] pub struct Run { url: String, @@ -240,40 +235,70 @@ pub struct Run { } impl Run { - /// Total number of test cases the documentation defines. - /// - /// This is how many test cases the run would execute if it were driven to the end; - /// a run that is stopped early reports fewer, see [`Report::executed_count`]. + /// Number of test cases the documentation defines pub fn total_count(&self) -> usize { self.total_count } - /// Number of test cases executed so far - pub fn completed_count(&self) -> usize { - self.results.len() - } - /// Executes the next test case, `None` once every one of them has been executed. /// /// # Cancellation /// /// This is not cancellation safe. The test case is taken off the queue before the /// request is sent, so dropping the returned future part way through — racing it - /// against a timeout, or selecting on it — loses that test case: it is neither - /// retried nor reported. Drive it to completion, and stop the run between calls. + /// against a timeout, or selecting on it — loses that test case: it is not reported, + /// even though its request may already have reached the server, and the variables it + /// would have extracted are missing for the test cases after it. Drive it to + /// completion, and stop the run between calls. pub async fn next(&mut self) -> Option { let (id, test_case) = self.pending.next()?; - let failure = executor::execute(&self.url, test_case, &mut self.variables) - .await - .err(); - let result = TestCaseResult::new(id, failure); - self.results.push(result.clone()); - Some(result) + Some(self.execute(id, test_case).await) + } + + /// Executes every remaining test case, printing each one to stdout as it is executed, + /// and returns the [`Report`]. + /// + /// The output follows `cargo test`: the number of test cases, then one line per test + /// case, printed as soon as it starts and completed with ✅ or ❌ once it is done, and + /// finally what a [`Report`] displays after those lines, see [`Report::summary`]: + /// + /// ```text + /// 2 tests + /// GET /blog (README.md:12) ✅ + /// POST /blog (README.md:30) ❌ + /// + /// failures: + /// ------------- + /// POST /blog (README.md:30): response at line 36: expected response code 201, got 500 + /// + /// test result: FAILED. 1 passed; 1 failed + /// ``` + pub async fn print_progress(mut self) -> Report { + println!("{} tests", self.total_count); + while let Some((id, test_case)) = self.pending.next() { + // the test case is named before it is executed so that a slow one, or one + // being retried, shows what is being waited for + print!("{} ", id); + let _ = std::io::stdout().flush(); + let result = self.execute(id, test_case).await; + println!("{}", result.mark()); + } + + let report = self.finish(); + println!("{}", report.summary()); + report } /// Returns the [`Report`] of the test cases executed so far. pub fn finish(self) -> Report { - Report::new(self.results) + Report::new(self.results, self.total_count) + } + + async fn execute(&mut self, id: TestCaseId, test_case: TestCase) -> TestCaseResult { + let outcome = executor::execute(&self.url, test_case, &mut self.variables).await; + let result = TestCaseResult::new(id, outcome); + self.results.push(result.clone()); + result } } @@ -282,6 +307,7 @@ impl Run { /// This is not how a failed test case is reported; a run that executed its test cases /// always produces a [`Report`], whether they passed or not. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum Error { /// No documentation file was given NoDocuments, @@ -299,7 +325,7 @@ impl Display for Error { match self { Error::NoDocuments => write!(f, "no documentation file to test"), Error::Parse { doc_path, reason } => { - write!(f, "error parsing {}: {}", doc_path, reason) + write!(f, "cannot parse {}: {}", doc_path, reason) } } } diff --git a/src/main.rs b/src/main.rs index 4d29fb6..89bc880 100644 --- a/src/main.rs +++ b/src/main.rs @@ -37,12 +37,12 @@ impl FromStr for JSONVars { #[doc(hidden)] macro_rules! handle_error { ($code:expr, $msg:expr, $($arg:tt)*) => { - println!($msg, $($arg)*); + eprintln!($msg, $($arg)*); std::process::exit($code); }; ($code:expr, $msg:expr) => { - println!($msg); + eprintln!($msg); std::process::exit($code); }; } @@ -95,13 +95,13 @@ async fn main() { for file in cli.files.iter() { let Some(file) = file.to_str() else { - handle_error!(Code::INVALID_ARGUMENT, "error: Invalid file path"); + handle_error!(Code::INVALID_ARGUMENT, "Error: invalid file path"); }; doc_assert = doc_assert.with_doc_path(file); } - let mut run = match doc_assert.start() { + let run = match doc_assert.start() { Ok(run) => run, Err(err @ Error::NoDocuments) => { handle_error!(Code::INVALID_ARGUMENT, "Error: {}", err); @@ -111,17 +111,7 @@ async fn main() { } }; - // the test cases are printed as they are executed, only the failures and the final - // result are left to print once the run is over - println!("{} tests", run.total_count()); - while let Some(result) = run.next().await { - println!("{}", result); - } - - let report = run.finish(); - println!("{}", report.summary()); - - if report.passed() { + if run.print_progress().await.passed() { std::process::exit(Code::SUCCESS); } std::process::exit(Code::DOC_ASSERTION_ERROR); diff --git a/src/parser.rs b/src/parser.rs index cd8d67d..25bc48b 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -87,7 +87,8 @@ pub(crate) fn parse(path: String) -> Result, String> { if responses.is_empty() || responses.len() != requests.len() { return Err(format!("misplaced retry at line {}: {}", line_no, line)); } - let retry_policy = get_retry_policy(line)?; + let retry_policy = get_retry_policy(line) + .map_err(|err| format!("invalid retry policy at line {}: {}", line_no, err))?; let l = responses.len(); responses[l - 1].retries = retry_policy; @@ -124,7 +125,7 @@ fn get_code(lines: &mut Enumerate) -> String { buff } -fn get_ignore_path(line: &str) -> Result { +fn get_ignore_path(line: &str) -> Result { let no_whitespace = line .chars() .filter(|c| !c.is_whitespace()) @@ -133,32 +134,28 @@ fn get_ignore_path(line: &str) -> Result { path.remove(0); path.pop(); - if let Err(e) = path.jsonpath() { - return Err(format!("invalid ignore path {}", e)); - } - - Ok(path) + path.jsonpath() + .map_err(|e| format!("invalid ignore path {}: {}", path, e)) } fn get_retry_policy(line: &str) -> Result { - let re = Regex::new(r"^\[retry\]:\s#\s\((?\d+),\s*(?\d+)\)").unwrap(); + let re = Regex::new(r"^\[retry\]:\s#\s\((?\d+),\s*(?\d+)\)").unwrap(); let caps = re .captures(line) .ok_or(format!("invalid retry properties: {}", line))?; - let max_retries = caps - .name("max_retries") + let max_attempts = caps + .name("max_attempts") .ok_or(format!("invalid retry properties: {}", line))? .as_str() .parse::() - .map_err(|e| format!("invalid max_retries: {}", e))?; + .map_err(|e| format!("invalid number of attempts: {}", e))?; - // a policy allowing no attempt at all would define a test case that can never be - // executed, so it is rejected here rather than reported as a failure later on - if max_retries == 0 { + // a test case allowing no attempt at all could never be executed + if max_attempts == 0 { return Err(format!( - "max_retries must be at least 1, got 0 in: {}", + "the number of attempts must be at least 1, got 0 in: {}", line )); } @@ -170,7 +167,10 @@ fn get_retry_policy(line: &str) -> Result { .parse::() .map_err(|e| format!("invalid delay: {}", e))?; - Ok(RetryPolicy { max_retries, delay }) + Ok(RetryPolicy { + max_attempts, + delay, + }) } fn get_variable_template(line: &str) -> Result<(String, Path), String> { @@ -278,6 +278,7 @@ fn get_headers_and_body( mod tests { use crate::{ domain::RetryPolicy, + json_diff::path::JSONPath, parser::{get_retry_policy, parse}, }; @@ -311,7 +312,10 @@ mod tests { test_cases[0].response.body.as_ref().unwrap(), "{\"id\": 1,\"name\": \"test\"}" ); - assert_eq!(test_cases[0].response.ignore_paths[0], "$.id".to_string()); + assert_eq!( + test_cases[0].response.ignore_paths[0], + "$.id".jsonpath().unwrap() + ); assert_eq!( test_cases[0] @@ -326,7 +330,7 @@ mod tests { assert_eq!( &test_cases[0].response.retries, &RetryPolicy { - max_retries: 3, + max_attempts: 3, delay: 4500 } ) @@ -336,7 +340,7 @@ mod tests { fn test_a_retry_policy_allowing_no_attempt_is_rejected() { let err = get_retry_policy("[retry]: # (0, 100)").unwrap_err(); assert!( - err.contains("max_retries must be at least 1"), + err.contains("the number of attempts must be at least 1"), "unexpected error: {}", err ); @@ -346,7 +350,7 @@ mod tests { fn test_a_retry_policy_of_one_attempt_is_accepted() { assert_eq!( Ok(RetryPolicy { - max_retries: 1, + max_attempts: 1, delay: 100 }), get_retry_policy("[retry]: # (1, 100)") diff --git a/src/report.rs b/src/report.rs index eedb362..4de5497 100644 --- a/src/report.rs +++ b/src/report.rs @@ -46,7 +46,7 @@ impl TestCaseId { &self.http_method } - /// URI the request is sent to + /// URI the request is sent to, as written in the documentation pub fn uri(&self) -> &str { &self.uri } @@ -56,7 +56,7 @@ impl TestCaseId { &self.doc_path } - /// Line number the request is defined at + /// Line number the request code block starts at pub fn line_number(&self) -> usize { self.line_number } @@ -79,11 +79,20 @@ impl Display for TestCaseId { #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum Failure { - /// A placeholder in the request or the expected response could not be resolved, - /// usually because no variable of that name was defined or extracted earlier + /// Placeholders in the request or the expected response could not be resolved, + /// usually because no variable of that name was defined or extracted earlier. + /// The request was not sent. UnresolvedVariables { - /// The URI, body or header value the placeholders were left in - input: String, + /// Names of the placeholders left, in the order they appear + names: Vec, + }, + /// The documentation describes something that cannot be checked, such as an + /// expected body that is not valid JSON. The request was not sent. + InvalidDocumentation { + /// Line number of the code block at fault + line_number: usize, + /// What is wrong with it + reason: String, }, /// The request could not be sent RequestFailed { @@ -103,9 +112,21 @@ pub enum Failure { impl Display for Failure { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Failure::UnresolvedVariables { input } => { - write!(f, "unresolved variable placeholders in {}", input) + Failure::UnresolvedVariables { names } => { + let names = names + .iter() + .map(|name| format!("`{}`", name)) + .collect::>(); + write!(f, "unresolved variables {}", names.join(", ")) } + Failure::InvalidDocumentation { + line_number, + reason, + } => write!( + f, + "invalid documentation at line {}: {}", + line_number, reason + ), Failure::RequestFailed { reason } => write!(f, "request failed: {}", reason), Failure::ResponseMismatch { line_number, cause } => { write!(f, "response at line {}: {}", line_number, cause) @@ -151,13 +172,6 @@ pub enum Mismatch { /// Name of the variable name: String, }, - /// An `[ignore]` or `[ignore-order]` JSONPath could not be parsed - InvalidIgnorePath { - /// The JSONPath as written in the documentation - path: String, - /// Why it could not be parsed - reason: String, - }, /// The response body could not be read UnreadableResponseBody { /// Why it could not be read @@ -168,11 +182,6 @@ pub enum Mismatch { /// Why it could not be parsed reason: String, }, - /// The body described in the documentation was not valid JSON - MalformedExpectedBody { - /// Why it could not be parsed - reason: String, - }, } impl Display for Mismatch { @@ -191,26 +200,24 @@ impl Display for Mismatch { name, expected, actual ), Mismatch::MissingHeader { name } => write!(f, "expected header {} not found", name), - Mismatch::Body { differences } => write!( - f, - "expected response differs from actual {}", - differences.join("\n") - ), + Mismatch::Body { differences } => { + write!(f, "body differs from the expected one:")?; + for difference in differences { + write!(f, "\n{}", difference)?; + } + Ok(()) + } Mismatch::VariableNotFound { name } => write!( f, "variable template {} not found in the response body", name ), - Mismatch::InvalidIgnorePath { path, reason } => { - write!(f, "invalid path {}: {}", path, reason) + Mismatch::UnreadableResponseBody { reason } => { + write!(f, "error reading the response body: {}", reason) } - Mismatch::UnreadableResponseBody { reason } => write!(f, "{}", reason), Mismatch::MalformedResponseBody { reason } => { write!(f, "error parsing JSON response from the server: {}", reason) } - Mismatch::MalformedExpectedBody { reason } => { - write!(f, "error parsing JSON: {}", reason) - } } } } @@ -221,12 +228,12 @@ impl Display for Mismatch { #[derive(Debug, Clone, PartialEq, Eq)] pub struct TestCaseResult { id: TestCaseId, - failure: Option, + outcome: Result<(), Failure>, } impl TestCaseResult { - pub(crate) fn new(id: TestCaseId, failure: Option) -> Self { - Self { id, failure } + pub(crate) fn new(id: TestCaseId, outcome: Result<(), Failure>) -> Self { + Self { id, outcome } } /// The test case this is the outcome of @@ -236,31 +243,39 @@ impl TestCaseResult { /// Whether the test case passed pub fn passed(&self) -> bool { - self.failure.is_none() + self.outcome.is_ok() } /// Why the test case failed, `None` if it passed pub fn failure(&self) -> Option<&Failure> { - self.failure.as_ref() + self.outcome.as_ref().err() + } + + /// What follows the [`TestCaseId`] on the line of the test case + pub(crate) fn mark(&self) -> &'static str { + if self.passed() { + "✅" + } else { + "❌" + } } } impl Display for TestCaseResult { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mark = if self.passed() { "✅" } else { "❌" }; - write!(f, "{} {}", self.id, mark) + write!(f, "{} {}", self.id, self.mark()) } } -/// Outcome of a whole run. +/// Outcome of a run. /// -/// A report is produced whenever the run completed, whether the test cases passed or not, -/// so it always describes what happened. Use [`Report::passed`] for the verdict. +/// Displaying a report renders the whole thing, the same way `doc-assert` prints it: the +/// number of test cases, one line per executed test case, the details of the failures +/// and the final result. /// -/// Displaying a report renders the whole thing: the number of test cases, one line per -/// test case, the details of the failures and the final result. When the test cases are -/// printed while they are executed, print [`Report::summary`] instead so that the lines -/// are not repeated. +/// A report produced by [`Run::finish`](crate::Run::finish) before the run was over only +/// holds the test cases that were executed. The ones left are counted as not run, and +/// such a report never [`passed`](Report::passed). /// /// # Examples /// @@ -273,34 +288,40 @@ impl Display for TestCaseResult { /// .await /// .unwrap(); /// println!("{}", report); -/// assert!(report.passed()); /// } /// ``` #[derive(Debug, Clone, PartialEq, Eq)] pub struct Report { results: Vec, + total_count: usize, } impl Report { - pub(crate) fn new(results: Vec) -> Self { - Self { results } + pub(crate) fn new(results: Vec, total_count: usize) -> Self { + Self { + results, + total_count, + } } - /// Outcome of every test case, in the order they were executed + /// Outcome of every executed test case, in the order they were executed pub fn results(&self) -> &[TestCaseResult] { &self.results } - /// Outcome of every test case that failed - pub fn failures(&self) -> impl Iterator { - self.results.iter().filter(|r| !r.passed()) + /// Every test case that failed, along with the reason it failed + pub fn failures(&self) -> impl Iterator { + self.results + .iter() + .filter_map(|r| r.failure().map(|failure| (r.id(), failure))) } - /// Number of test cases that were executed. - /// - /// A run that was stopped early only reports the test cases it got to, so this is - /// not necessarily the number of test cases the documentation defines, which is - /// what [`Run::total_count`](crate::Run::total_count) gives. + /// Number of test cases the documentation defines + pub fn total_count(&self) -> usize { + self.total_count + } + + /// Number of test cases that were executed pub fn executed_count(&self) -> usize { self.results.len() } @@ -315,35 +336,29 @@ impl Report { self.executed_count() - self.passed_count() } - /// Whether every executed test case passed + /// Number of test cases that were not executed because the run was stopped early + pub fn not_run_count(&self) -> usize { + self.total_count - self.executed_count() + } + + /// Whether every test case the documentation defines was executed and passed pub fn passed(&self) -> bool { - self.failed_count() == 0 + self.failed_count() == 0 && self.not_run_count() == 0 } /// The details of the failures followed by the final result. /// - /// This is everything a [`Report`] displays except the number of test cases and the - /// line of every test case, so it is what is left to print once the test cases have - /// been printed as they were executed. - pub fn summary(&self) -> Summary<'_> { + /// This is what a [`Report`] displays after the line of every test case, so it is + /// what is left to print once those lines have been printed as the test cases were + /// executed. + pub fn summary(&self) -> impl Display + '_ { Summary(self) } - - /// Panics with the whole report if any test case failed. - /// - /// Meant for use inside a test, where a failed test case should fail the test. - /// - /// # Panics - /// - /// Panics if [`Report::passed`] is `false`. - pub fn assert_passed(&self) { - assert!(self.passed(), "{}", self); - } } impl Display for Report { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - writeln!(f, "{} tests", self.executed_count())?; + writeln!(f, "{} tests", self.total_count)?; for result in &self.results { writeln!(f, "{}", result)?; } @@ -351,20 +366,15 @@ impl Display for Report { } } -/// The details of the failures followed by the final result, returned by [`Report::summary`]. -#[derive(Debug)] -pub struct Summary<'a>(&'a Report); +struct Summary<'a>(&'a Report); impl Display for Summary<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let report = self.0; - if !report.passed() { + if report.failed_count() > 0 { writeln!(f, "\nfailures:")?; - for result in report.failures() { - // `failure` is always set on a result that did not pass - if let Some(failure) = result.failure() { - writeln!(f, "-------------\n{}: {}", result.id(), failure)?; - } + for (id, failure) in report.failures() { + writeln!(f, "-------------\n{}: {}", id, failure)?; } } write!( @@ -373,7 +383,11 @@ impl Display for Summary<'_> { if report.passed() { "PASSED" } else { "FAILED" }, report.passed_count(), report.failed_count() - ) + )?; + if report.not_run_count() > 0 { + write!(f, "; {} not run", report.not_run_count())?; + } + Ok(()) } } @@ -381,47 +395,185 @@ impl Display for Summary<'_> { mod tests { use super::{Failure, Mismatch, Report, TestCaseId, TestCaseResult}; + fn id(http_method: &str, line_number: usize) -> TestCaseId { + TestCaseId::new( + http_method.to_string(), + "/blog".to_string(), + "README.md".to_string(), + line_number, + ) + } + + fn status_code_failure() -> Failure { + Failure::ResponseMismatch { + line_number: 36, + cause: Mismatch::StatusCode { + expected: 201, + actual: 500, + }, + } + } + + #[test] + fn test_report_of_a_passing_run() { + let report = Report::new(vec![TestCaseResult::new(id("GET", 12), Ok(()))], 1); + + assert!(report.passed()); + assert_eq!( + "1 tests\n\ + GET /blog (README.md:12) ✅\n\ + \n\ + test result: PASSED. 1 passed; 0 failed", + report.to_string() + ); + } + #[test] - fn test_report_renders_the_same_thing_as_the_streamed_output() { - let report = Report::new(vec![ - TestCaseResult::new( - TestCaseId::new( - "GET".to_string(), - "/blog".to_string(), - "README.md".to_string(), - 12, + fn test_report_of_a_failing_run() { + let report = Report::new( + vec![ + TestCaseResult::new(id("GET", 12), Ok(())), + TestCaseResult::new(id("POST", 30), Err(status_code_failure())), + TestCaseResult::new( + id("DELETE", 40), + Err(Failure::RequestFailed { + reason: "connection refused".to_string(), + }), ), - None, + ], + 3, + ); + + assert!(!report.passed()); + assert_eq!(2, report.failures().count()); + assert_eq!( + "3 tests\n\ + GET /blog (README.md:12) ✅\n\ + POST /blog (README.md:30) ❌\n\ + DELETE /blog (README.md:40) ❌\n\ + \n\ + failures:\n\ + -------------\n\ + POST /blog (README.md:30): response at line 36: expected response code 201, got 500\n\ + -------------\n\ + DELETE /blog (README.md:40): request failed: connection refused\n\ + \n\ + test result: FAILED. 1 passed; 2 failed", + report.to_string() + ); + } + + #[test] + fn test_report_of_a_run_stopped_early_does_not_pass() { + let report = Report::new(vec![TestCaseResult::new(id("GET", 12), Ok(()))], 3); + + assert_eq!(2, report.not_run_count()); + assert!(!report.passed()); + assert_eq!( + "3 tests\n\ + GET /blog (README.md:12) ✅\n\ + \n\ + test result: FAILED. 1 passed; 0 failed; 2 not run", + report.to_string() + ); + } + + #[test] + fn test_report_without_test_cases_passes() { + let report = Report::new(vec![], 0); + + assert!(report.passed()); + assert_eq!( + "0 tests\n\ntest result: PASSED. 0 passed; 0 failed", + report.to_string() + ); + } + + #[test] + fn test_failures_are_rendered() { + let cases = [ + ( + Failure::UnresolvedVariables { + names: vec!["id".to_string(), "token".to_string()], + }, + "unresolved variables `id`, `token`", ), - TestCaseResult::new( - TestCaseId::new( - "POST".to_string(), - "/blog".to_string(), - "README.md".to_string(), - 30, - ), - Some(Failure::ResponseMismatch { - line_number: 36, - cause: Mismatch::StatusCode { - expected: 201, - actual: 500, - }, - }), + ( + Failure::InvalidDocumentation { + line_number: 7, + reason: "expected body is not valid JSON".to_string(), + }, + "invalid documentation at line 7: expected body is not valid JSON", ), - ]); - - // what the binary prints line by line has to add up to what `Report` displays - let streamed = format!( - "{} tests\n{}\n{}\n", - report.executed_count(), - report - .results() - .iter() - .map(|r| r.to_string()) - .collect::>() - .join("\n"), - report.summary() - ); - assert_eq!(format!("{}\n", report), streamed); + ( + Failure::RequestFailed { + reason: "connection refused".to_string(), + }, + "request failed: connection refused", + ), + ( + status_code_failure(), + "response at line 36: expected response code 201, got 500", + ), + ]; + + for (failure, expected) in cases { + assert_eq!(expected, failure.to_string()); + } + } + + #[test] + fn test_mismatches_are_rendered() { + let cases = [ + ( + Mismatch::StatusCode { + expected: 200, + actual: 404, + }, + "expected response code 200, got 404", + ), + ( + Mismatch::Header { + name: "Content-Type".to_string(), + expected: "application/json".to_string(), + actual: "text/plain".to_string(), + }, + "expected header Content-Type to be application/json, got text/plain", + ), + ( + Mismatch::MissingHeader { + name: "Content-Type".to_string(), + }, + "expected header Content-Type not found", + ), + ( + Mismatch::Body { + differences: vec!["first".to_string(), "second".to_string()], + }, + "body differs from the expected one:\nfirst\nsecond", + ), + ( + Mismatch::VariableNotFound { + name: "id".to_string(), + }, + "variable template id not found in the response body", + ), + ( + Mismatch::UnreadableResponseBody { + reason: "connection reset".to_string(), + }, + "error reading the response body: connection reset", + ), + ( + Mismatch::MalformedResponseBody { + reason: "expected value".to_string(), + }, + "error parsing JSON response from the server: expected value", + ), + ]; + + for (mismatch, expected) in cases { + assert_eq!(expected, mismatch.to_string()); + } } } diff --git a/src/variables.rs b/src/variables.rs index c5aeb2d..d4f366b 100644 --- a/src/variables.rs +++ b/src/variables.rs @@ -18,8 +18,10 @@ use crate::{ json_diff::path::{Key, Path}, report::{Failure, Mismatch}, }; +use regex::Regex; use serde_json::Value; use std::collections::HashMap; +use std::sync::OnceLock; /// Variables to be used in the request and response bodies. /// @@ -101,19 +103,25 @@ impl Variables { self.map.insert(name.into(), value.into()); } - pub(crate) fn obtain_from_response( - &mut self, + /// Extracts the variables the documentation defines from a response body, leaving the + /// ones already known untouched: they are only updated once the response passed, see + /// [`Variables::extend`]. + pub(crate) fn extract_from_response( response: &Value, variable_templates: &HashMap, - ) -> Result<(), Mismatch> { - for (name, path) in variable_templates { - let value = extract_value(path, response) - .ok_or_else(|| Mismatch::VariableNotFound { name: name.clone() })?; - - self.map.insert(name.clone(), value); - } + ) -> Result, Mismatch> { + variable_templates + .iter() + .map(|(name, path)| { + extract_value(path, response) + .map(|value| (name.clone(), value)) + .ok_or_else(|| Mismatch::VariableNotFound { name: name.clone() }) + }) + .collect() + } - Ok(()) + pub(crate) fn extend(&mut self, variables: HashMap) { + self.map.extend(variables); } fn replace_placeholders(&self, input: &mut String, trim_quotes: bool) -> Result<(), Failure> { @@ -130,10 +138,11 @@ impl Variables { *input = input.replace(&placeholder, value); } - if input.contains('`') { - return Err(Failure::UnresolvedVariables { - input: input.clone(), - }); + // only the names are reported: the input may hold the values of other variables, + // such as tokens, which should not end up in a report + let names = unresolved_placeholders(input); + if !names.is_empty() { + return Err(Failure::UnresolvedVariables { names }); } Ok(()) @@ -169,6 +178,22 @@ impl Variables { } } +/// Names of the placeholders left in `input`, in the order they appear and without +/// repetitions. +fn unresolved_placeholders(input: &str) -> Vec { + static PLACEHOLDER: OnceLock = OnceLock::new(); + let placeholder = PLACEHOLDER.get_or_init(|| Regex::new(r"`([^`\s]+)`").unwrap()); + + let mut names: Vec = vec![]; + for caps in placeholder.captures_iter(input) { + let name = &caps[1]; + if !names.iter().any(|n| n == name) { + names.push(name.to_string()); + } + } + names +} + fn extract_value(path: &Path, value: &Value) -> Option { match path { Path::Root => None, @@ -185,3 +210,84 @@ fn extract_value(path: &Path, value: &Value) -> Option { } } } + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use serde_json::{json, Value}; + + use super::Variables; + use crate::json_diff::path::JSONPath; + use crate::report::{Failure, Mismatch}; + + #[test] + fn test_from_json_accepts_only_an_object() { + assert!(Variables::from_json(&json!({"id": 1})).is_some()); + for json in [json!("id"), json!([1]), json!(1), Value::Null] { + assert!(Variables::from_json(&json).is_none(), "{}", json); + } + } + + #[test] + fn test_insert_overwrites_a_variable_of_the_same_name() { + let mut variables = Variables::new(); + variables.insert("id", "x"); + variables.insert("id", 1); + + let mut input = "/users/`id`".to_string(); + variables.replace_placeholders(&mut input, true).unwrap(); + assert_eq!("/users/1", input); + } + + #[test] + fn test_unresolved_placeholders_are_reported_by_name_only() { + let mut variables = Variables::new(); + variables.insert("token", "secret"); + + let mut input = "`token` `id` `name` `id`".to_string(); + let err = variables + .replace_placeholders(&mut input, false) + .unwrap_err(); + + assert_eq!( + Failure::UnresolvedVariables { + names: vec!["id".to_string(), "name".to_string()] + }, + err + ); + assert!(!err.to_string().contains("secret")); + } + + #[test] + fn test_a_lone_backtick_is_not_a_placeholder() { + let mut input = "{\"quote\": \"it`s\"}".to_string(); + assert_eq!( + Ok(()), + Variables::new().replace_placeholders(&mut input, false) + ); + } + + #[test] + fn test_every_variable_or_none_is_extracted_from_a_response() { + let templates: HashMap<_, _> = [ + ("id".to_string(), "$.id".jsonpath().unwrap()), + ("name".to_string(), "$.name".jsonpath().unwrap()), + ] + .into_iter() + .collect(); + + let extracted = + Variables::extract_from_response(&json!({"id": 1, "name": "John"}), &templates) + .unwrap(); + assert_eq!(Some(&json!(1)), extracted.get("id")); + assert_eq!(Some(&json!("John")), extracted.get("name")); + + assert_eq!( + Err(Mismatch::VariableNotFound { + name: "name".to_string() + }), + Variables::extract_from_response(&json!({"id": 1}), &templates) + ); + } +} diff --git a/tests/api.rs b/tests/api.rs index 33d4840..de6d2c2 100644 --- a/tests/api.rs +++ b/tests/api.rs @@ -14,92 +14,65 @@ //! Exercises the library the way a test suite depending on it would: through its public //! API only, compiled as a separate crate. -use doc_assert::{DocAssert, Error, Failure, Mismatch}; - -/// Documentation file removing itself once it goes out of scope. -struct TempDoc { - path: std::path::PathBuf, -} - -impl TempDoc { - fn new(content: &str) -> Self { - let path = std::env::temp_dir().join(format!( - "doc_assert_{}_{}.md", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - std::fs::write(&path, content).unwrap(); - Self { path } - } +mod common; - fn path(&self) -> &str { - self.path.to_str().unwrap() - } -} +use common::{TempDoc, DOC}; +use doc_assert::{DocAssert, Error, Failure, Mismatch}; -impl Drop for TempDoc { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - } +struct Server { + server: mockito::ServerGuard, + passing: mockito::Mock, + failing: mockito::Mock, } -/// One test case hitting `/passing`, then one hitting `/failing`. -const DOC: &str = "```docassertrequest\n\ - GET /passing\n\ - ```\n\ - ```docassertresponse\n\ - HTTP 200\n\ - ```\n\ - ```docassertrequest\n\ - GET /failing\n\ - ```\n\ - ```docassertresponse\n\ - HTTP 200\n\ - ```\n"; - -async fn server() -> mockito::ServerGuard { +/// Serves `/passing` with a 200 and `/failing` with a 500. +async fn server() -> Server { let mut server = mockito::Server::new_async().await; - server + let passing = server .mock("GET", "/passing") .with_status(200) .create_async() .await; - server + let failing = server .mock("GET", "/failing") .with_status(500) .create_async() .await; - server + Server { + server, + passing, + failing, + } } #[tokio::test] -async fn test_every_test_case_is_available_as_soon_as_it_is_executed() { +async fn test_test_cases_are_executed_one_at_a_time() { let server = server().await; let doc = TempDoc::new(DOC); - let mut run = DocAssert::new(server.url()) + let mut run = DocAssert::new(server.server.url()) .with_doc_path(doc.path()) .start() .unwrap(); // the count is known before anything has been executed assert_eq!(2, run.total_count()); - assert_eq!(0, run.completed_count()); + assert!(!server.passing.matched_async().await); let first = run.next().await.unwrap(); assert!(first.passed()); assert_eq!("GET", first.id().http_method()); assert_eq!("/passing", first.id().uri()); - // the second test case has not been executed at the point the first is returned - assert_eq!(1, run.completed_count()); + assert!(server.passing.matched_async().await); + // the result of the first is available before the second has been sent + assert!(!server.failing.matched_async().await); let second = run.next().await.unwrap(); assert!(!second.passed()); assert_eq!("/failing", second.id().uri()); + assert!(server.failing.matched_async().await); + assert!(run.next().await.is_none()); assert!(run.next().await.is_none()); let report = run.finish(); @@ -107,39 +80,42 @@ async fn test_every_test_case_is_available_as_soon_as_it_is_executed() { assert_eq!(1, report.passed_count()); assert_eq!(1, report.failed_count()); assert!(!report.passed()); - assert_eq!(1, report.failures().count()); } #[tokio::test] -async fn test_failure_carries_the_reason_it_failed() { +async fn test_failure_says_where_and_why_it_failed() { let server = server().await; let doc = TempDoc::new(DOC); - let report = DocAssert::new(server.url()) + let report = DocAssert::new(server.server.url()) .with_doc_path(doc.path()) .run() .await .unwrap(); - let failure = report.failures().next().unwrap().failure().unwrap(); - match failure { - Failure::ResponseMismatch { cause, .. } => assert_eq!( - &Mismatch::StatusCode { + let failures = report.failures().collect::>(); + assert_eq!(1, failures.len()); + let (id, failure) = failures[0]; + assert_eq!(doc.path(), id.doc_path()); + assert_eq!(7, id.line_number()); + assert_eq!( + &Failure::ResponseMismatch { + line_number: 10, + cause: Mismatch::StatusCode { expected: 200, actual: 500 - }, - cause - ), - other => panic!("unexpected failure: {:?}", other), - } + } + }, + failure + ); } #[tokio::test] -async fn test_run_can_be_stopped_early_and_still_report() { +async fn test_run_stopped_early_reports_what_was_not_run() { let server = server().await; let doc = TempDoc::new(DOC); - let mut run = DocAssert::new(server.url()) + let mut run = DocAssert::new(server.server.url()) .with_doc_path(doc.path()) .start() .unwrap(); @@ -147,9 +123,93 @@ async fn test_run_can_be_stopped_early_and_still_report() { run.next().await.unwrap(); let report = run.finish(); - // only the test cases that were executed are reported + assert_eq!(2, report.total_count()); assert_eq!(1, report.executed_count()); - assert!(report.passed()); + assert_eq!(1, report.not_run_count()); + // nothing failed, but not everything passed either + assert!(!report.passed()); + assert!(!server.failing.matched_async().await); +} + +#[tokio::test] +async fn test_documentation_files_run_in_order_and_share_variables() { + let mut server = mockito::Server::new_async().await; + server + .mock("POST", "/users") + .with_status(201) + .with_body(r#"{"id": 42}"#) + .create_async() + .await; + server + .mock("GET", "/users/42") + .with_status(200) + .create_async() + .await; + let create = TempDoc::new( + "```docassertrequest\n\ + POST /users\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 201\n\ + {\"id\": 0}\n\ + ```\n\ + [ignore]: # ($.id)\n\ + [let id]: # ($.id)\n", + ); + let get = TempDoc::new( + "```docassertrequest\n\ + GET /users/`id`\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n", + ); + + let report = DocAssert::new(server.url()) + .with_doc_path(create.path()) + .with_doc_path(get.path()) + .run() + .await + .unwrap(); + + assert!(report.passed(), "{}", report); + let doc_paths = report + .results() + .iter() + .map(|r| r.id().doc_path()) + .collect::>(); + assert_eq!(vec![create.path(), get.path()], doc_paths); +} + +#[tokio::test] +async fn test_parsing_error_stops_the_run_before_any_request() { + let server = server().await; + let valid = TempDoc::new(DOC); + let invalid = TempDoc::new( + "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n\ + [retry]: # (0, 10)\n", + ); + + let err = DocAssert::new(server.server.url()) + .with_doc_path(valid.path()) + .with_doc_path(invalid.path()) + .start() + .unwrap_err(); + + match err { + Error::Parse { doc_path, reason } => { + assert_eq!(invalid.path(), doc_path); + assert!(reason.contains("at line 7"), "{}", reason); + assert!(reason.contains("at least 1"), "{}", reason); + } + other => panic!("unexpected error: {:?}", other), + } + assert!(!server.passing.matched_async().await); } #[test] @@ -172,3 +232,42 @@ fn test_parsing_error_names_the_file_it_comes_from() { other => panic!("unexpected error: {:?}", other), } } + +#[tokio::test] +async fn test_assert_passes_when_every_test_case_does() { + let server = server().await; + let doc = TempDoc::new( + "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n", + ); + + DocAssert::new(server.server.url()) + .with_doc_path(doc.path()) + .assert() + .await; +} + +#[tokio::test] +#[should_panic(expected = "1 of 2 documentation test cases failed")] +async fn test_assert_panics_when_a_test_case_fails() { + let server = server().await; + let doc = TempDoc::new(DOC); + + DocAssert::new(server.server.url()) + .with_doc_path(doc.path()) + .assert() + .await; +} + +#[tokio::test] +#[should_panic(expected = "cannot parse this/file/does/not/exist.md")] +async fn test_assert_panics_when_the_run_cannot_be_performed() { + DocAssert::new("http://localhost:8080") + .with_doc_path("this/file/does/not/exist.md") + .assert() + .await; +} diff --git a/tests/cli.rs b/tests/cli.rs new file mode 100644 index 0000000..e6aae09 --- /dev/null +++ b/tests/cli.rs @@ -0,0 +1,301 @@ +// Copyright 2024 The DocAssert Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Exercises the `doc-assert` binary: what it prints, when it prints it, and how it exits. + +mod common; + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::process::{Child, Command, Output, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::{self, Receiver}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use common::{TempDoc, DOC}; + +/// How long the binary is given to print what is expected of it +const TIMEOUT: Duration = Duration::from_secs(30); + +/// Minimal HTTP server answering every request with the status `route` gives its path. +struct Server { + url: String, + requests: Arc, +} + +impl Server { + fn start(route: impl Fn(&str) -> u16 + Send + Sync + 'static) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicUsize::new(0)); + let route = Arc::new(route); + + let counter = requests.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { return }; + let route = route.clone(); + let counter = counter.clone(); + std::thread::spawn(move || { + let mut reader = BufReader::new(stream); + let mut request_line = String::new(); + reader.read_line(&mut request_line).unwrap(); + // the rest of the head is read before answering; none of the requests + // the tests send has a body + let mut line = String::new(); + while reader.read_line(&mut line).unwrap() > 2 { + line.clear(); + } + counter.fetch_add(1, Ordering::SeqCst); + + let path = request_line.split_whitespace().nth(1).unwrap_or(""); + let status = route(path); + let _ = write!( + reader.get_mut(), + "HTTP/1.1 {} Status\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + status + ); + }); + } + }); + + Self { url, requests } + } + + fn requests(&self) -> usize { + self.requests.load(Ordering::SeqCst) + } +} + +fn doc_assert(args: &[&str]) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_doc-assert")); + command.args(args); + command +} + +fn run(args: &[&str]) -> Output { + doc_assert(args).output().unwrap() +} + +fn stdout(output: &Output) -> String { + String::from_utf8(output.stdout.clone()).unwrap() +} + +fn stderr(output: &Output) -> String { + String::from_utf8(output.stderr.clone()).unwrap() +} + +/// Kills the binary if a test gives up on it. +struct Running(Child); + +impl Drop for Running { + fn drop(&mut self) { + let _ = self.0.kill(); + } +} + +/// Forwards everything the binary prints as soon as it is printed. +fn forward_stdout(child: &mut Child) -> Receiver> { + let mut stdout = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let mut buf = [0; 1024]; + while let Ok(n) = stdout.read(&mut buf) { + if n == 0 || tx.send(buf[..n].to_vec()).is_err() { + return; + } + } + }); + rx +} + +/// Accumulates the output into `printed` until it holds `expected`, failing the test if it +/// takes too long or the output does not match. +fn expect_printed(output: &Receiver>, printed: &mut Vec, expected: &str) { + while printed.len() < expected.len() { + let chunk = output.recv_timeout(TIMEOUT).unwrap_or_else(|_| { + panic!( + "expected {:?} to be printed, got {:?}", + expected, + String::from_utf8_lossy(printed) + ) + }); + printed.extend(chunk); + } + assert_eq!(expected, String::from_utf8_lossy(printed)); +} + +#[test] +fn test_every_test_case_is_printed_as_it_is_executed() { + // `/failing` is held until the test releases it, so whatever is printed before that + // was printed while the run was still going on + let (release, released) = mpsc::channel::<()>(); + let released = Mutex::new(released); + let server = Server::start(move |path| { + if path == "/failing" { + let _ = released.lock().unwrap().recv(); + } + 200 + }); + let doc = TempDoc::new(DOC); + + let mut child = doc_assert(&["--url", &server.url, doc.path()]) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let output = forward_stdout(&mut child); + let mut child = Running(child); + + let mut printed = vec![]; + let while_running = format!( + "2 tests\n\ + GET /passing ({path}:1) ✅\n\ + GET /failing ({path}:7) ", + path = doc.path() + ); + // the test case being executed is named before its response comes back + expect_printed(&output, &mut printed, &while_running); + + release.send(()).unwrap(); + let status = child.0.wait().unwrap(); + while let Ok(chunk) = output.recv_timeout(TIMEOUT) { + printed.extend(chunk); + } + + assert_eq!( + format!( + "{}✅\n\ + \n\ + test result: PASSED. 2 passed; 0 failed\n", + while_running + ), + String::from_utf8_lossy(&printed) + ); + assert_eq!(Some(0), status.code()); +} + +#[test] +fn test_failures_are_detailed_once_every_test_case_was_executed() { + let server = Server::start(|path| if path == "/failing" { 500 } else { 200 }); + let doc = TempDoc::new(DOC); + + let output = run(&["--url", &server.url, doc.path()]); + + assert_eq!( + format!( + "2 tests\n\ + GET /passing ({path}:1) ✅\n\ + GET /failing ({path}:7) ❌\n\ + \n\ + failures:\n\ + -------------\n\ + GET /failing ({path}:7): response at line 10: expected response code 200, got 500\n\ + \n\ + test result: FAILED. 1 passed; 1 failed\n", + path = doc.path() + ), + stdout(&output) + ); + assert_eq!(Some(4), output.status.code()); +} + +#[test] +fn test_variables_are_substituted() { + let server = Server::start(|path| if path == "/users/42" { 200 } else { 404 }); + let doc = TempDoc::new( + "```docassertrequest\n\ + GET /users/`id`\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n", + ); + + let output = run(&["--url", &server.url, "-v", r#"{"id": 42}"#, doc.path()]); + + assert_eq!(Some(0), output.status.code(), "{}", stdout(&output)); +} + +#[test] +fn test_an_unreachable_server_fails_the_test_cases() { + let doc = TempDoc::new(DOC); + + let output = run(&["--url", "http://127.0.0.1:1", doc.path()]); + + assert!(stdout(&output).contains("request failed: ")); + assert_eq!(Some(4), output.status.code()); +} + +#[test] +fn test_invalid_arguments_are_rejected() { + let doc = TempDoc::new(DOC); + let cases: [(&[&str], &str); 4] = [ + (&[], "Error: no documentation file to test"), + ( + &["-v", r#""id""#, doc.path()], + "Error: Variables must be a JSON object", + ), + ( + &["-v", "[1]", doc.path()], + "Error: Variables must be a JSON object", + ), + (&["-v", "{", doc.path()], "invalid value"), + ]; + + for (args, message) in cases { + let args = [&["--url", "http://127.0.0.1:1"], args].concat(); + let output = run(&args); + + assert_eq!(Some(2), output.status.code(), "{:?}", args); + assert!( + stderr(&output).contains(message), + "{:?}: {}", + args, + stderr(&output) + ); + assert_eq!("", stdout(&output), "{:?}", args); + } +} + +#[test] +fn test_documentation_that_cannot_be_parsed_is_rejected_before_any_request() { + let server = Server::start(|_| 200); + let valid = TempDoc::new(DOC); + let invalid = TempDoc::new( + "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n\ + [retry]: # (0, 10)\n", + ); + + for args in [ + vec![valid.path(), "this/file/does/not/exist.md"], + vec![valid.path(), invalid.path()], + ] { + let args = [vec!["--url", server.url.as_str()], args].concat(); + let output = run(&args); + + assert_eq!(Some(3), output.status.code(), "{:?}", args); + assert!( + stderr(&output).starts_with("Error: cannot parse "), + "{:?}: {}", + args, + stderr(&output) + ); + } + assert_eq!(0, server.requests()); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000..e849235 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,60 @@ +// Copyright 2024 The DocAssert Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Helpers shared by the integration tests. + +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// One test case requesting `GET /passing`, then one requesting `GET /failing`, both +/// expecting a 200. The requests are defined at lines 1 and 7, the responses at 4 and 10. +pub const DOC: &str = "```docassertrequest\n\ + GET /passing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n\ + ```docassertrequest\n\ + GET /failing\n\ + ```\n\ + ```docassertresponse\n\ + HTTP 200\n\ + ```\n"; + +/// Documentation file removing itself once it goes out of scope. +pub struct TempDoc { + path: std::path::PathBuf, +} + +impl TempDoc { + pub fn new(content: &str) -> Self { + // tests run in parallel, so every file of this process gets a number of its own + static NEXT: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "doc_assert_{}_{}.md", + std::process::id(), + NEXT.fetch_add(1, Ordering::SeqCst) + )); + std::fs::write(&path, content).unwrap(); + Self { path } + } + + pub fn path(&self) -> &str { + self.path.to_str().unwrap() + } +} + +impl Drop for TempDoc { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} From 21ecb4c8aa92d5632d3c72f20f725f4a179d0368 Mon Sep 17 00:00:00 2001 From: dogenkigen Date: Fri, 2 Oct 2026 17:19:23 +0200 Subject: [PATCH 6/6] more --- README.md | 3 ++- src/lib.rs | 13 +++++++------ tests/api.rs | 23 +++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9684996..3cb0b38 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,8 @@ async fn fail_fast() { ``` Stop the run between calls to `next` rather than racing it against a timeout or selecting on it: a test case -dropped part way through is lost, even though its request may already have reached the server. +dropped part way through is abandoned and reported as not run, even though its request may already have reached +the server. Failures are structured, so you can inspect them, or render them your own way, by matching on them: diff --git a/src/lib.rs b/src/lib.rs index 96a1468..30080e5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -244,12 +244,13 @@ impl Run { /// /// # Cancellation /// - /// This is not cancellation safe. The test case is taken off the queue before the - /// request is sent, so dropping the returned future part way through — racing it - /// against a timeout, or selecting on it — loses that test case: it is not reported, - /// even though its request may already have reached the server, and the variables it - /// would have extracted are missing for the test cases after it. Drive it to - /// completion, and stop the run between calls. + /// This is not cancellation safe. The test case is taken off the queue before its + /// request is sent, so dropping the returned future part way through, for instance by + /// racing it against a timeout or selecting on it, abandons that test case and the + /// next call moves on to the one after it. The abandoned test case is reported as not + /// run, so the [`Report`] does not pass, and the variables it would have extracted + /// are not defined. Its request may still have reached the server. Drive the future + /// to completion, and stop the run between calls. pub async fn next(&mut self) -> Option { let (id, test_case) = self.pending.next()?; Some(self.execute(id, test_case).await) diff --git a/tests/api.rs b/tests/api.rs index de6d2c2..c278b2d 100644 --- a/tests/api.rs +++ b/tests/api.rs @@ -16,6 +16,8 @@ mod common; +use std::time::Duration; + use common::{TempDoc, DOC}; use doc_assert::{DocAssert, Error, Failure, Mismatch}; @@ -131,6 +133,27 @@ async fn test_run_stopped_early_reports_what_was_not_run() { assert!(!server.failing.matched_async().await); } +#[tokio::test] +async fn test_test_case_dropped_part_way_through_is_reported_as_not_run() { + // accepts connections but never answers, so the request is still in flight when the + // future is dropped + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let doc = TempDoc::new(DOC); + + let mut run = DocAssert::new(format!("http://{}", listener.local_addr().unwrap())) + .with_doc_path(doc.path()) + .start() + .unwrap(); + + let timed_out = tokio::time::timeout(Duration::from_millis(100), run.next()).await; + assert!(timed_out.is_err()); + let report = run.finish(); + + assert_eq!(0, report.executed_count()); + assert_eq!(2, report.not_run_count()); + assert!(!report.passed()); +} + #[tokio::test] async fn test_documentation_files_run_in_order_and_share_variables() { let mut server = mockito::Server::new_async().await;