diff --git a/CHANGELOG.md b/CHANGELOG.md index 89f2dc3..14d92a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,3 +16,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - Node.js bindings (`markkit-js` crate) - Benchmark harness (`markkit-bench` crate) - LLM evaluation harness (`markkit-eval` crate) +- **HTML → Markdown pipeline** via canonical IR (`ir.rs`, `html/parser.rs`, `render/markdown.rs`) + - Full element mapping: headings, paragraphs, lists (nested), blockquotes, code blocks, tables (GFM), inline formatting, links, images, thematic breaks, line breaks + - Configurable CSS include/exclude selectors + - Depth limiting (MAX_DEPTH=200) to prevent stack overflow + - Whitespace normalization with inter-element space preservation + - Non-content element stripping (script, style, iframe, form, svg, etc.) + - Resource limit enforcement (max input size) +- **Canonical IR** with serde JSON serialization support +- **Sync API**: `extract_html_sync()` and `parse_html()` for non-async usage +- **CLI stdin support**: `--stdin` flag and `-` source convention +- **Python bindings**: `extract_html_sync()` and `extract_html_sync_with_options()` with keyword args +- **JS bindings**: `extract_html_sync()` synchronous function +- **Specs**: IR specification, threat model, comprehensive HTML extraction spec, examples spec +- **Examples**: `basic.rs`, `custom_selectors.rs`, `ir_access.rs`, `configuration.rs` +- **Test fixtures**: 9 paired input/output HTML→Markdown fixture files diff --git a/crates/markkit-cli/src/main.rs b/crates/markkit-cli/src/main.rs index c8b52e2..06020f4 100644 --- a/crates/markkit-cli/src/main.rs +++ b/crates/markkit-cli/src/main.rs @@ -1,3 +1,5 @@ +use std::io::Read; + use anyhow::Result; use clap::{Parser, Subcommand}; use markkit::Markkit; @@ -17,13 +19,17 @@ struct Cli { enum Commands { /// Extract markdown from a URL or file Extract { - /// URL or file path to extract from - source: String, + /// URL or file path to extract from (use "-" to read from stdin) + source: Option, /// Treat source as a file path instead of URL #[arg(long)] file: bool, + /// Read HTML from stdin + #[arg(long)] + stdin: bool, + /// CSS selector to include (can be repeated) #[arg(long = "include", short = 'i')] include_selectors: Vec, @@ -50,11 +56,18 @@ async fn main() -> Result<()> { Commands::Extract { source, file, + stdin, include_selectors, exclude_selectors, max_size, json: output_json, } => { + let read_stdin = stdin || source.as_deref() == Some("-"); + + if !read_stdin && source.is_none() { + anyhow::bail!("source is required unless --stdin or \"-\" is used"); + } + let mut builder = Markkit::builder().max_input_size(max_size); for sel in &include_selectors { @@ -66,9 +79,16 @@ async fn main() -> Result<()> { let markkit = builder.build(); - let markdown = if file { - let bytes = tokio::fs::read(&source).await?; - let ext = std::path::Path::new(&source) + let source_label = source.clone().unwrap_or_else(|| "".to_string()); + + let markdown = if read_stdin { + let mut buf = String::new(); + std::io::stdin().read_to_string(&mut buf)?; + markkit.extract_html_sync(&buf)? + } else if file { + let src = source.as_deref().unwrap(); + let bytes = tokio::fs::read(src).await?; + let ext = std::path::Path::new(src) .extension() .and_then(|e| e.to_str()) .unwrap_or(""); @@ -82,12 +102,13 @@ async fn main() -> Result<()> { }; markkit.extract_bytes(&bytes, content_type).await? } else { - markkit.extract_url(&source).await? + let src = source.as_deref().unwrap(); + markkit.extract_url(src).await? }; if output_json { let output = serde_json::json!({ - "source": source, + "source": source_label, "markdown": markdown, }); println!("{}", serde_json::to_string_pretty(&output)?); diff --git a/crates/markkit-cli/tests/cli_integration.rs b/crates/markkit-cli/tests/cli_integration.rs new file mode 100644 index 0000000..2d50944 --- /dev/null +++ b/crates/markkit-cli/tests/cli_integration.rs @@ -0,0 +1,201 @@ +use std::io::Write; +use std::process::Command; + +fn markkit_bin() -> Command { + Command::new(env!("CARGO_BIN_EXE_markkit")) +} + +#[test] +fn help_flag_works() { + let output = markkit_bin() + .arg("--help") + .output() + .expect("failed to run markkit"); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Extract clean markdown")); +} + +#[test] +fn extract_help_works() { + let output = markkit_bin() + .args(["extract", "--help"]) + .output() + .expect("failed to run markkit"); + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("--stdin")); +} + +#[test] +fn extract_from_stdin_flag() { + let mut child = markkit_bin() + .args(["extract", "--stdin"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + + child + .stdin + .as_mut() + .unwrap() + .write_all(b"

Hello World

Some text here.

") + .unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Hello World"), "output: {stdout}"); + assert!(stdout.contains("Some text here"), "output: {stdout}"); +} + +#[test] +fn extract_from_stdin_dash() { + let mut child = markkit_bin() + .args(["extract", "-"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + + child + .stdin + .as_mut() + .unwrap() + .write_all(b"

Dash Test

") + .unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Dash Test"), "output: {stdout}"); +} + +#[test] +fn extract_from_file() { + let dir = tempfile::tempdir().unwrap(); + let file_path = dir.path().join("test.html"); + std::fs::write(&file_path, "

File Test

Paragraph.

").unwrap(); + + let output = markkit_bin() + .args(["extract", "--file", file_path.to_str().unwrap()]) + .output() + .expect("failed to run markkit"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("File Test"), "output: {stdout}"); + assert!(stdout.contains("Paragraph"), "output: {stdout}"); +} + +#[test] +fn extract_json_output() { + let mut child = markkit_bin() + .args(["extract", "--stdin", "--json"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + + child + .stdin + .as_mut() + .unwrap() + .write_all(b"

JSON Test

") + .unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output"); + assert_eq!(parsed["source"], ""); + assert!(parsed["markdown"].as_str().unwrap().contains("JSON Test")); +} + +#[test] +fn extract_with_include_selector() { + let mut child = markkit_bin() + .args(["extract", "--stdin", "-i", "article"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + + child + .stdin + .as_mut() + .unwrap() + .write_all(b"
Ignore this

Keep this

") + .unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Keep this"), "output: {stdout}"); +} + +#[test] +fn extract_with_exclude_selector() { + let mut child = markkit_bin() + .args(["extract", "--stdin", "-e", "nav"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("failed to spawn markkit"); + + child + .stdin + .as_mut() + .unwrap() + .write_all(b"

Content

") + .unwrap(); + + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Content"), "output: {stdout}"); + assert!( + !stdout.contains("Menu"), + "nav should be excluded, output: {stdout}" + ); +} + +#[test] +fn missing_source_errors() { + let output = markkit_bin() + .args(["extract"]) + .output() + .expect("failed to run markkit"); + + assert!(!output.status.success()); +} diff --git a/crates/markkit-js/src/lib.rs b/crates/markkit-js/src/lib.rs index 0f049ce..08f7926 100644 --- a/crates/markkit-js/src/lib.rs +++ b/crates/markkit-js/src/lib.rs @@ -1,6 +1,6 @@ use napi_derive::napi; -/// Extract markdown from an HTML string. +/// Extract markdown from an HTML string (async). #[napi] pub async fn extract_html(html: String) -> napi::Result { let m = markkit::Markkit::builder().build(); @@ -9,6 +9,14 @@ pub async fn extract_html(html: String) -> napi::Result { .map_err(|e| napi::Error::from_reason(e.to_string())) } +/// Extract markdown from an HTML string (synchronous). +#[napi] +pub fn extract_html_sync(html: String) -> napi::Result { + let m = markkit::Markkit::builder().build(); + m.extract_html_sync(&html) + .map_err(|e| napi::Error::from_reason(e.to_string())) +} + /// Extract markdown from a URL. #[napi] pub async fn extract_url(url: String) -> napi::Result { diff --git a/crates/markkit-python/src/lib.rs b/crates/markkit-python/src/lib.rs index 6320465..fab7641 100644 --- a/crates/markkit-python/src/lib.rs +++ b/crates/markkit-python/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; -/// Extract markdown from an HTML string. +/// Extract markdown from an HTML string (async). #[pyfunction] fn extract_html(py: Python<'_>, html: String) -> PyResult> { pyo3_async_runtimes::tokio::future_into_py(py, async move { @@ -11,6 +11,47 @@ fn extract_html(py: Python<'_>, html: String) -> PyResult> { }) } +/// Extract markdown from an HTML string (synchronous). +#[pyfunction] +fn extract_html_sync(html: String) -> PyResult { + let m = markkit::Markkit::builder().build(); + m.extract_html_sync(&html) + .map_err(|e| PyErr::new::(e.to_string())) +} + +/// Extract markdown from an HTML string with configuration options (synchronous). +/// +/// Keyword arguments: +/// include_selectors: list of CSS selectors to include +/// exclude_selectors: list of CSS selectors to exclude +/// max_input_size: maximum input size in bytes +#[pyfunction] +#[pyo3(signature = (html, *, include_selectors=None, exclude_selectors=None, max_input_size=None))] +fn extract_html_sync_with_options( + html: String, + include_selectors: Option>, + exclude_selectors: Option>, + max_input_size: Option, +) -> PyResult { + let mut builder = markkit::Markkit::builder(); + if let Some(size) = max_input_size { + builder = builder.max_input_size(size); + } + if let Some(selectors) = include_selectors { + for sel in &selectors { + builder = builder.include_selector(sel); + } + } + if let Some(selectors) = exclude_selectors { + for sel in &selectors { + builder = builder.exclude_selector(sel); + } + } + let m = builder.build(); + m.extract_html_sync(&html) + .map_err(|e| PyErr::new::(e.to_string())) +} + /// Extract markdown from a URL. #[pyfunction] fn extract_url(py: Python<'_>, url: String) -> PyResult> { @@ -40,6 +81,8 @@ fn extract_bytes( #[pymodule] fn markkit_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(extract_html, m)?)?; + m.add_function(wrap_pyfunction!(extract_html_sync, m)?)?; + m.add_function(wrap_pyfunction!(extract_html_sync_with_options, m)?)?; m.add_function(wrap_pyfunction!(extract_url, m)?)?; m.add_function(wrap_pyfunction!(extract_bytes, m)?)?; Ok(()) diff --git a/crates/markkit/Cargo.toml b/crates/markkit/Cargo.toml index 09afd79..d354a1d 100644 --- a/crates/markkit/Cargo.toml +++ b/crates/markkit/Cargo.toml @@ -74,6 +74,22 @@ tempfile.workspace = true proptest.workspace = true criterion.workspace = true +[[example]] +name = "basic" +path = "../../examples/basic.rs" + +[[example]] +name = "custom_selectors" +path = "../../examples/custom_selectors.rs" + +[[example]] +name = "ir_access" +path = "../../examples/ir_access.rs" + +[[example]] +name = "configuration" +path = "../../examples/configuration.rs" + [[bench]] name = "extraction" harness = false diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index c8be920..a7ceed3 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -21,6 +21,11 @@ impl Markkit { /// Extract markdown from an HTML string. pub async fn extract_html(&self, html: &str) -> crate::Result { + self.extract_html_sync(html) + } + + /// Synchronous HTML extraction (no async needed for parsing). + pub fn extract_html_sync(&self, html: &str) -> crate::Result { if html.len() > self.config.max_input_size { return Err(Error::ResourceLimit(format!( "Input size {} exceeds limit {}", @@ -29,8 +34,31 @@ impl Markkit { ))); } - // TODO: Implement HTML-to-markdown conversion - Ok(String::new()) + #[cfg(feature = "html")] + { + let parser = crate::html::HtmlParser::new(&self.config); + let doc = parser.parse(html)?; + let mut renderer = crate::render::MarkdownRenderer::new(); + Ok(renderer.render(&doc)) + } + #[cfg(not(feature = "html"))] + Err(Error::UnsupportedFormat( + "HTML support requires the 'html' feature".to_string(), + )) + } + + /// Parse HTML into the IR without rendering. + #[cfg(feature = "html")] + pub fn parse_html(&self, html: &str) -> crate::Result { + if html.len() > self.config.max_input_size { + return Err(Error::ResourceLimit(format!( + "Input size {} exceeds limit {}", + html.len(), + self.config.max_input_size + ))); + } + let parser = crate::html::HtmlParser::new(&self.config); + parser.parse(html) } /// Extract markdown from raw bytes, auto-detecting the format. diff --git a/crates/markkit/src/html/mod.rs b/crates/markkit/src/html/mod.rs new file mode 100644 index 0000000..4809ce9 --- /dev/null +++ b/crates/markkit/src/html/mod.rs @@ -0,0 +1,7 @@ +//! HTML parsing module. +//! +//! Converts HTML input into the canonical IR. + +mod parser; + +pub use parser::HtmlParser; diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs new file mode 100644 index 0000000..6341d29 --- /dev/null +++ b/crates/markkit/src/html/parser.rs @@ -0,0 +1,517 @@ +//! HTML → IR parser using scraper/html5ever. + +use scraper::{Html, Node as HtmlNode, Selector}; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::ir::{Document, ListItem, Node}; + +/// Maximum recursion depth for nested elements. +const MAX_DEPTH: usize = 200; + +/// Parses HTML into the canonical IR. +pub struct HtmlParser<'a> { + config: &'a MarkkitBuilder, +} + +impl<'a> HtmlParser<'a> { + pub fn new(config: &'a MarkkitBuilder) -> Self { + Self { config } + } + + /// Parse an HTML string into a [`Document`]. + pub fn parse(&self, html: &str) -> Result { + let dom = Html::parse_document(html); + let root = dom.root_element(); + + let content_nodes: Vec> = + if !self.config.include_selectors.is_empty() { + let sels: Vec = self + .config + .include_selectors + .iter() + .filter_map(|s| Selector::parse(s).ok()) + .collect(); + sels.iter().flat_map(|sel| dom.select(sel)).collect() + } else { + let body_sel = Selector::parse("body").unwrap(); + let body: Vec<_> = dom.select(&body_sel).collect(); + if body.is_empty() { vec![root] } else { body } + }; + + let exclude_selectors: Vec = self + .config + .exclude_selectors + .iter() + .filter_map(|s| Selector::parse(s).ok()) + .collect(); + + let mut nodes = Vec::new(); + for content_root in content_nodes { + self.convert_children(content_root, &exclude_selectors, &mut nodes, 0); + } + + Ok(Document::new(nodes)) + } + + fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool { + exclude.iter().any(|sel| sel.matches(el)) + } + + fn convert_children( + &self, + parent: scraper::ElementRef<'_>, + exclude: &[Selector], + out: &mut Vec, + depth: usize, + ) { + if depth >= MAX_DEPTH { + // At max depth, just extract text + let text: String = parent.text().collect(); + let trimmed = text.trim(); + if !trimmed.is_empty() { + out.push(Node::Text(trimmed.to_string())); + } + return; + } + + for child in parent.children() { + match child.value() { + HtmlNode::Text(text) => { + let t = text.to_string(); + let collapsed = collapse_whitespace(&t); + if !collapsed.is_empty() { + out.push(Node::Text(collapsed)); + } else if t.contains(char::is_whitespace) && !out.is_empty() { + out.push(Node::Text(" ".into())); + } + } + HtmlNode::Element(_) => { + if let Some(el) = scraper::ElementRef::wrap(child) { + if self.is_excluded(&el, exclude) { + continue; + } + // Transparent containers: recurse directly into their children + if is_transparent_container(el.value().name()) { + self.convert_children(el, exclude, out, depth + 1); + } else if let Some(node) = self.convert_element(el, exclude, depth + 1) { + out.push(node); + } + } + } + _ => {} + } + } + } + + fn convert_element( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + if depth >= MAX_DEPTH { + let text: String = el.text().collect(); + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + return Some(Node::Text(trimmed.to_string())); + } + + let tag = el.value().name(); + + match tag { + "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => { + let level = tag[1..].parse::().unwrap_or(1); + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Heading { level, children }) + } + + "p" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Paragraph { children }) + } + + "div" | "section" | "article" | "main" | "span" | "figure" | "figcaption" => { + let mut nodes = Vec::new(); + self.convert_children(el, exclude, &mut nodes, depth); + if nodes.len() == 1 { + Some(nodes.remove(0)) + } else if nodes.is_empty() { + None + } else if nodes.iter().all(is_inline) { + Some(Node::Paragraph { children: nodes }) + } else { + None + } + } + + "ul" => { + let items = self.collect_list_items(el, exclude, depth); + if items.is_empty() { + return None; + } + Some(Node::List { + ordered: false, + start: None, + items, + }) + } + + "ol" => { + let start = el.value().attr("start").and_then(|s| s.parse::().ok()); + let items = self.collect_list_items(el, exclude, depth); + if items.is_empty() { + return None; + } + Some(Node::List { + ordered: true, + start, + items, + }) + } + + "blockquote" => { + let mut children = Vec::new(); + self.convert_children(el, exclude, &mut children, depth); + if children.is_empty() { + return None; + } + Some(Node::BlockQuote { children }) + } + + "pre" => { + let code_sel = Selector::parse("code").unwrap(); + if let Some(code_el) = el.select(&code_sel).next() { + let language = code_el + .value() + .attr("class") + .and_then(extract_language_from_class); + let code = code_el.text().collect::(); + Some(Node::CodeBlock { language, code }) + } else { + let code = el.text().collect::(); + Some(Node::CodeBlock { + language: None, + code, + }) + } + } + + "code" => { + let text = el.text().collect::(); + Some(Node::InlineCode(text)) + } + + "table" => self.convert_table(el, exclude, depth), + + "strong" | "b" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Bold { children }) + } + + "em" | "i" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Italic { children }) + } + + "del" | "s" | "strike" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Strikethrough { children }) + } + + "a" => { + if !self.config.preserve_links { + let children = self.collect_inline(el, exclude, depth); + return if children.is_empty() { + None + } else if children.len() == 1 { + Some(children.into_iter().next().unwrap()) + } else { + Some(Node::Paragraph { children }) + }; + } + let url = el.value().attr("href").unwrap_or("").to_string(); + let title = el.value().attr("title").map(|s| s.to_string()); + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Link { + url, + title, + children, + }) + } + + "img" => { + if !self.config.preserve_images { + return None; + } + let src = el.value().attr("src").unwrap_or("").to_string(); + let alt = el.value().attr("alt").unwrap_or("").to_string(); + Some(Node::Image { src, alt }) + } + + "br" => Some(Node::LineBreak), + + "hr" => Some(Node::ThematicBreak), + + // Skip non-content elements + "script" | "style" | "noscript" | "iframe" | "object" | "embed" | "form" | "input" + | "button" | "select" | "textarea" | "svg" | "canvas" | "video" | "audio" + | "source" | "track" | "map" | "area" => None, + + // Unknown element: try to extract inline content + _ => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + None + } else if children.len() == 1 { + Some(children.into_iter().next().unwrap()) + } else { + Some(Node::Paragraph { children }) + } + } + } + } + + fn collect_inline( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Vec { + let mut nodes = Vec::new(); + if depth >= MAX_DEPTH { + let text: String = el.text().collect(); + let trimmed = text.trim(); + if !trimmed.is_empty() { + nodes.push(Node::Text(trimmed.to_string())); + } + return nodes; + } + + for child in el.children() { + match child.value() { + HtmlNode::Text(text) => { + let t = text.to_string(); + let collapsed = collapse_whitespace(&t); + if !collapsed.is_empty() { + nodes.push(Node::Text(collapsed)); + } else if t.contains(char::is_whitespace) && !nodes.is_empty() { + nodes.push(Node::Text(" ".into())); + } + } + HtmlNode::Element(_) => { + if let Some(child_el) = scraper::ElementRef::wrap(child) { + if self.is_excluded(&child_el, exclude) { + continue; + } + if let Some(node) = self.convert_element(child_el, exclude, depth + 1) { + nodes.push(node); + } + } + } + _ => {} + } + } + nodes + } + + fn collect_list_items( + &self, + list_el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Vec { + let mut items = Vec::new(); + for child in list_el.children() { + if let Some(el) = scraper::ElementRef::wrap(child) + && el.value().name() == "li" + { + let mut children = Vec::new(); + self.convert_children(el, exclude, &mut children, depth + 1); + if !children.is_empty() { + items.push(ListItem::new(children)); + } + } + } + items + } + + fn convert_table( + &self, + table_el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + let thead_sel = Selector::parse("thead").unwrap(); + let tbody_sel = Selector::parse("tbody").unwrap(); + let tr_sel = Selector::parse("tr").unwrap(); + let th_sel = Selector::parse("th").unwrap(); + let td_sel = Selector::parse("td").unwrap(); + + let mut headers: Vec> = Vec::new(); + let mut rows: Vec>> = Vec::new(); + + if let Some(thead) = table_el.select(&thead_sel).next() + && let Some(tr) = thead.select(&tr_sel).next() + { + for th in tr.select(&th_sel) { + headers.push(self.collect_inline(th, exclude, depth + 1)); + } + } + + let body_rows: Vec> = + if let Some(tbody) = table_el.select(&tbody_sel).next() { + tbody.select(&tr_sel).collect() + } else { + table_el.select(&tr_sel).collect() + }; + + for tr in body_rows { + if headers.is_empty() { + let ths: Vec<_> = tr.select(&th_sel).collect(); + if !ths.is_empty() { + for th in ths { + headers.push(self.collect_inline(th, exclude, depth + 1)); + } + continue; + } + } + + let cells: Vec> = tr + .select(&td_sel) + .map(|td| self.collect_inline(td, exclude, depth + 1)) + .collect(); + + if !cells.is_empty() { + rows.push(cells); + } + } + + if headers.is_empty() && rows.is_empty() { + return None; + } + + Some(Node::Table { headers, rows }) + } +} + +/// Extract language identifier from a class like "language-rust" or "lang-python". +fn extract_language_from_class(class: &str) -> Option { + for cls in class.split_whitespace() { + if let Some(lang) = cls.strip_prefix("language-") { + return Some(lang.to_string()); + } + if let Some(lang) = cls.strip_prefix("lang-") { + return Some(lang.to_string()); + } + } + None +} + +/// Collapse runs of whitespace into single spaces. +/// Preserves a leading/trailing space if the original had one (important for inline context). +fn collapse_whitespace(s: &str) -> String { + let has_leading_space = s.starts_with(char::is_whitespace); + let has_trailing_space = s.ends_with(char::is_whitespace); + + let mut result = String::with_capacity(s.len()); + let mut prev_space = true; // suppress leading space initially + for ch in s.chars() { + if ch.is_whitespace() { + if !prev_space { + result.push(' '); + } + prev_space = true; + } else { + prev_space = false; + result.push(ch); + } + } + if result.ends_with(' ') { + result.pop(); + } + + if has_leading_space && !result.is_empty() { + result.insert(0, ' '); + } + if has_trailing_space && !result.is_empty() { + result.push(' '); + } + + result +} + +/// Check if a tag is a transparent container (should be unwrapped in block context). +fn is_transparent_container(tag: &str) -> bool { + matches!( + tag, + "div" | "section" | "article" | "main" | "figure" | "figcaption" + ) +} + +/// Check if a node is inline content. +fn is_inline(node: &Node) -> bool { + matches!( + node, + Node::Text(_) + | Node::Bold { .. } + | Node::Italic { .. } + | Node::Strikethrough { .. } + | Node::InlineCode(_) + | Node::Link { .. } + | Node::Image { .. } + | Node::LineBreak + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collapse_whitespace() { + assert_eq!(collapse_whitespace(" hello world "), " hello world "); + assert_eq!(collapse_whitespace("a"), "a"); + assert_eq!(collapse_whitespace(" "), ""); + assert_eq!(collapse_whitespace("a\n b"), "a b"); + assert_eq!(collapse_whitespace(" bold "), " bold "); + assert_eq!(collapse_whitespace("no-spaces"), "no-spaces"); + } + + #[test] + fn test_extract_language_from_class() { + assert_eq!( + extract_language_from_class("language-rust"), + Some("rust".into()) + ); + assert_eq!( + extract_language_from_class("lang-python"), + Some("python".into()) + ); + assert_eq!( + extract_language_from_class("highlight language-js extra"), + Some("js".into()) + ); + assert_eq!(extract_language_from_class("highlight"), None); + } +} diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs new file mode 100644 index 0000000..5fefd68 --- /dev/null +++ b/crates/markkit/src/ir.rs @@ -0,0 +1,116 @@ +//! Canonical intermediate representation for documents. +//! +//! All parsers convert their input into this IR, and all renderers +//! consume it. This decouples input formats from output formats. + +use serde::{Deserialize, Serialize}; + +/// A parsed document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Document { + pub nodes: Vec, +} + +impl Document { + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } +} + +/// A block-level or inline node in the document tree. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Node { + /// Heading with level 1–6. + Heading { level: u8, children: Vec }, + /// Paragraph containing inline content. + Paragraph { children: Vec }, + /// Ordered or unordered list. + List { + ordered: bool, + start: Option, + items: Vec, + }, + /// Block quotation. + BlockQuote { children: Vec }, + /// Fenced code block. + CodeBlock { + language: Option, + code: String, + }, + /// Table with optional header row. + Table { + headers: Vec>, + rows: Vec>>, + }, + /// Horizontal rule / thematic break. + ThematicBreak, + /// Plain text. + Text(String), + /// Bold / strong emphasis. + Bold { children: Vec }, + /// Italic / emphasis. + Italic { children: Vec }, + /// Strikethrough. + Strikethrough { children: Vec }, + /// Inline code. + InlineCode(String), + /// Hyperlink. + Link { + url: String, + title: Option, + children: Vec, + }, + /// Image. + Image { src: String, alt: String }, + /// Hard line break. + LineBreak, +} + +/// A single item in a list. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ListItem { + pub children: Vec, +} + +impl ListItem { + pub fn new(children: Vec) -> Self { + Self { children } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn document_is_empty() { + assert!(Document::new(vec![]).is_empty()); + assert!(!Document::new(vec![Node::ThematicBreak]).is_empty()); + } + + #[test] + fn ir_roundtrip_json() { + let doc = Document::new(vec![ + Node::Heading { + level: 1, + children: vec![Node::Text("Hello".into())], + }, + Node::Paragraph { + children: vec![ + Node::Text("Some ".into()), + Node::Bold { + children: vec![Node::Text("bold".into())], + }, + Node::Text(" text".into()), + ], + }, + ]); + let json = serde_json::to_string(&doc).unwrap(); + let parsed: Document = serde_json::from_str(&json).unwrap(); + assert_eq!(doc, parsed); + } +} diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 36e58a4..738a063 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -35,10 +35,16 @@ pub mod config; pub mod error; pub mod extract; pub mod format; +#[cfg(feature = "html")] +pub mod html; +pub mod ir; +pub mod render; pub use config::MarkkitBuilder; pub use error::Error; pub use extract::Markkit; +pub use ir::Document; +pub use render::MarkdownRenderer; /// Result type alias for markkit operations. pub type Result = std::result::Result; diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs new file mode 100644 index 0000000..0534f57 --- /dev/null +++ b/crates/markkit/src/render/markdown.rs @@ -0,0 +1,382 @@ +//! IR → Markdown renderer. + +use crate::ir::{Document, ListItem, Node}; + +/// Renders a [`Document`] IR into Markdown text. +pub struct MarkdownRenderer { + /// Output buffer. + buf: String, +} + +impl MarkdownRenderer { + pub fn new() -> Self { + Self { + buf: String::with_capacity(4096), + } + } + + /// Render a document to a Markdown string. + pub fn render(&mut self, doc: &Document) -> String { + self.buf.clear(); + self.render_nodes(&doc.nodes, 0); + normalize_output(&self.buf) + } + + /// Render a slice of nodes. + fn render_nodes(&mut self, nodes: &[Node], depth: usize) { + for (i, node) in nodes.iter().enumerate() { + if i > 0 && is_block(node) && is_block(&nodes[i - 1]) { + self.buf.push('\n'); + } + self.render_node(node, depth); + } + } + + fn render_node(&mut self, node: &Node, depth: usize) { + match node { + Node::Heading { level, children } => { + let prefix = "#".repeat(*level as usize); + self.buf.push_str(&prefix); + self.buf.push(' '); + let start = self.buf.len(); + self.render_inline(children); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + } + + Node::Paragraph { children } => { + let start = self.buf.len(); + self.render_inline(children); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + } + + Node::List { + ordered, + start, + items, + } => { + self.render_list(*ordered, *start, items, depth); + } + + Node::BlockQuote { children } => { + let inner = { + let mut sub = MarkdownRenderer::new(); + sub.render_nodes(children, depth); + normalize_output(&sub.buf) + }; + for line in inner.lines() { + if line.is_empty() { + self.buf.push_str(">\n"); + } else { + self.buf.push_str("> "); + self.buf.push_str(line); + self.buf.push('\n'); + } + } + } + + Node::CodeBlock { language, code } => { + self.buf.push_str("```"); + if let Some(lang) = language { + self.buf.push_str(lang); + } + self.buf.push('\n'); + self.buf.push_str(code); + if !code.ends_with('\n') { + self.buf.push('\n'); + } + self.buf.push_str("```\n"); + } + + Node::Table { headers, rows } => { + self.render_table(headers, rows); + } + + Node::ThematicBreak => { + self.buf.push_str("---\n"); + } + + // Inline nodes rendered at block level get wrapped in a line + Node::Text(_) + | Node::Bold { .. } + | Node::Italic { .. } + | Node::Strikethrough { .. } + | Node::InlineCode(_) + | Node::Link { .. } + | Node::Image { .. } + | Node::LineBreak => { + self.render_inline_node(node); + } + } + } + + /// Render a list. + fn render_list(&mut self, ordered: bool, start: Option, items: &[ListItem], depth: usize) { + let indent = " ".repeat(depth); + let start_num = start.unwrap_or(1); + + for (i, item) in items.iter().enumerate() { + let marker = if ordered { + format!("{}{}. ", indent, start_num + i as u64) + } else { + format!("{}- ", indent) + }; + self.buf.push_str(&marker); + + // Render item children + let has_block = item.children.iter().any(|n| !is_inline_node(n)); + if has_block { + // Multi-line item: render sub-blocks indented + let mut first = true; + for child in &item.children { + if first { + first = false; + match child { + Node::Paragraph { children } => { + self.render_inline(children); + self.buf.push('\n'); + } + Node::List { + ordered: sub_ord, + start: sub_start, + items: sub_items, + } => { + self.buf.push('\n'); + self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); + } + _ => { + let mut sub = MarkdownRenderer::new(); + sub.render_node(child, depth + 1); + let text = sub.buf.trim_end().to_string(); + self.buf.push_str(&text); + self.buf.push('\n'); + } + } + } else { + match child { + Node::List { + ordered: sub_ord, + start: sub_start, + items: sub_items, + } => { + self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); + } + _ => { + let mut sub = MarkdownRenderer::new(); + sub.render_node(child, depth + 1); + let inner = sub.buf.trim_end().to_string(); + let sub_indent = " ".repeat(depth + 1); + for line in inner.lines() { + self.buf.push_str(&sub_indent); + self.buf.push_str(line); + self.buf.push('\n'); + } + } + } + } + } + } else { + // Simple inline item + self.render_inline(&item.children); + self.buf.push('\n'); + } + } + } + + /// Render inline content. + fn render_inline(&mut self, nodes: &[Node]) { + for node in nodes { + self.render_inline_node(node); + } + } + + /// Render a single inline node. + fn render_inline_node(&mut self, node: &Node) { + match node { + Node::Text(t) => self.buf.push_str(t), + Node::Bold { children } => { + self.buf.push_str("**"); + self.render_inline(children); + self.buf.push_str("**"); + } + Node::Italic { children } => { + self.buf.push('*'); + self.render_inline(children); + self.buf.push('*'); + } + Node::Strikethrough { children } => { + self.buf.push_str("~~"); + self.render_inline(children); + self.buf.push_str("~~"); + } + Node::InlineCode(code) => { + self.buf.push('`'); + self.buf.push_str(code); + self.buf.push('`'); + } + Node::Link { + url, + title, + children, + } => { + self.buf.push('['); + self.render_inline(children); + self.buf.push_str("]("); + self.buf.push_str(url); + if let Some(t) = title { + self.buf.push_str(" \""); + self.buf.push_str(t); + self.buf.push('"'); + } + self.buf.push(')'); + } + Node::Image { src, alt } => { + self.buf.push_str("!["); + self.buf.push_str(alt); + self.buf.push_str("]("); + self.buf.push_str(src); + self.buf.push(')'); + } + Node::LineBreak => { + self.buf.push_str(" \n"); + } + // Block nodes shouldn't appear here but handle gracefully + other => self.render_node(other, 0), + } + } + + /// Render a GFM table. + fn render_table(&mut self, headers: &[Vec], rows: &[Vec>]) { + // Determine column count + let col_count = headers + .len() + .max(rows.iter().map(|r| r.len()).max().unwrap_or(0)); + + if col_count == 0 { + return; + } + + // Render header row + self.buf.push('|'); + for i in 0..col_count { + self.buf.push(' '); + if let Some(cells) = headers.get(i) { + self.render_inline(cells); + } + self.buf.push_str(" |"); + } + self.buf.push('\n'); + + // Separator + self.buf.push('|'); + for _ in 0..col_count { + self.buf.push_str(" --- |"); + } + self.buf.push('\n'); + + // Data rows + for row in rows { + self.buf.push('|'); + for i in 0..col_count { + self.buf.push(' '); + if let Some(cells) = row.get(i) { + self.render_inline(cells); + } + self.buf.push_str(" |"); + } + self.buf.push('\n'); + } + } +} + +impl Default for MarkdownRenderer { + fn default() -> Self { + Self::new() + } +} + +/// Trim leading and trailing whitespace from inline content rendered since `start`. +fn trim_trailing_inline(buf: &mut String, start: usize) { + // Trim trailing spaces + while buf.len() > start && buf.ends_with(' ') { + buf.pop(); + } + // Trim leading spaces after start + let content = &buf[start..]; + let trimmed_start = content.len() - content.trim_start().len(); + if trimmed_start > 0 { + buf.drain(start..start + trimmed_start); + } +} + +/// Check if a node is block-level. +fn is_block(node: &Node) -> bool { + matches!( + node, + Node::Heading { .. } + | Node::Paragraph { .. } + | Node::List { .. } + | Node::BlockQuote { .. } + | Node::CodeBlock { .. } + | Node::Table { .. } + | Node::ThematicBreak + ) +} + +fn is_inline_node(node: &Node) -> bool { + !is_block(node) +} + +/// Normalize output: collapse multiple blank lines, preserve hard breaks (two trailing spaces). +fn normalize_output(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut blank_count = 0; + + for line in s.lines() { + // Preserve markdown hard break (line ending in two+ spaces with content) + let core = line.trim_end(); + let is_hard_break = line.ends_with(" ") && !core.is_empty(); + let trimmed = if is_hard_break { + core.to_string() + " " + } else { + core.to_string() + }; + + if trimmed.is_empty() || trimmed == " " { + blank_count += 1; + if blank_count <= 1 { + result.push('\n'); + } + } else { + blank_count = 0; + result.push_str(&trimmed); + result.push('\n'); + } + } + + // Trim leading/trailing blank lines + let trimmed = result.trim_matches('\n'); + if trimmed.is_empty() { + String::new() + } else { + let mut out = trimmed.to_string(); + out.push('\n'); + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_output() { + assert_eq!(normalize_output("a\n\n\n\nb\n"), "a\n\nb\n"); + assert_eq!(normalize_output("\n\na\n\n"), "a\n"); + assert_eq!(normalize_output(""), ""); + assert_eq!(normalize_output(" \n \n"), ""); + // Hard break preserved + assert_eq!(normalize_output("a \nb\n"), "a \nb\n"); + } +} diff --git a/crates/markkit/src/render/mod.rs b/crates/markkit/src/render/mod.rs new file mode 100644 index 0000000..b91fc60 --- /dev/null +++ b/crates/markkit/src/render/mod.rs @@ -0,0 +1,7 @@ +//! Rendering module. +//! +//! Converts the canonical IR into output formats. + +mod markdown; + +pub use markdown::MarkdownRenderer; diff --git a/crates/markkit/tests/fixtures/html/basic_headings.html b/crates/markkit/tests/fixtures/html/basic_headings.html new file mode 100644 index 0000000..deb727b --- /dev/null +++ b/crates/markkit/tests/fixtures/html/basic_headings.html @@ -0,0 +1,6 @@ +

Heading Level 1

+

Heading Level 2

+

Heading Level 3

+

Heading Level 4

+
Heading Level 5
+
Heading Level 6
diff --git a/crates/markkit/tests/fixtures/html/basic_headings.md b/crates/markkit/tests/fixtures/html/basic_headings.md new file mode 100644 index 0000000..d4fff1b --- /dev/null +++ b/crates/markkit/tests/fixtures/html/basic_headings.md @@ -0,0 +1,6 @@ +# Heading Level 1 + ## Heading Level 2 + ### Heading Level 3 + #### Heading Level 4 + ##### Heading Level 5 + ###### Heading Level 6 diff --git a/crates/markkit/tests/fixtures/html/blockquotes.html b/crates/markkit/tests/fixtures/html/blockquotes.html new file mode 100644 index 0000000..0788bb8 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/blockquotes.html @@ -0,0 +1,8 @@ +

A simple blockquote.

+
+

Outer quote

+
+

Nested quote

+
+
+

Quote with bold and italic text.

diff --git a/crates/markkit/tests/fixtures/html/blockquotes.md b/crates/markkit/tests/fixtures/html/blockquotes.md new file mode 100644 index 0000000..a74c40e --- /dev/null +++ b/crates/markkit/tests/fixtures/html/blockquotes.md @@ -0,0 +1,4 @@ +> A simple blockquote. + > Outer quote +> > Nested quote + > Quote with **bold** and *italic* text. diff --git a/crates/markkit/tests/fixtures/html/code_blocks.html b/crates/markkit/tests/fixtures/html/code_blocks.html new file mode 100644 index 0000000..d509430 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/code_blocks.html @@ -0,0 +1,8 @@ +
plain code block
+with multiple lines
+
fn main() {
+    println!("Hello, world!");
+}
+
def hello():
+    print("hi")
+
preformatted without code tag
diff --git a/crates/markkit/tests/fixtures/html/code_blocks.md b/crates/markkit/tests/fixtures/html/code_blocks.md new file mode 100644 index 0000000..b1b2278 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/code_blocks.md @@ -0,0 +1,16 @@ +``` +plain code block +with multiple lines +``` + ```rust +fn main() { + println!("Hello, world!"); +} +``` + ```python +def hello(): + print("hi") +``` + ``` +preformatted without code tag +``` diff --git a/crates/markkit/tests/fixtures/html/full_article.html b/crates/markkit/tests/fixtures/html/full_article.html new file mode 100644 index 0000000..c2bd743 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/full_article.html @@ -0,0 +1,29 @@ + + +Test Article + + +

Article Title

+

This is the first paragraph with some links and formatting.

+

Section One

+

Section one content with inline code.

+
    +
  • Point one
    • +
  • Point two
    • +
+

Section Two

+

An important quote.

+ 
console.log("hello");
+ + + +
Col ACol B
12
+

Final paragraph.

+
+

Copyright 2024. All rights reserved.

+
+ + diff --git a/crates/markkit/tests/fixtures/html/full_article.md b/crates/markkit/tests/fixtures/html/full_article.md new file mode 100644 index 0000000..8c6b1fb --- /dev/null +++ b/crates/markkit/tests/fixtures/html/full_article.md @@ -0,0 +1,15 @@ +# Article Title + This is the **first paragraph** with some [links](/link) and *formatting*. + ## Section One + Section one content with `inline code`. + - Point one +- Point two + ## Section Two + > An important quote. + ```javascript +console.log("hello"); +``` + | Col A | Col B | +| --- | --- | +| 1 | 2 | + Final paragraph. diff --git a/crates/markkit/tests/fixtures/html/inline_formatting.html b/crates/markkit/tests/fixtures/html/inline_formatting.html new file mode 100644 index 0000000..132f097 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/inline_formatting.html @@ -0,0 +1,6 @@ +

This is bold text in a sentence.

+

This is italic text in a sentence.

+

This is strikethrough text in a sentence.

+

This is inline code in a sentence.

+

This has bold and italic together.

+

This has bold, italic, and strikethrough.

diff --git a/crates/markkit/tests/fixtures/html/inline_formatting.md b/crates/markkit/tests/fixtures/html/inline_formatting.md new file mode 100644 index 0000000..12e006d --- /dev/null +++ b/crates/markkit/tests/fixtures/html/inline_formatting.md @@ -0,0 +1,6 @@ +This is **bold text** in a sentence. + This is *italic text* in a sentence. + This is ~~strikethrough text~~ in a sentence. + This is `inline code` in a sentence. + This has ***bold and italic*** together. + This has **bold**, *italic*, and ~~strikethrough~~. diff --git a/crates/markkit/tests/fixtures/html/links_and_images.html b/crates/markkit/tests/fixtures/html/links_and_images.html new file mode 100644 index 0000000..9c5eed9 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/links_and_images.html @@ -0,0 +1,7 @@ +

Basic Link

+

Link with Title

+

Relative Link

+

Bold Link Text

+

An image

+

+

diff --git a/crates/markkit/tests/fixtures/html/links_and_images.md b/crates/markkit/tests/fixtures/html/links_and_images.md new file mode 100644 index 0000000..3f01ed1 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/links_and_images.md @@ -0,0 +1,7 @@ +[Basic Link](https://example.com) + [Link with Title](https://example.com "Example Site") + [Relative Link](/relative) + [**Bold Link Text**](/bold-link) + ![An image](image.jpg) + ![](photo.png) + ![](noalt.gif) diff --git a/crates/markkit/tests/fixtures/html/lists.html b/crates/markkit/tests/fixtures/html/lists.html new file mode 100644 index 0000000..d3cb0b6 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/lists.html @@ -0,0 +1,23 @@ +
    +
  • Apple
    • +
  • Banana
    • +
  • Cherry
    • +
+
    +
  1. First
    2. +
  2. Second
    3. +
  3. Third
    4. +
+
    +
  • Outer A +
      +
    • Inner A1
      • +
    • Inner A2
      • +
    +
    • +
  • Outer B
    • +
+
    +
  • Bold item with text
    • +
  • Italic item
    • +
diff --git a/crates/markkit/tests/fixtures/html/lists.md b/crates/markkit/tests/fixtures/html/lists.md new file mode 100644 index 0000000..72159b2 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/lists.md @@ -0,0 +1,12 @@ +- Apple +- Banana +- Cherry + 1. First +2. Second +3. Third + - Outer A + - Inner A1 + - Inner A2 +- Outer B + - **Bold item** with text +- *Italic item* diff --git a/crates/markkit/tests/fixtures/html/malformed.html b/crates/markkit/tests/fixtures/html/malformed.html new file mode 100644 index 0000000..cb4b496 --- /dev/null +++ b/crates/markkit/tests/fixtures/html/malformed.html @@ -0,0 +1,5 @@ +

Unclosed paragraph +

Next paragraph

+

Mismatched bold with no close

+

Entities: & < > " '

+

Nested italic and bold crossed

diff --git a/crates/markkit/tests/fixtures/html/malformed.md b/crates/markkit/tests/fixtures/html/malformed.md new file mode 100644 index 0000000..466deac --- /dev/null +++ b/crates/markkit/tests/fixtures/html/malformed.md @@ -0,0 +1,7 @@ +Unclosed paragraph + +Next paragraph + Mismatched **bold with no close** +**Entities: & < > " ' + Nested *italic **and bold***** crossed** + ** diff --git a/crates/markkit/tests/fixtures/html/tables.html b/crates/markkit/tests/fixtures/html/tables.html new file mode 100644 index 0000000..bf1b55a --- /dev/null +++ b/crates/markkit/tests/fixtures/html/tables.html @@ -0,0 +1,25 @@ + + + + + + + + +
NameAgeCity
Alice30NYC
Bob25LA
+ + + + + + + +
FeatureStatus
Bold FeatureActive
+ + + +
Col1Col2
ab
+ + + +
NoHeader
RowData
diff --git a/crates/markkit/tests/fixtures/html/tables.md b/crates/markkit/tests/fixtures/html/tables.md new file mode 100644 index 0000000..3aab99a --- /dev/null +++ b/crates/markkit/tests/fixtures/html/tables.md @@ -0,0 +1,14 @@ +| Name | Age | City | +| --- | --- | --- | +| Alice | 30 | NYC | +| Bob | 25 | LA | + | Feature | Status | +| --- | --- | +| **Bold Feature** | *Active* | + | Col1 | Col2 | +| --- | --- | +| a | b | + | | | +| --- | --- | +| No | Header | +| Row | Data | diff --git a/crates/markkit/tests/html_fixtures.rs b/crates/markkit/tests/html_fixtures.rs new file mode 100644 index 0000000..3d87016 --- /dev/null +++ b/crates/markkit/tests/html_fixtures.rs @@ -0,0 +1,64 @@ +//! Fixture-based tests for HTML → Markdown conversion. +//! +//! Each fixture is a pair of files: `{name}.html` (input) and `{name}.md` (expected output). + +use std::fs; + +use markkit::Markkit; + +fn fixture_test(name: &str) { + let html = fs::read_to_string(format!("tests/fixtures/html/{}.html", name)).unwrap(); + let expected = fs::read_to_string(format!("tests/fixtures/html/{}.md", name)).unwrap(); + let m = Markkit::builder().build(); + let actual = m.extract_html_sync(&html).unwrap(); + assert_eq!( + actual, expected, + "Fixture '{}' mismatch.\n\n--- Actual ---\n{}\n--- Expected ---\n{}", + name, actual, expected + ); +} + +#[test] +fn fixture_basic_headings() { + fixture_test("basic_headings"); +} + +#[test] +fn fixture_inline_formatting() { + fixture_test("inline_formatting"); +} + +#[test] +fn fixture_links_and_images() { + fixture_test("links_and_images"); +} + +#[test] +fn fixture_lists() { + fixture_test("lists"); +} + +#[test] +fn fixture_code_blocks() { + fixture_test("code_blocks"); +} + +#[test] +fn fixture_tables() { + fixture_test("tables"); +} + +#[test] +fn fixture_blockquotes() { + fixture_test("blockquotes"); +} + +#[test] +fn fixture_full_article() { + fixture_test("full_article"); +} + +#[test] +fn fixture_malformed() { + fixture_test("malformed"); +} diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs new file mode 100644 index 0000000..a3db669 --- /dev/null +++ b/crates/markkit/tests/html_integration.rs @@ -0,0 +1,693 @@ +//! Integration tests for HTML → Markdown conversion. + +use markkit::Markkit; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +fn convert_with(html: &str, configure: F) -> String +where + F: FnOnce(markkit::MarkkitBuilder) -> markkit::MarkkitBuilder, +{ + let m = configure(Markkit::builder()).build(); + m.extract_html_sync(html).unwrap() +} + +// ─── Headings ──────────────────────────────────────────────────────── + +#[test] +fn heading_h1() { + assert_eq!(convert("

Hello World

"), "# Hello World\n"); +} + +#[test] +fn heading_h2() { + assert_eq!(convert("

Sub Heading

"), "## Sub Heading\n"); +} + +#[test] +fn heading_h3_through_h6() { + assert_eq!(convert("

H3

"), "### H3\n"); + assert_eq!(convert("

H4

"), "#### H4\n"); + assert_eq!(convert("
H5
"), "##### H5\n"); + assert_eq!(convert("
H6
"), "###### H6\n"); +} + +#[test] +fn heading_with_inline_formatting() { + assert_eq!( + convert("

Hello World

"), + "# Hello *World*\n" + ); +} + +#[test] +fn heading_empty_is_skipped() { + assert_eq!(convert("

"), ""); +} + +// ─── Paragraphs ────────────────────────────────────────────────────── + +#[test] +fn paragraph_simple() { + assert_eq!(convert("

Hello world

"), "Hello world\n"); +} + +#[test] +fn paragraph_multiple() { + let md = convert("

First

Second

"); + assert_eq!(md, "First\n\nSecond\n"); +} + +#[test] +fn paragraph_with_inline_elements() { + let md = convert("

This is bold and italic

"); + assert_eq!(md, "This is **bold** and *italic*\n"); +} + +#[test] +fn paragraph_whitespace_collapse() { + let md = convert("

hello world

"); + assert_eq!(md, "hello world\n"); +} + +// ─── Inline Formatting ────────────────────────────────────────────── + +#[test] +fn bold_strong() { + assert_eq!(convert("

bold

"), "**bold**\n"); +} + +#[test] +fn bold_b_tag() { + assert_eq!(convert("

bold

"), "**bold**\n"); +} + +#[test] +fn italic_em() { + assert_eq!(convert("

italic

"), "*italic*\n"); +} + +#[test] +fn italic_i_tag() { + assert_eq!(convert("

italic

"), "*italic*\n"); +} + +#[test] +fn strikethrough_del() { + assert_eq!(convert("

removed

"), "~~removed~~\n"); +} + +#[test] +fn strikethrough_s() { + assert_eq!(convert("

removed

"), "~~removed~~\n"); +} + +#[test] +fn inline_code() { + assert_eq!(convert("

let x = 1

"), "`let x = 1`\n"); +} + +#[test] +fn nested_inline_bold_italic() { + assert_eq!( + convert("

bold italic

"), + "***bold italic***\n" + ); +} + +// ─── Links ─────────────────────────────────────────────────────────── + +#[test] +fn link_basic() { + assert_eq!( + convert(r#"

Example

"#), + "[Example](https://example.com)\n" + ); +} + +#[test] +fn link_with_title() { + assert_eq!( + convert(r#"

Example

"#), + "[Example](https://example.com \"A site\")\n" + ); +} + +#[test] +fn link_with_inline_formatting() { + assert_eq!( + convert(r#"

Bold Link

"#), + "[**Bold Link**](/url)\n" + ); +} + +#[test] +fn link_preservation_disabled() { + let md = convert_with(r#"

Example

"#, |b| { + b.preserve_links(false) + }); + assert_eq!(md, "Example\n"); +} + +// ─── Images ────────────────────────────────────────────────────────── + +#[test] +fn image_basic() { + assert_eq!( + convert(r#"A photo"#), + "![A photo](photo.jpg)\n" + ); +} + +#[test] +fn image_no_alt() { + assert_eq!(convert(r#""#), "![](photo.jpg)\n"); +} + +#[test] +fn image_preservation_disabled() { + let md = convert_with(r#"A photo"#, |b| { + b.preserve_images(false) + }); + assert_eq!(md, ""); +} + +// ─── Lists ─────────────────────────────────────────────────────────── + +#[test] +fn unordered_list() { + let md = convert("
  • One
  • Two
  • Three
"); + assert_eq!(md, "- One\n- Two\n- Three\n"); +} + +#[test] +fn ordered_list() { + let md = convert("
  1. First
  2. Second
"); + assert_eq!(md, "1. First\n2. Second\n"); +} + +#[test] +fn ordered_list_custom_start() { + let md = convert(r#"
  1. Five
  2. Six
"#); + assert_eq!(md, "5. Five\n6. Six\n"); +} + +#[test] +fn nested_list() { + let md = + convert("
  • Outer
    • Inner A
    • Inner B
  • Other
"); + assert!(md.contains("- Outer")); + assert!(md.contains(" - Inner A")); + assert!(md.contains(" - Inner B")); + assert!(md.contains("- Other")); +} + +#[test] +fn list_with_inline_formatting() { + let md = convert("
  • Bold item
"); + assert_eq!(md, "- **Bold** item\n"); +} + +// ─── Blockquotes ───────────────────────────────────────────────────── + +#[test] +fn blockquote_simple() { + let md = convert("

Quoted text

"); + assert_eq!(md, "> Quoted text\n"); +} + +#[test] +fn blockquote_nested() { + let md = convert("

Outer

Inner

"); + assert!(md.contains("> Outer")); + assert!(md.contains("> > Inner")); +} + +// ─── Code Blocks ───────────────────────────────────────────────────── + +#[test] +fn code_block_plain() { + let md = convert("
fn main() {}
"); + assert_eq!(md, "```\nfn main() {}\n```\n"); +} + +#[test] +fn code_block_with_language() { + let md = convert(r#"
fn main() {}
"#); + assert_eq!(md, "```rust\nfn main() {}\n```\n"); +} + +#[test] +fn code_block_lang_prefix() { + let md = convert(r#"
print("hi")
"#); + assert_eq!(md, "```python\nprint(\"hi\")\n```\n"); +} + +#[test] +fn code_block_pre_without_code() { + let md = convert("
preformatted text
"); + assert_eq!(md, "```\npreformatted text\n```\n"); +} + +// ─── Tables ────────────────────────────────────────────────────────── + +#[test] +fn table_basic() { + let html = r#" + + + + + + +
NameValue
A1
B2
"#; + let md = convert(html); + assert!(md.contains("| Name | Value |")); + assert!(md.contains("| --- | --- |")); + assert!(md.contains("| A | 1 |")); + assert!(md.contains("| B | 2 |")); +} + +#[test] +fn table_th_in_body() { + let html = r#" + + + +
Col1Col2
ab
"#; + let md = convert(html); + assert!(md.contains("| Col1 | Col2 |")); + assert!(md.contains("| a | b |")); +} + +#[test] +fn table_with_inline_formatting() { + let html = r#" + + + +
Name
Bold
"#; + let md = convert(html); + assert!(md.contains("| **Bold** |")); +} + +// ─── Thematic Break ────────────────────────────────────────────────── + +#[test] +fn horizontal_rule() { + let md = convert("

Above

Below

"); + assert!(md.contains("Above")); + assert!(md.contains("---")); + assert!(md.contains("Below")); +} + +// ─── Line Break ────────────────────────────────────────────────────── + +#[test] +fn line_break_br() { + let md = convert("

Line one
Line two

"); + assert!(md.contains("Line one \nLine two")); +} + +// ─── Non-Content Elements ──────────────────────────────────────────── + +#[test] +fn script_stripped() { + let md = convert("

Hello

World

"); + assert!(!md.contains("alert")); + assert!(md.contains("Hello")); + assert!(md.contains("World")); +} + +#[test] +fn style_stripped() { + let md = convert("

Content

"); + assert!(!md.contains("color")); + assert!(md.contains("Content")); +} + +#[test] +fn nav_excluded_by_default() { + let md = convert("

Content

"); + assert!(!md.contains("Home")); + assert!(md.contains("Content")); +} + +#[test] +fn footer_excluded_by_default() { + let md = convert("
Copyright 2024

Content

"); + assert!(!md.contains("Copyright")); + assert!(md.contains("Content")); +} + +#[test] +fn form_elements_stripped() { + let md = convert("

Before

After

"); + assert!(!md.contains("input")); + assert!(md.contains("Before")); + assert!(md.contains("After")); +} + +// ─── Include/Exclude Selectors ─────────────────────────────────────── + +#[test] +fn include_selector() { + let html = r#"
Main content
"#; + let md = convert_with(html, |b| b.include_selector("article")); + assert!(md.contains("Main content")); + assert!(!md.contains("Sidebar")); +} + +#[test] +fn exclude_selector_custom() { + let html = r#"

Keep this

Buy stuff

And this

"#; + let md = convert_with(html, |b| b.exclude_selector(".ad")); + assert!(md.contains("Keep this")); + assert!(md.contains("And this")); + assert!(!md.contains("Buy stuff")); +} + +// ─── Full Document ─────────────────────────────────────────────────── + +#[test] +fn full_html_document() { + let html = r#" + +Test + +

Title

+

First paragraph with bold and a link.

+

Section

+
    +
  • Item one
    • +
  • Item two
    • +
+

A quote

+ 
fn main() {}
+ +"#; + let md = convert(html); + assert!(md.contains("# Title")); + assert!(md.contains("**bold**")); + assert!(md.contains("[a link](/link)")); + assert!(md.contains("## Section")); + assert!(md.contains("- Item one")); + assert!(md.contains("- Item two")); + assert!(md.contains("> A quote")); + assert!(md.contains("```rust")); + assert!(md.contains("fn main() {}")); +} + +// ─── Edge Cases ────────────────────────────────────────────────────── + +#[test] +fn empty_input() { + assert_eq!(convert(""), ""); +} + +#[test] +fn whitespace_only() { + assert_eq!(convert(" \n\n "), ""); +} + +#[test] +fn plain_text_no_tags() { + let md = convert("Just plain text"); + assert!(md.contains("Just plain text")); +} + +#[test] +fn deeply_nested_divs() { + let html = "

Deep content

"; + let md = convert(html); + assert!(md.contains("Deep content")); +} + +#[test] +fn mixed_block_and_inline() { + let html = "

Title

Text with code inside.

"; + let md = convert(html); + assert!(md.contains("# Title")); + assert!(md.contains("Text with `code` inside.")); +} + +// ─── Resource Limits ───────────────────────────────────────────────── + +#[test] +fn resource_limit_enforced() { + let m = Markkit::builder().max_input_size(10).build(); + let result = m.extract_html_sync("a]".repeat(100).as_str()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, markkit::Error::ResourceLimit(_))); +} + +// ─── IR Roundtrip ──────────────────────────────────────────────────── + +#[test] +fn parse_then_render() { + let m = Markkit::builder().build(); + let doc = m.parse_html("

Hello

World

").unwrap(); + assert!(!doc.is_empty()); + + let mut renderer = markkit::MarkdownRenderer::new(); + let md = renderer.render(&doc); + assert!(md.contains("# Hello")); + assert!(md.contains("World")); +} + +#[test] +fn ir_serialization_roundtrip() { + let m = Markkit::builder().build(); + let doc = m + .parse_html("

Test

Bold text

") + .unwrap(); + let json = serde_json::to_string(&doc).unwrap(); + let doc2: markkit::Document = serde_json::from_str(&json).unwrap(); + assert_eq!(doc, doc2); +} + +// ─── Determinism ───────────────────────────────────────────────────── + +#[test] +fn deterministic_output() { + let html = r#" +

Title

+

Some bold and italic text.

+
  • A
  • B
+
X
1
+ "#; + let md1 = convert(html); + let md2 = convert(html); + assert_eq!(md1, md2, "Output must be deterministic"); +} + +#[test] +fn idempotent_double_extract() { + let html = "

Title

Content

"; + let md1 = convert(html); + let md2 = convert(html); + assert_eq!(md1, md2); +} + +// ─── Tables Edge Cases ────────────────────────────────────────────── + +#[test] +fn table_no_thead() { + let html = r#" + + + + + +
NameValue
A1
"#; + let md = convert(html); + assert!(md.contains("| Name | Value |")); + assert!(md.contains("| --- | --- |")); + assert!(md.contains("| A | 1 |")); +} + +#[test] +fn table_headers_only() { + let html = r#" + + +
Col1Col2
"#; + let md = convert(html); + assert!(md.contains("| Col1 | Col2 |")); + assert!(md.contains("| --- | --- |")); +} + +#[test] +fn table_mismatched_columns() { + let html = r#" + + + + + +
ABC
12
"#; + let md = convert(html); + assert!(md.contains("| A | B | C |")); + assert!(md.contains("| --- | --- | --- |")); + // Row with fewer cells still renders (padded with empty cells) + assert!(md.contains("| 1 | 2 |")); +} + +#[test] +fn table_empty() { + let html = "
"; + let md = convert(html); + assert_eq!(md, ""); +} + +// ─── Lists Edge Cases ─────────────────────────────────────────────── + +#[test] +fn nested_list_three_levels() { + let html = "
  • L1
    • L2
      • L3
"; + let md = convert(html); + assert!(md.contains("- L1")); + assert!(md.contains(" - L2")); + assert!(md.contains(" - L3")); +} + +#[test] +fn ordered_list_default_start() { + let html = "
  1. Alpha
  2. Beta
  3. Gamma
"; + let md = convert(html); + assert!(md.contains("1. Alpha")); + assert!(md.contains("2. Beta")); + assert!(md.contains("3. Gamma")); +} + +// ─── Renderer Edge Cases ──────────────────────────────────────────── + +#[test] +fn link_empty_url() { + let md = convert(r#"

text

"#); + assert!(md.contains("[text]()")); +} + +#[test] +fn image_empty_src() { + let md = convert(r#"x"#); + assert!(md.contains("![x]()")); +} + +#[test] +fn code_block_with_backticks() { + let html = r#"
let x = `hello`;
"#; + let md = convert(html); + assert!(md.contains("```")); + assert!(md.contains("let x = `hello`;")); +} + +// ─── Config Tests ─────────────────────────────────────────────────── + +#[test] +fn config_max_processing_secs() { + let m = Markkit::builder().max_processing_secs(30).build(); + assert_eq!(m.config().max_processing_secs, 30); +} + +#[test] +fn config_multiple_include_selectors() { + let html = r#"
AAA
BBB
CCC
"#; + let md = convert_with(html, |b| b.include_selector(".a").include_selector(".b")); + assert!(md.contains("AAA")); + assert!(md.contains("BBB")); + assert!(!md.contains("CCC")); +} + +#[test] +fn config_include_and_exclude() { + let html = r#"

Keep

Remove

Also keep

"#; + let md = convert_with(html, |b| { + b.include_selector("article").exclude_selector(".ad") + }); + assert!(md.contains("Keep")); + assert!(md.contains("Also keep")); + assert!(!md.contains("Remove")); +} + +// ─── Format Detection ─────────────────────────────────────────────── + +#[test] +fn format_content_type_with_params() { + use markkit::format::Format; + assert_eq!( + Format::from_content_type("text/html; charset=utf-8; boundary=foo"), + Format::Html + ); +} + +#[test] +fn format_empty_extension() { + use markkit::format::Format; + assert_eq!(Format::from_extension(""), Format::Unknown); +} + +#[test] +fn format_mixed_case_extension() { + use markkit::format::Format; + assert_eq!(Format::from_extension("HtMl"), Format::Html); +} + +// ─── Extract Bytes ────────────────────────────────────────────────── + +#[tokio::test] +async fn extract_bytes_html() { + let m = Markkit::builder().build(); + let html = b"

Hello

World

"; + let md = m.extract_bytes(html, Some("text/html")).await.unwrap(); + assert!(md.contains("# Hello")); + assert!(md.contains("World")); +} + +#[tokio::test] +async fn extract_bytes_unknown_format() { + let m = Markkit::builder().build(); + let data = b"some random bytes"; + let result = m + .extract_bytes(data, Some("application/octet-stream")) + .await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn extract_bytes_no_content_type() { + let m = Markkit::builder().build(); + let data = b"some random bytes"; + let result = m.extract_bytes(data, None).await; + assert!(result.is_err()); +} + +// ─── Parser Edge Cases ────────────────────────────────────────────── + +#[test] +fn html_no_body() { + let md = convert("

Title

Content

"); + assert!(md.contains("# Title")); + assert!(md.contains("Content")); +} + +#[test] +fn empty_list_items_skipped() { + let md = convert("
  • text
"); + assert!(md.contains("- text")); + // Empty list item should be skipped + let dash_count = md.matches("- ").count(); + assert_eq!(dash_count, 1, "Expected only one list item, got: {}", md); +} + +#[test] +fn div_with_multiple_blocks() { + let html = "

First

Second

"; + let md = convert(html); + assert!(md.contains("First")); + assert!(md.contains("Second")); +} diff --git a/crates/markkit/tests/html_proptest.proptest-regressions b/crates/markkit/tests/html_proptest.proptest-regressions new file mode 100644 index 0000000..9c8b73e --- /dev/null +++ b/crates/markkit/tests/html_proptest.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc e46232cfcc941ef8a82d555aa373e8205f36365b7ee03063ff42dad62414025d # shrinks to text = "A a", tag = "h1" diff --git a/crates/markkit/tests/html_proptest.rs b/crates/markkit/tests/html_proptest.rs new file mode 100644 index 0000000..1f56668 --- /dev/null +++ b/crates/markkit/tests/html_proptest.rs @@ -0,0 +1,69 @@ +//! Property-based tests for HTML extraction. + +use markkit::Markkit; +use proptest::prelude::*; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +proptest! { + /// Arbitrary HTML input should never panic. + #[test] + fn no_panic_on_arbitrary_input(s in "\\PC{0,500}") { + let m = Markkit::builder().build(); + let _ = m.extract_html_sync(&s); + } + + /// Extraction is deterministic: same input always gives same output. + #[test] + fn deterministic_output(s in "\\PC{0,200}") { + let m = Markkit::builder().build(); + let r1 = m.extract_html_sync(&s); + let r2 = m.extract_html_sync(&s); + prop_assert_eq!(r1.ok(), r2.ok()); + } + + /// Output never contains raw HTML tags for dangerous elements. + #[test] + fn no_script_tags_in_output(s in "\\PC{0,300}") { + let md = convert(&s); + prop_assert!(!md.contains("")); + } + + /// Output never contains style blocks. + #[test] + fn no_style_tags_in_output(s in "\\PC{0,300}") { + let md = convert(&s); + prop_assert!(!md.contains("")); + } + + /// Resource limit is always enforced. + #[test] + fn resource_limit_enforced(size in 1usize..100, input_len in 100usize..500) { + let m = Markkit::builder().max_input_size(size).build(); + let input = "x".repeat(input_len); + if input_len > size { + prop_assert!(m.extract_html_sync(&input).is_err()); + } + } + + /// Generated HTML with valid tags should produce non-empty output. + #[test] + fn valid_html_produces_output( + text in "[a-zA-Z]{1,50}", + tag in prop_oneof!["h1", "h2", "p", "em", "strong"].boxed() + ) { + let html = format!("<{}>{}", tag, text, tag); + let md = convert(&html); + prop_assert!( + md.contains(&text), + "Output '{}' should contain text '{}'", + md, + text + ); + } +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs new file mode 100644 index 0000000..508b068 --- /dev/null +++ b/crates/markkit/tests/html_security.rs @@ -0,0 +1,204 @@ +//! Security-focused tests for HTML parsing. + +use markkit::Markkit; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +// ─── XSS Payloads Stripped ─────────────────────────────────────────── + +#[test] +fn xss_script_tag() { + let md = convert("

Hello

"); + assert!(!md.contains("document.cookie")); + assert!(!md.contains("" alt="xss">"#); + // Image is rendered as markdown image — the data URI is in the URL portion + // which is safe in markdown context (no script execution) + assert!(md.contains("![xss]")); + // The output should not contain raw HTML script tags outside of a URL + assert!(!md.starts_with("` | Stripped (non-content element) | +| Event handlers | `

` | Attribute not read | +| JavaScript URLs | `` | URL preserved as-is in markdown (no execution context) | +| Data URI payloads | `` | Rendered as markdown image (no execution) | +| Style expressions | `style="background:url(javascript:...)"` | Attribute not read | +| Deep nesting | 1000+ nested `

` | Flattened to text at depth 200 | +| Huge text nodes | 100KB+ single text node | Passed through (bounded by input limit) | +| Null bytes | `\0` in content | Replaced with U+FFFD by html5ever | +| Unclosed tags | `

text

more` | html5ever auto-closes per spec | +| Mismatched tags | `

...

...

` | html5ever repairs per spec | +| Empty tags | `

` | Elided (empty containers skipped) | +| HTML entities | `> & <` | Decoded by html5ever | +| Encoding edge cases | Non-UTF-8 bytes | `String::from_utf8_lossy` for byte input | +| SVG with scripts | `` | SVG element stripped entirely | +| iframe injection | `