From 336db80350a97e18a653516dd766165881d59b22 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Fri, 27 Mar 2026 02:52:00 +0000 Subject: [PATCH 1/3] feat(markkit): implement HTML-to-Markdown pipeline via canonical IR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the core Parse → Normalize → Render pipeline: - ir.rs: Canonical intermediate representation (Document, Node, ListItem) with serde support for JSON AST output - html/parser.rs: HTML → IR parser using scraper/html5ever with configurable include/exclude CSS selectors, depth limiting (200), whitespace normalization, and non-content element stripping - render/markdown.rs: IR → Markdown renderer supporting headings, paragraphs, inline formatting, links, images, lists (nested), blockquotes, code blocks, tables (GFM), and thematic breaks - extract.rs: Wires parser + renderer together with sync/async APIs and separate parse_html() for IR-only access 92 tests across 4 test suites: - Unit tests (9): IR serialization, format detection, whitespace handling - Integration tests (58): every HTML element mapping, selectors, full documents, edge cases, determinism - Security tests (18): XSS payloads, deep nesting, malformed HTML, resource limits, unicode, null bytes - Property tests (6): fuzz with arbitrary input, determinism, no dangerous tags in output --- crates/markkit/src/extract.rs | 32 +- crates/markkit/src/html/mod.rs | 7 + crates/markkit/src/html/parser.rs | 513 ++++++++++++++++++ crates/markkit/src/ir.rs | 132 +++++ crates/markkit/src/lib.rs | 6 + crates/markkit/src/render/markdown.rs | 388 +++++++++++++ crates/markkit/src/render/mod.rs | 7 + crates/markkit/tests/html_integration.rs | 498 +++++++++++++++++ .../tests/html_proptest.proptest-regressions | 7 + crates/markkit/tests/html_proptest.rs | 69 +++ crates/markkit/tests/html_security.rs | 161 ++++++ 11 files changed, 1818 insertions(+), 2 deletions(-) create mode 100644 crates/markkit/src/html/mod.rs create mode 100644 crates/markkit/src/html/parser.rs create mode 100644 crates/markkit/src/ir.rs create mode 100644 crates/markkit/src/render/markdown.rs create mode 100644 crates/markkit/src/render/mod.rs create mode 100644 crates/markkit/tests/html_integration.rs create mode 100644 crates/markkit/tests/html_proptest.proptest-regressions create mode 100644 crates/markkit/tests/html_proptest.rs create mode 100644 crates/markkit/tests/html_security.rs diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index c8be920..a7ceed3 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -21,6 +21,11 @@ impl Markkit { /// Extract markdown from an HTML string. pub async fn extract_html(&self, html: &str) -> crate::Result { + self.extract_html_sync(html) + } + + /// Synchronous HTML extraction (no async needed for parsing). + pub fn extract_html_sync(&self, html: &str) -> crate::Result { if html.len() > self.config.max_input_size { return Err(Error::ResourceLimit(format!( "Input size {} exceeds limit {}", @@ -29,8 +34,31 @@ impl Markkit { ))); } - // TODO: Implement HTML-to-markdown conversion - Ok(String::new()) + #[cfg(feature = "html")] + { + let parser = crate::html::HtmlParser::new(&self.config); + let doc = parser.parse(html)?; + let mut renderer = crate::render::MarkdownRenderer::new(); + Ok(renderer.render(&doc)) + } + #[cfg(not(feature = "html"))] + Err(Error::UnsupportedFormat( + "HTML support requires the 'html' feature".to_string(), + )) + } + + /// Parse HTML into the IR without rendering. + #[cfg(feature = "html")] + pub fn parse_html(&self, html: &str) -> crate::Result { + if html.len() > self.config.max_input_size { + return Err(Error::ResourceLimit(format!( + "Input size {} exceeds limit {}", + html.len(), + self.config.max_input_size + ))); + } + let parser = crate::html::HtmlParser::new(&self.config); + parser.parse(html) } /// Extract markdown from raw bytes, auto-detecting the format. diff --git a/crates/markkit/src/html/mod.rs b/crates/markkit/src/html/mod.rs new file mode 100644 index 0000000..4809ce9 --- /dev/null +++ b/crates/markkit/src/html/mod.rs @@ -0,0 +1,7 @@ +//! HTML parsing module. +//! +//! Converts HTML input into the canonical IR. + +mod parser; + +pub use parser::HtmlParser; diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs new file mode 100644 index 0000000..8f3efb3 --- /dev/null +++ b/crates/markkit/src/html/parser.rs @@ -0,0 +1,513 @@ +//! HTML → IR parser using scraper/html5ever. + +use scraper::{Html, Node as HtmlNode, Selector}; + +use crate::config::MarkkitBuilder; +use crate::error::Error; +use crate::ir::{Document, ListItem, Node}; + +/// Maximum recursion depth for nested elements. +const MAX_DEPTH: usize = 200; + +/// Parses HTML into the canonical IR. +pub struct HtmlParser<'a> { + config: &'a MarkkitBuilder, +} + +impl<'a> HtmlParser<'a> { + pub fn new(config: &'a MarkkitBuilder) -> Self { + Self { config } + } + + /// Parse an HTML string into a [`Document`]. + pub fn parse(&self, html: &str) -> Result { + let dom = Html::parse_document(html); + let root = dom.root_element(); + + let content_nodes: Vec> = + if !self.config.include_selectors.is_empty() { + let sels: Vec = self + .config + .include_selectors + .iter() + .filter_map(|s| Selector::parse(s).ok()) + .collect(); + sels.iter().flat_map(|sel| dom.select(sel)).collect() + } else { + let body_sel = Selector::parse("body").unwrap(); + let body: Vec<_> = dom.select(&body_sel).collect(); + if body.is_empty() { + vec![root] + } else { + body + } + }; + + let exclude_selectors: Vec = self + .config + .exclude_selectors + .iter() + .filter_map(|s| Selector::parse(s).ok()) + .collect(); + + let mut nodes = Vec::new(); + for content_root in content_nodes { + self.convert_children(content_root, &exclude_selectors, &mut nodes, 0); + } + + Ok(Document::new(nodes)) + } + + fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool { + exclude.iter().any(|sel| sel.matches(el)) + } + + fn convert_children( + &self, + parent: scraper::ElementRef<'_>, + exclude: &[Selector], + out: &mut Vec, + depth: usize, + ) { + if depth >= MAX_DEPTH { + // At max depth, just extract text + let text: String = parent.text().collect(); + let trimmed = text.trim(); + if !trimmed.is_empty() { + out.push(Node::Text(trimmed.to_string())); + } + return; + } + + for child in parent.children() { + match child.value() { + HtmlNode::Text(text) => { + let t = text.to_string(); + let collapsed = collapse_whitespace(&t); + if !collapsed.is_empty() { + out.push(Node::Text(collapsed)); + } else if t.contains(char::is_whitespace) && !out.is_empty() { + out.push(Node::Text(" ".into())); + } + } + HtmlNode::Element(_) => { + if let Some(el) = scraper::ElementRef::wrap(child) { + if self.is_excluded(&el, exclude) { + continue; + } + if let Some(node) = self.convert_element(el, exclude, depth + 1) { + out.push(node); + } + } + } + _ => {} + } + } + } + + fn convert_element( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + if depth >= MAX_DEPTH { + let text: String = el.text().collect(); + let trimmed = text.trim(); + if trimmed.is_empty() { + return None; + } + return Some(Node::Text(trimmed.to_string())); + } + + let tag = el.value().name(); + + match tag { + "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => { + let level = tag[1..].parse::().unwrap_or(1); + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Heading { level, children }) + } + + "p" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Paragraph { children }) + } + + "div" | "section" | "article" | "main" | "span" | "figure" | "figcaption" => { + let mut nodes = Vec::new(); + self.convert_children(el, exclude, &mut nodes, depth); + if nodes.len() == 1 { + Some(nodes.remove(0)) + } else if nodes.is_empty() { + None + } else if nodes.iter().all(is_inline) { + Some(Node::Paragraph { children: nodes }) + } else { + None + } + } + + "ul" => { + let items = self.collect_list_items(el, exclude, depth); + if items.is_empty() { + return None; + } + Some(Node::List { + ordered: false, + start: None, + items, + }) + } + + "ol" => { + let start = el + .value() + .attr("start") + .and_then(|s| s.parse::().ok()); + let items = self.collect_list_items(el, exclude, depth); + if items.is_empty() { + return None; + } + Some(Node::List { + ordered: true, + start, + items, + }) + } + + "blockquote" => { + let mut children = Vec::new(); + self.convert_children(el, exclude, &mut children, depth); + if children.is_empty() { + return None; + } + Some(Node::BlockQuote { children }) + } + + "pre" => { + let code_sel = Selector::parse("code").unwrap(); + if let Some(code_el) = el.select(&code_sel).next() { + let language = code_el + .value() + .attr("class") + .and_then(extract_language_from_class); + let code = code_el.text().collect::(); + Some(Node::CodeBlock { language, code }) + } else { + let code = el.text().collect::(); + Some(Node::CodeBlock { + language: None, + code, + }) + } + } + + "code" => { + let text = el.text().collect::(); + Some(Node::InlineCode(text)) + } + + "table" => self.convert_table(el, exclude, depth), + + "strong" | "b" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Bold { children }) + } + + "em" | "i" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Italic { children }) + } + + "del" | "s" | "strike" => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Strikethrough { children }) + } + + "a" => { + if !self.config.preserve_links { + let children = self.collect_inline(el, exclude, depth); + return if children.is_empty() { + None + } else if children.len() == 1 { + Some(children.into_iter().next().unwrap()) + } else { + Some(Node::Paragraph { children }) + }; + } + let url = el.value().attr("href").unwrap_or("").to_string(); + let title = el.value().attr("title").map(|s| s.to_string()); + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + return None; + } + Some(Node::Link { + url, + title, + children, + }) + } + + "img" => { + if !self.config.preserve_images { + return None; + } + let src = el.value().attr("src").unwrap_or("").to_string(); + let alt = el.value().attr("alt").unwrap_or("").to_string(); + Some(Node::Image { src, alt }) + } + + "br" => Some(Node::LineBreak), + + "hr" => Some(Node::ThematicBreak), + + // Skip non-content elements + "script" | "style" | "noscript" | "iframe" | "object" | "embed" | "form" + | "input" | "button" | "select" | "textarea" | "svg" | "canvas" | "video" + | "audio" | "source" | "track" | "map" | "area" => None, + + // Unknown element: try to extract inline content + _ => { + let children = self.collect_inline(el, exclude, depth); + if children.is_empty() { + None + } else if children.len() == 1 { + Some(children.into_iter().next().unwrap()) + } else { + Some(Node::Paragraph { children }) + } + } + } + } + + fn collect_inline( + &self, + el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Vec { + let mut nodes = Vec::new(); + if depth >= MAX_DEPTH { + let text: String = el.text().collect(); + let trimmed = text.trim(); + if !trimmed.is_empty() { + nodes.push(Node::Text(trimmed.to_string())); + } + return nodes; + } + + for child in el.children() { + match child.value() { + HtmlNode::Text(text) => { + let t = text.to_string(); + let collapsed = collapse_whitespace(&t); + if !collapsed.is_empty() { + nodes.push(Node::Text(collapsed)); + } else if t.contains(char::is_whitespace) && !nodes.is_empty() { + nodes.push(Node::Text(" ".into())); + } + } + HtmlNode::Element(_) => { + if let Some(child_el) = scraper::ElementRef::wrap(child) { + if self.is_excluded(&child_el, exclude) { + continue; + } + if let Some(node) = self.convert_element(child_el, exclude, depth + 1) { + nodes.push(node); + } + } + } + _ => {} + } + } + nodes + } + + fn collect_list_items( + &self, + list_el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Vec { + let mut items = Vec::new(); + for child in list_el.children() { + if let Some(el) = scraper::ElementRef::wrap(child) + && el.value().name() == "li" + { + let mut children = Vec::new(); + self.convert_children(el, exclude, &mut children, depth + 1); + if !children.is_empty() { + items.push(ListItem::new(children)); + } + } + } + items + } + + fn convert_table( + &self, + table_el: scraper::ElementRef<'_>, + exclude: &[Selector], + depth: usize, + ) -> Option { + let thead_sel = Selector::parse("thead").unwrap(); + let tbody_sel = Selector::parse("tbody").unwrap(); + let tr_sel = Selector::parse("tr").unwrap(); + let th_sel = Selector::parse("th").unwrap(); + let td_sel = Selector::parse("td").unwrap(); + + let mut headers: Vec> = Vec::new(); + let mut rows: Vec>> = Vec::new(); + + if let Some(thead) = table_el.select(&thead_sel).next() + && let Some(tr) = thead.select(&tr_sel).next() + { + for th in tr.select(&th_sel) { + headers.push(self.collect_inline(th, exclude, depth + 1)); + } + } + + let body_rows: Vec> = + if let Some(tbody) = table_el.select(&tbody_sel).next() { + tbody.select(&tr_sel).collect() + } else { + table_el.select(&tr_sel).collect() + }; + + for tr in body_rows { + if headers.is_empty() { + let ths: Vec<_> = tr.select(&th_sel).collect(); + if !ths.is_empty() { + for th in ths { + headers.push(self.collect_inline(th, exclude, depth + 1)); + } + continue; + } + } + + let cells: Vec> = tr + .select(&td_sel) + .map(|td| self.collect_inline(td, exclude, depth + 1)) + .collect(); + + if !cells.is_empty() { + rows.push(cells); + } + } + + if headers.is_empty() && rows.is_empty() { + return None; + } + + Some(Node::Table { headers, rows }) + } +} + +/// Extract language identifier from a class like "language-rust" or "lang-python". +fn extract_language_from_class(class: &str) -> Option { + for cls in class.split_whitespace() { + if let Some(lang) = cls.strip_prefix("language-") { + return Some(lang.to_string()); + } + if let Some(lang) = cls.strip_prefix("lang-") { + return Some(lang.to_string()); + } + } + None +} + +/// Collapse runs of whitespace into single spaces. +/// Preserves a leading/trailing space if the original had one (important for inline context). +fn collapse_whitespace(s: &str) -> String { + let has_leading_space = s.starts_with(char::is_whitespace); + let has_trailing_space = s.ends_with(char::is_whitespace); + + let mut result = String::with_capacity(s.len()); + let mut prev_space = true; // suppress leading space initially + for ch in s.chars() { + if ch.is_whitespace() { + if !prev_space { + result.push(' '); + } + prev_space = true; + } else { + prev_space = false; + result.push(ch); + } + } + if result.ends_with(' ') { + result.pop(); + } + + if has_leading_space && !result.is_empty() { + result.insert(0, ' '); + } + if has_trailing_space && !result.is_empty() { + result.push(' '); + } + + result +} + +/// Check if a node is inline content. +fn is_inline(node: &Node) -> bool { + matches!( + node, + Node::Text(_) + | Node::Bold { .. } + | Node::Italic { .. } + | Node::Strikethrough { .. } + | Node::InlineCode(_) + | Node::Link { .. } + | Node::Image { .. } + | Node::LineBreak + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collapse_whitespace() { + assert_eq!(collapse_whitespace(" hello world "), " hello world "); + assert_eq!(collapse_whitespace("a"), "a"); + assert_eq!(collapse_whitespace(" "), ""); + assert_eq!(collapse_whitespace("a\n b"), "a b"); + assert_eq!(collapse_whitespace(" bold "), " bold "); + assert_eq!(collapse_whitespace("no-spaces"), "no-spaces"); + } + + #[test] + fn test_extract_language_from_class() { + assert_eq!( + extract_language_from_class("language-rust"), + Some("rust".into()) + ); + assert_eq!( + extract_language_from_class("lang-python"), + Some("python".into()) + ); + assert_eq!( + extract_language_from_class("highlight language-js extra"), + Some("js".into()) + ); + assert_eq!(extract_language_from_class("highlight"), None); + } +} diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs new file mode 100644 index 0000000..8770565 --- /dev/null +++ b/crates/markkit/src/ir.rs @@ -0,0 +1,132 @@ +//! Canonical intermediate representation for documents. +//! +//! All parsers convert their input into this IR, and all renderers +//! consume it. This decouples input formats from output formats. + +use serde::{Deserialize, Serialize}; + +/// A parsed document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Document { + pub nodes: Vec, +} + +impl Document { + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } +} + +/// A block-level or inline node in the document tree. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Node { + /// Heading with level 1–6. + Heading { + level: u8, + children: Vec, + }, + /// Paragraph containing inline content. + Paragraph { + children: Vec, + }, + /// Ordered or unordered list. + List { + ordered: bool, + start: Option, + items: Vec, + }, + /// Block quotation. + BlockQuote { + children: Vec, + }, + /// Fenced code block. + CodeBlock { + language: Option, + code: String, + }, + /// Table with optional header row. + Table { + headers: Vec>, + rows: Vec>>, + }, + /// Horizontal rule / thematic break. + ThematicBreak, + /// Plain text. + Text(String), + /// Bold / strong emphasis. + Bold { + children: Vec, + }, + /// Italic / emphasis. + Italic { + children: Vec, + }, + /// Strikethrough. + Strikethrough { + children: Vec, + }, + /// Inline code. + InlineCode(String), + /// Hyperlink. + Link { + url: String, + title: Option, + children: Vec, + }, + /// Image. + Image { + src: String, + alt: String, + }, + /// Hard line break. + LineBreak, +} + +/// A single item in a list. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ListItem { + pub children: Vec, +} + +impl ListItem { + pub fn new(children: Vec) -> Self { + Self { children } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn document_is_empty() { + assert!(Document::new(vec![]).is_empty()); + assert!(!Document::new(vec![Node::ThematicBreak]).is_empty()); + } + + #[test] + fn ir_roundtrip_json() { + let doc = Document::new(vec![ + Node::Heading { + level: 1, + children: vec![Node::Text("Hello".into())], + }, + Node::Paragraph { + children: vec![ + Node::Text("Some ".into()), + Node::Bold { + children: vec![Node::Text("bold".into())], + }, + Node::Text(" text".into()), + ], + }, + ]); + let json = serde_json::to_string(&doc).unwrap(); + let parsed: Document = serde_json::from_str(&json).unwrap(); + assert_eq!(doc, parsed); + } +} diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 36e58a4..738a063 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -35,10 +35,16 @@ pub mod config; pub mod error; pub mod extract; pub mod format; +#[cfg(feature = "html")] +pub mod html; +pub mod ir; +pub mod render; pub use config::MarkkitBuilder; pub use error::Error; pub use extract::Markkit; +pub use ir::Document; +pub use render::MarkdownRenderer; /// Result type alias for markkit operations. pub type Result = std::result::Result; diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs new file mode 100644 index 0000000..fc41c9d --- /dev/null +++ b/crates/markkit/src/render/markdown.rs @@ -0,0 +1,388 @@ +//! IR → Markdown renderer. + +use crate::ir::{Document, ListItem, Node}; + +/// Renders a [`Document`] IR into Markdown text. +pub struct MarkdownRenderer { + /// Output buffer. + buf: String, +} + +impl MarkdownRenderer { + pub fn new() -> Self { + Self { + buf: String::with_capacity(4096), + } + } + + /// Render a document to a Markdown string. + pub fn render(&mut self, doc: &Document) -> String { + self.buf.clear(); + self.render_nodes(&doc.nodes, 0); + normalize_output(&self.buf) + } + + /// Render a slice of nodes. + fn render_nodes(&mut self, nodes: &[Node], depth: usize) { + for (i, node) in nodes.iter().enumerate() { + if i > 0 && is_block(node) && is_block(&nodes[i - 1]) { + self.buf.push('\n'); + } + self.render_node(node, depth); + } + } + + fn render_node(&mut self, node: &Node, depth: usize) { + match node { + Node::Heading { level, children } => { + let prefix = "#".repeat(*level as usize); + self.buf.push_str(&prefix); + self.buf.push(' '); + let start = self.buf.len(); + self.render_inline(children); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + } + + Node::Paragraph { children } => { + let start = self.buf.len(); + self.render_inline(children); + trim_trailing_inline(&mut self.buf, start); + self.buf.push('\n'); + } + + Node::List { + ordered, + start, + items, + } => { + self.render_list(*ordered, *start, items, depth); + } + + Node::BlockQuote { children } => { + let inner = { + let mut sub = MarkdownRenderer::new(); + sub.render_nodes(children, depth); + normalize_output(&sub.buf) + }; + for line in inner.lines() { + if line.is_empty() { + self.buf.push_str(">\n"); + } else { + self.buf.push_str("> "); + self.buf.push_str(line); + self.buf.push('\n'); + } + } + } + + Node::CodeBlock { language, code } => { + self.buf.push_str("```"); + if let Some(lang) = language { + self.buf.push_str(lang); + } + self.buf.push('\n'); + self.buf.push_str(code); + if !code.ends_with('\n') { + self.buf.push('\n'); + } + self.buf.push_str("```\n"); + } + + Node::Table { headers, rows } => { + self.render_table(headers, rows); + } + + Node::ThematicBreak => { + self.buf.push_str("---\n"); + } + + // Inline nodes rendered at block level get wrapped in a line + Node::Text(_) + | Node::Bold { .. } + | Node::Italic { .. } + | Node::Strikethrough { .. } + | Node::InlineCode(_) + | Node::Link { .. } + | Node::Image { .. } + | Node::LineBreak => { + self.render_inline_node(node); + } + } + } + + /// Render a list. + fn render_list( + &mut self, + ordered: bool, + start: Option, + items: &[ListItem], + depth: usize, + ) { + let indent = " ".repeat(depth); + let start_num = start.unwrap_or(1); + + for (i, item) in items.iter().enumerate() { + let marker = if ordered { + format!("{}{}. ", indent, start_num + i as u64) + } else { + format!("{}- ", indent) + }; + self.buf.push_str(&marker); + + // Render item children + let has_block = item.children.iter().any(|n| !is_inline_node(n)); + if has_block { + // Multi-line item: render sub-blocks indented + let mut first = true; + for child in &item.children { + if first { + first = false; + match child { + Node::Paragraph { children } => { + self.render_inline(children); + self.buf.push('\n'); + } + Node::List { + ordered: sub_ord, + start: sub_start, + items: sub_items, + } => { + self.buf.push('\n'); + self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); + } + _ => { + let mut sub = MarkdownRenderer::new(); + sub.render_node(child, depth + 1); + let text = sub.buf.trim_end().to_string(); + self.buf.push_str(&text); + self.buf.push('\n'); + } + } + } else { + match child { + Node::List { + ordered: sub_ord, + start: sub_start, + items: sub_items, + } => { + self.render_list(*sub_ord, *sub_start, sub_items, depth + 1); + } + _ => { + let mut sub = MarkdownRenderer::new(); + sub.render_node(child, depth + 1); + let inner = sub.buf.trim_end().to_string(); + let sub_indent = " ".repeat(depth + 1); + for line in inner.lines() { + self.buf.push_str(&sub_indent); + self.buf.push_str(line); + self.buf.push('\n'); + } + } + } + } + } + } else { + // Simple inline item + self.render_inline(&item.children); + self.buf.push('\n'); + } + } + } + + /// Render inline content. + fn render_inline(&mut self, nodes: &[Node]) { + for node in nodes { + self.render_inline_node(node); + } + } + + /// Render a single inline node. + fn render_inline_node(&mut self, node: &Node) { + match node { + Node::Text(t) => self.buf.push_str(t), + Node::Bold { children } => { + self.buf.push_str("**"); + self.render_inline(children); + self.buf.push_str("**"); + } + Node::Italic { children } => { + self.buf.push('*'); + self.render_inline(children); + self.buf.push('*'); + } + Node::Strikethrough { children } => { + self.buf.push_str("~~"); + self.render_inline(children); + self.buf.push_str("~~"); + } + Node::InlineCode(code) => { + self.buf.push('`'); + self.buf.push_str(code); + self.buf.push('`'); + } + Node::Link { + url, + title, + children, + } => { + self.buf.push('['); + self.render_inline(children); + self.buf.push_str("]("); + self.buf.push_str(url); + if let Some(t) = title { + self.buf.push_str(" \""); + self.buf.push_str(t); + self.buf.push('"'); + } + self.buf.push(')'); + } + Node::Image { src, alt } => { + self.buf.push_str("!["); + self.buf.push_str(alt); + self.buf.push_str("]("); + self.buf.push_str(src); + self.buf.push(')'); + } + Node::LineBreak => { + self.buf.push_str(" \n"); + } + // Block nodes shouldn't appear here but handle gracefully + other => self.render_node(other, 0), + } + } + + /// Render a GFM table. + fn render_table(&mut self, headers: &[Vec], rows: &[Vec>]) { + // Determine column count + let col_count = headers + .len() + .max(rows.iter().map(|r| r.len()).max().unwrap_or(0)); + + if col_count == 0 { + return; + } + + // Render header row + self.buf.push('|'); + for i in 0..col_count { + self.buf.push(' '); + if let Some(cells) = headers.get(i) { + self.render_inline(cells); + } + self.buf.push_str(" |"); + } + self.buf.push('\n'); + + // Separator + self.buf.push('|'); + for _ in 0..col_count { + self.buf.push_str(" --- |"); + } + self.buf.push('\n'); + + // Data rows + for row in rows { + self.buf.push('|'); + for i in 0..col_count { + self.buf.push(' '); + if let Some(cells) = row.get(i) { + self.render_inline(cells); + } + self.buf.push_str(" |"); + } + self.buf.push('\n'); + } + } +} + +impl Default for MarkdownRenderer { + fn default() -> Self { + Self::new() + } +} + +/// Trim leading and trailing whitespace from inline content rendered since `start`. +fn trim_trailing_inline(buf: &mut String, start: usize) { + // Trim trailing spaces + while buf.len() > start && buf.ends_with(' ') { + buf.pop(); + } + // Trim leading spaces after start + let content = &buf[start..]; + let trimmed_start = content.len() - content.trim_start().len(); + if trimmed_start > 0 { + buf.drain(start..start + trimmed_start); + } +} + +/// Check if a node is block-level. +fn is_block(node: &Node) -> bool { + matches!( + node, + Node::Heading { .. } + | Node::Paragraph { .. } + | Node::List { .. } + | Node::BlockQuote { .. } + | Node::CodeBlock { .. } + | Node::Table { .. } + | Node::ThematicBreak + ) +} + +fn is_inline_node(node: &Node) -> bool { + !is_block(node) +} + +/// Normalize output: collapse multiple blank lines, preserve hard breaks (two trailing spaces). +fn normalize_output(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut blank_count = 0; + + for line in s.lines() { + // Preserve markdown hard break (line ending in two+ spaces with content) + let core = line.trim_end(); + let is_hard_break = line.ends_with(" ") && !core.is_empty(); + let trimmed = if is_hard_break { + core.to_string() + " " + } else { + core.to_string() + }; + + if trimmed.is_empty() || trimmed == " " { + blank_count += 1; + if blank_count <= 1 { + result.push('\n'); + } + } else { + blank_count = 0; + result.push_str(&trimmed); + result.push('\n'); + } + } + + // Trim leading/trailing blank lines + let trimmed = result.trim_matches('\n'); + if trimmed.is_empty() { + String::new() + } else { + let mut out = trimmed.to_string(); + out.push('\n'); + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_output() { + assert_eq!(normalize_output("a\n\n\n\nb\n"), "a\n\nb\n"); + assert_eq!(normalize_output("\n\na\n\n"), "a\n"); + assert_eq!(normalize_output(""), ""); + assert_eq!(normalize_output(" \n \n"), ""); + // Hard break preserved + assert_eq!(normalize_output("a \nb\n"), "a \nb\n"); + } +} diff --git a/crates/markkit/src/render/mod.rs b/crates/markkit/src/render/mod.rs new file mode 100644 index 0000000..b91fc60 --- /dev/null +++ b/crates/markkit/src/render/mod.rs @@ -0,0 +1,7 @@ +//! Rendering module. +//! +//! Converts the canonical IR into output formats. + +mod markdown; + +pub use markdown::MarkdownRenderer; diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs new file mode 100644 index 0000000..bd54d77 --- /dev/null +++ b/crates/markkit/tests/html_integration.rs @@ -0,0 +1,498 @@ +//! Integration tests for HTML → Markdown conversion. + +use markkit::Markkit; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +fn convert_with(html: &str, configure: F) -> String +where + F: FnOnce(markkit::MarkkitBuilder) -> markkit::MarkkitBuilder, +{ + let m = configure(Markkit::builder()).build(); + m.extract_html_sync(html).unwrap() +} + +// ─── Headings ──────────────────────────────────────────────────────── + +#[test] +fn heading_h1() { + assert_eq!(convert("

Hello World

"), "# Hello World\n"); +} + +#[test] +fn heading_h2() { + assert_eq!(convert("

Sub Heading

"), "## Sub Heading\n"); +} + +#[test] +fn heading_h3_through_h6() { + assert_eq!(convert("

H3

"), "### H3\n"); + assert_eq!(convert("

H4

"), "#### H4\n"); + assert_eq!(convert("
H5
"), "##### H5\n"); + assert_eq!(convert("
H6
"), "###### H6\n"); +} + +#[test] +fn heading_with_inline_formatting() { + assert_eq!( + convert("

Hello World

"), + "# Hello *World*\n" + ); +} + +#[test] +fn heading_empty_is_skipped() { + assert_eq!(convert("

"), ""); +} + +// ─── Paragraphs ────────────────────────────────────────────────────── + +#[test] +fn paragraph_simple() { + assert_eq!(convert("

Hello world

"), "Hello world\n"); +} + +#[test] +fn paragraph_multiple() { + let md = convert("

First

Second

"); + assert_eq!(md, "First\n\nSecond\n"); +} + +#[test] +fn paragraph_with_inline_elements() { + let md = convert("

This is bold and italic

"); + assert_eq!(md, "This is **bold** and *italic*\n"); +} + +#[test] +fn paragraph_whitespace_collapse() { + let md = convert("

hello world

"); + assert_eq!(md, "hello world\n"); +} + +// ─── Inline Formatting ────────────────────────────────────────────── + +#[test] +fn bold_strong() { + assert_eq!(convert("

bold

"), "**bold**\n"); +} + +#[test] +fn bold_b_tag() { + assert_eq!(convert("

bold

"), "**bold**\n"); +} + +#[test] +fn italic_em() { + assert_eq!(convert("

italic

"), "*italic*\n"); +} + +#[test] +fn italic_i_tag() { + assert_eq!(convert("

italic

"), "*italic*\n"); +} + +#[test] +fn strikethrough_del() { + assert_eq!(convert("

removed

"), "~~removed~~\n"); +} + +#[test] +fn strikethrough_s() { + assert_eq!(convert("

removed

"), "~~removed~~\n"); +} + +#[test] +fn inline_code() { + assert_eq!(convert("

let x = 1

"), "`let x = 1`\n"); +} + +#[test] +fn nested_inline_bold_italic() { + assert_eq!( + convert("

bold italic

"), + "***bold italic***\n" + ); +} + +// ─── Links ─────────────────────────────────────────────────────────── + +#[test] +fn link_basic() { + assert_eq!( + convert(r#"

Example

"#), + "[Example](https://example.com)\n" + ); +} + +#[test] +fn link_with_title() { + assert_eq!( + convert(r#"

Example

"#), + "[Example](https://example.com \"A site\")\n" + ); +} + +#[test] +fn link_with_inline_formatting() { + assert_eq!( + convert(r#"

Bold Link

"#), + "[**Bold Link**](/url)\n" + ); +} + +#[test] +fn link_preservation_disabled() { + let md = convert_with( + r#"

Example

"#, + |b| b.preserve_links(false), + ); + assert_eq!(md, "Example\n"); +} + +// ─── Images ────────────────────────────────────────────────────────── + +#[test] +fn image_basic() { + assert_eq!( + convert(r#"A photo"#), + "![A photo](photo.jpg)\n" + ); +} + +#[test] +fn image_no_alt() { + assert_eq!(convert(r#""#), "![](photo.jpg)\n"); +} + +#[test] +fn image_preservation_disabled() { + let md = convert_with(r#"A photo"#, |b| { + b.preserve_images(false) + }); + assert_eq!(md, ""); +} + +// ─── Lists ─────────────────────────────────────────────────────────── + +#[test] +fn unordered_list() { + let md = convert("
  • One
  • Two
  • Three
"); + assert_eq!(md, "- One\n- Two\n- Three\n"); +} + +#[test] +fn ordered_list() { + let md = convert("
  1. First
  2. Second
"); + assert_eq!(md, "1. First\n2. Second\n"); +} + +#[test] +fn ordered_list_custom_start() { + let md = convert(r#"
  1. Five
  2. Six
"#); + assert_eq!(md, "5. Five\n6. Six\n"); +} + +#[test] +fn nested_list() { + let md = convert( + "
  • Outer
    • Inner A
    • Inner B
  • Other
", + ); + assert!(md.contains("- Outer")); + assert!(md.contains(" - Inner A")); + assert!(md.contains(" - Inner B")); + assert!(md.contains("- Other")); +} + +#[test] +fn list_with_inline_formatting() { + let md = convert("
  • Bold item
"); + assert_eq!(md, "- **Bold** item\n"); +} + +// ─── Blockquotes ───────────────────────────────────────────────────── + +#[test] +fn blockquote_simple() { + let md = convert("

Quoted text

"); + assert_eq!(md, "> Quoted text\n"); +} + +#[test] +fn blockquote_nested() { + let md = convert( + "

Outer

Inner

", + ); + assert!(md.contains("> Outer")); + assert!(md.contains("> > Inner")); +} + +// ─── Code Blocks ───────────────────────────────────────────────────── + +#[test] +fn code_block_plain() { + let md = convert("
fn main() {}
"); + assert_eq!(md, "```\nfn main() {}\n```\n"); +} + +#[test] +fn code_block_with_language() { + let md = convert(r#"
fn main() {}
"#); + assert_eq!(md, "```rust\nfn main() {}\n```\n"); +} + +#[test] +fn code_block_lang_prefix() { + let md = convert(r#"
print("hi")
"#); + assert_eq!(md, "```python\nprint(\"hi\")\n```\n"); +} + +#[test] +fn code_block_pre_without_code() { + let md = convert("
preformatted text
"); + assert_eq!(md, "```\npreformatted text\n```\n"); +} + +// ─── Tables ────────────────────────────────────────────────────────── + +#[test] +fn table_basic() { + let html = r#" + + + + + + +
NameValue
A1
B2
"#; + let md = convert(html); + assert!(md.contains("| Name | Value |")); + assert!(md.contains("| --- | --- |")); + assert!(md.contains("| A | 1 |")); + assert!(md.contains("| B | 2 |")); +} + +#[test] +fn table_th_in_body() { + let html = r#" + + + +
Col1Col2
ab
"#; + let md = convert(html); + assert!(md.contains("| Col1 | Col2 |")); + assert!(md.contains("| a | b |")); +} + +#[test] +fn table_with_inline_formatting() { + let html = r#" + + + +
Name
Bold
"#; + let md = convert(html); + assert!(md.contains("| **Bold** |")); +} + +// ─── Thematic Break ────────────────────────────────────────────────── + +#[test] +fn horizontal_rule() { + let md = convert("

Above

Below

"); + assert!(md.contains("Above")); + assert!(md.contains("---")); + assert!(md.contains("Below")); +} + +// ─── Line Break ────────────────────────────────────────────────────── + +#[test] +fn line_break_br() { + let md = convert("

Line one
Line two

"); + assert!(md.contains("Line one \nLine two")); +} + +// ─── Non-Content Elements ──────────────────────────────────────────── + +#[test] +fn script_stripped() { + let md = convert("

Hello

World

"); + assert!(!md.contains("alert")); + assert!(md.contains("Hello")); + assert!(md.contains("World")); +} + +#[test] +fn style_stripped() { + let md = convert("

Content

"); + assert!(!md.contains("color")); + assert!(md.contains("Content")); +} + +#[test] +fn nav_excluded_by_default() { + let md = convert("

Content

"); + assert!(!md.contains("Home")); + assert!(md.contains("Content")); +} + +#[test] +fn footer_excluded_by_default() { + let md = convert("
Copyright 2024

Content

"); + assert!(!md.contains("Copyright")); + assert!(md.contains("Content")); +} + +#[test] +fn form_elements_stripped() { + let md = convert("

Before

After

"); + assert!(!md.contains("input")); + assert!(md.contains("Before")); + assert!(md.contains("After")); +} + +// ─── Include/Exclude Selectors ─────────────────────────────────────── + +#[test] +fn include_selector() { + let html = r#"
Main content
"#; + let md = convert_with(html, |b| b.include_selector("article")); + assert!(md.contains("Main content")); + assert!(!md.contains("Sidebar")); +} + +#[test] +fn exclude_selector_custom() { + let html = r#"

Keep this

Buy stuff

And this

"#; + let md = convert_with(html, |b| b.exclude_selector(".ad")); + assert!(md.contains("Keep this")); + assert!(md.contains("And this")); + assert!(!md.contains("Buy stuff")); +} + +// ─── Full Document ─────────────────────────────────────────────────── + +#[test] +fn full_html_document() { + let html = r#" + +Test + +

Title

+

First paragraph with bold and a link.

+

Section

+
    +
  • Item one
    • +
  • Item two
    • +
+

A quote

+ 
fn main() {}
+ +"#; + let md = convert(html); + assert!(md.contains("# Title")); + assert!(md.contains("**bold**")); + assert!(md.contains("[a link](/link)")); + assert!(md.contains("## Section")); + assert!(md.contains("- Item one")); + assert!(md.contains("- Item two")); + assert!(md.contains("> A quote")); + assert!(md.contains("```rust")); + assert!(md.contains("fn main() {}")); +} + +// ─── Edge Cases ────────────────────────────────────────────────────── + +#[test] +fn empty_input() { + assert_eq!(convert(""), ""); +} + +#[test] +fn whitespace_only() { + assert_eq!(convert(" \n\n "), ""); +} + +#[test] +fn plain_text_no_tags() { + let md = convert("Just plain text"); + assert!(md.contains("Just plain text")); +} + +#[test] +fn deeply_nested_divs() { + let html = "

Deep content

"; + let md = convert(html); + assert!(md.contains("Deep content")); +} + +#[test] +fn mixed_block_and_inline() { + let html = "

Title

Text with code inside.

"; + let md = convert(html); + assert!(md.contains("# Title")); + assert!(md.contains("Text with `code` inside.")); +} + +// ─── Resource Limits ───────────────────────────────────────────────── + +#[test] +fn resource_limit_enforced() { + let m = Markkit::builder().max_input_size(10).build(); + let result = m.extract_html_sync("a]".repeat(100).as_str()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(matches!(err, markkit::Error::ResourceLimit(_))); +} + +// ─── IR Roundtrip ──────────────────────────────────────────────────── + +#[test] +fn parse_then_render() { + let m = Markkit::builder().build(); + let doc = m.parse_html("

Hello

World

").unwrap(); + assert!(!doc.is_empty()); + + let mut renderer = markkit::MarkdownRenderer::new(); + let md = renderer.render(&doc); + assert!(md.contains("# Hello")); + assert!(md.contains("World")); +} + +#[test] +fn ir_serialization_roundtrip() { + let m = Markkit::builder().build(); + let doc = m + .parse_html("

Test

Bold text

") + .unwrap(); + let json = serde_json::to_string(&doc).unwrap(); + let doc2: markkit::Document = serde_json::from_str(&json).unwrap(); + assert_eq!(doc, doc2); +} + +// ─── Determinism ───────────────────────────────────────────────────── + +#[test] +fn deterministic_output() { + let html = r#" +

Title

+

Some bold and italic text.

+
  • A
  • B
+
X
1
+ "#; + let md1 = convert(html); + let md2 = convert(html); + assert_eq!(md1, md2, "Output must be deterministic"); +} + +#[test] +fn idempotent_double_extract() { + let html = "

Title

Content

"; + let md1 = convert(html); + let md2 = convert(html); + assert_eq!(md1, md2); +} diff --git a/crates/markkit/tests/html_proptest.proptest-regressions b/crates/markkit/tests/html_proptest.proptest-regressions new file mode 100644 index 0000000..9c8b73e --- /dev/null +++ b/crates/markkit/tests/html_proptest.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc e46232cfcc941ef8a82d555aa373e8205f36365b7ee03063ff42dad62414025d # shrinks to text = "A a", tag = "h1" diff --git a/crates/markkit/tests/html_proptest.rs b/crates/markkit/tests/html_proptest.rs new file mode 100644 index 0000000..1f56668 --- /dev/null +++ b/crates/markkit/tests/html_proptest.rs @@ -0,0 +1,69 @@ +//! Property-based tests for HTML extraction. + +use markkit::Markkit; +use proptest::prelude::*; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +proptest! { + /// Arbitrary HTML input should never panic. + #[test] + fn no_panic_on_arbitrary_input(s in "\\PC{0,500}") { + let m = Markkit::builder().build(); + let _ = m.extract_html_sync(&s); + } + + /// Extraction is deterministic: same input always gives same output. + #[test] + fn deterministic_output(s in "\\PC{0,200}") { + let m = Markkit::builder().build(); + let r1 = m.extract_html_sync(&s); + let r2 = m.extract_html_sync(&s); + prop_assert_eq!(r1.ok(), r2.ok()); + } + + /// Output never contains raw HTML tags for dangerous elements. + #[test] + fn no_script_tags_in_output(s in "\\PC{0,300}") { + let md = convert(&s); + prop_assert!(!md.contains("")); + } + + /// Output never contains style blocks. + #[test] + fn no_style_tags_in_output(s in "\\PC{0,300}") { + let md = convert(&s); + prop_assert!(!md.contains("")); + } + + /// Resource limit is always enforced. + #[test] + fn resource_limit_enforced(size in 1usize..100, input_len in 100usize..500) { + let m = Markkit::builder().max_input_size(size).build(); + let input = "x".repeat(input_len); + if input_len > size { + prop_assert!(m.extract_html_sync(&input).is_err()); + } + } + + /// Generated HTML with valid tags should produce non-empty output. + #[test] + fn valid_html_produces_output( + text in "[a-zA-Z]{1,50}", + tag in prop_oneof!["h1", "h2", "p", "em", "strong"].boxed() + ) { + let html = format!("<{}>{}", tag, text, tag); + let md = convert(&html); + prop_assert!( + md.contains(&text), + "Output '{}' should contain text '{}'", + md, + text + ); + } +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs new file mode 100644 index 0000000..c68df29 --- /dev/null +++ b/crates/markkit/tests/html_security.rs @@ -0,0 +1,161 @@ +//! Security-focused tests for HTML parsing. + +use markkit::Markkit; + +fn convert(html: &str) -> String { + let m = Markkit::builder().build(); + m.extract_html_sync(html).unwrap() +} + +// ─── XSS Payloads Stripped ─────────────────────────────────────────── + +#[test] +fn xss_script_tag() { + let md = convert("

Hello

"); + assert!(!md.contains("document.cookie")); + assert!(!md.contains("" alt="xss">"#); + // Image is rendered as markdown image — the data URI is in the URL portion + // which is safe in markdown context (no script execution) + assert!(md.contains("![xss]")); + // The output should not contain raw HTML script tags outside of a URL + assert!(!md.starts_with("` | Stripped (non-content element) | +| Event handlers | `

` | Attribute not read | +| JavaScript URLs | `` | URL preserved as-is in markdown (no execution context) | +| Data URI payloads | `` | Rendered as markdown image (no execution) | +| Style expressions | `style="background:url(javascript:...)"` | Attribute not read | +| Deep nesting | 1000+ nested `

` | Flattened to text at depth 200 | +| Huge text nodes | 100KB+ single text node | Passed through (bounded by input limit) | +| Null bytes | `\0` in content | Replaced with U+FFFD by html5ever | +| Unclosed tags | `

text

more` | html5ever auto-closes per spec | +| Mismatched tags | `

...

...

` | html5ever repairs per spec | +| Empty tags | `

` | Elided (empty containers skipped) | +| HTML entities | `> & <` | Decoded by html5ever | +| Encoding edge cases | Non-UTF-8 bytes | `String::from_utf8_lossy` for byte input | +| SVG with scripts | `` | SVG element stripped entirely | +| iframe injection | `

File Test

Paragraph.

").unwrap(); let output = markkit_bin() - .args([ - "extract", - "--file", - file_path.to_str().unwrap(), - ]) + .args(["extract", "--file", file_path.to_str().unwrap()]) .output() .expect("failed to run markkit"); - assert!(output.status.success(), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("File Test"), "output: {stdout}"); assert!(stdout.contains("Paragraph"), "output: {stdout}"); @@ -113,7 +121,11 @@ fn extract_json_output() { .unwrap(); let output = child.wait_with_output().unwrap(); - assert!(output.status.success(), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); let stdout = String::from_utf8_lossy(&output.stdout); let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output"); assert_eq!(parsed["source"], ""); @@ -138,7 +150,11 @@ fn extract_with_include_selector() { .unwrap(); let output = child.wait_with_output().unwrap(); - assert!(output.status.success(), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("Keep this"), "output: {stdout}"); } @@ -161,10 +177,17 @@ fn extract_with_exclude_selector() { .unwrap(); let output = child.wait_with_output().unwrap(); - assert!(output.status.success(), "stderr: {}", String::from_utf8_lossy(&output.stderr)); + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("Content"), "output: {stdout}"); - assert!(!stdout.contains("Menu"), "nav should be excluded, output: {stdout}"); + assert!( + !stdout.contains("Menu"), + "nav should be excluded, output: {stdout}" + ); } #[test] diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 8f3efb3..6341d29 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -36,11 +36,7 @@ impl<'a> HtmlParser<'a> { } else { let body_sel = Selector::parse("body").unwrap(); let body: Vec<_> = dom.select(&body_sel).collect(); - if body.is_empty() { - vec![root] - } else { - body - } + if body.is_empty() { vec![root] } else { body } }; let exclude_selectors: Vec = self @@ -95,7 +91,10 @@ impl<'a> HtmlParser<'a> { if self.is_excluded(&el, exclude) { continue; } - if let Some(node) = self.convert_element(el, exclude, depth + 1) { + // Transparent containers: recurse directly into their children + if is_transparent_container(el.value().name()) { + self.convert_children(el, exclude, out, depth + 1); + } else if let Some(node) = self.convert_element(el, exclude, depth + 1) { out.push(node); } } @@ -167,10 +166,7 @@ impl<'a> HtmlParser<'a> { } "ol" => { - let start = el - .value() - .attr("start") - .and_then(|s| s.parse::().ok()); + let start = el.value().attr("start").and_then(|s| s.parse::().ok()); let items = self.collect_list_items(el, exclude, depth); if items.is_empty() { return None; @@ -278,9 +274,9 @@ impl<'a> HtmlParser<'a> { "hr" => Some(Node::ThematicBreak), // Skip non-content elements - "script" | "style" | "noscript" | "iframe" | "object" | "embed" | "form" - | "input" | "button" | "select" | "textarea" | "svg" | "canvas" | "video" - | "audio" | "source" | "track" | "map" | "area" => None, + "script" | "style" | "noscript" | "iframe" | "object" | "embed" | "form" | "input" + | "button" | "select" | "textarea" | "svg" | "canvas" | "video" | "audio" + | "source" | "track" | "map" | "area" => None, // Unknown element: try to extract inline content _ => { @@ -465,6 +461,14 @@ fn collapse_whitespace(s: &str) -> String { result } +/// Check if a tag is a transparent container (should be unwrapped in block context). +fn is_transparent_container(tag: &str) -> bool { + matches!( + tag, + "div" | "section" | "article" | "main" | "figure" | "figcaption" + ) +} + /// Check if a node is inline content. fn is_inline(node: &Node) -> bool { matches!( diff --git a/crates/markkit/src/ir.rs b/crates/markkit/src/ir.rs index 8770565..5fefd68 100644 --- a/crates/markkit/src/ir.rs +++ b/crates/markkit/src/ir.rs @@ -25,14 +25,9 @@ impl Document { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Node { /// Heading with level 1–6. - Heading { - level: u8, - children: Vec, - }, + Heading { level: u8, children: Vec }, /// Paragraph containing inline content. - Paragraph { - children: Vec, - }, + Paragraph { children: Vec }, /// Ordered or unordered list. List { ordered: bool, @@ -40,9 +35,7 @@ pub enum Node { items: Vec, }, /// Block quotation. - BlockQuote { - children: Vec, - }, + BlockQuote { children: Vec }, /// Fenced code block. CodeBlock { language: Option, @@ -58,17 +51,11 @@ pub enum Node { /// Plain text. Text(String), /// Bold / strong emphasis. - Bold { - children: Vec, - }, + Bold { children: Vec }, /// Italic / emphasis. - Italic { - children: Vec, - }, + Italic { children: Vec }, /// Strikethrough. - Strikethrough { - children: Vec, - }, + Strikethrough { children: Vec }, /// Inline code. InlineCode(String), /// Hyperlink. @@ -78,10 +65,7 @@ pub enum Node { children: Vec, }, /// Image. - Image { - src: String, - alt: String, - }, + Image { src: String, alt: String }, /// Hard line break. LineBreak, } diff --git a/crates/markkit/src/render/markdown.rs b/crates/markkit/src/render/markdown.rs index fc41c9d..0534f57 100644 --- a/crates/markkit/src/render/markdown.rs +++ b/crates/markkit/src/render/markdown.rs @@ -112,13 +112,7 @@ impl MarkdownRenderer { } /// Render a list. - fn render_list( - &mut self, - ordered: bool, - start: Option, - items: &[ListItem], - depth: usize, - ) { + fn render_list(&mut self, ordered: bool, start: Option, items: &[ListItem], depth: usize) { let indent = " ".repeat(depth); let start_num = start.unwrap_or(1); diff --git a/crates/markkit/tests/html_fixtures.rs b/crates/markkit/tests/html_fixtures.rs index 5b2d466..3d87016 100644 --- a/crates/markkit/tests/html_fixtures.rs +++ b/crates/markkit/tests/html_fixtures.rs @@ -7,10 +7,8 @@ use std::fs; use markkit::Markkit; fn fixture_test(name: &str) { - let html = - fs::read_to_string(format!("tests/fixtures/html/{}.html", name)).unwrap(); - let expected = - fs::read_to_string(format!("tests/fixtures/html/{}.md", name)).unwrap(); + let html = fs::read_to_string(format!("tests/fixtures/html/{}.html", name)).unwrap(); + let expected = fs::read_to_string(format!("tests/fixtures/html/{}.md", name)).unwrap(); let m = Markkit::builder().build(); let actual = m.extract_html_sync(&html).unwrap(); assert_eq!( diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index bd54d77..a3db669 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -146,10 +146,9 @@ fn link_with_inline_formatting() { #[test] fn link_preservation_disabled() { - let md = convert_with( - r#"

Example

"#, - |b| b.preserve_links(false), - ); + let md = convert_with(r#"

Example

"#, |b| { + b.preserve_links(false) + }); assert_eq!(md, "Example\n"); } @@ -198,9 +197,8 @@ fn ordered_list_custom_start() { #[test] fn nested_list() { - let md = convert( - "
  • Outer
    • Inner A
    • Inner B
  • Other
", - ); + let md = + convert("
  • Outer
    • Inner A
    • Inner B
  • Other
"); assert!(md.contains("- Outer")); assert!(md.contains(" - Inner A")); assert!(md.contains(" - Inner B")); @@ -223,9 +221,7 @@ fn blockquote_simple() { #[test] fn blockquote_nested() { - let md = convert( - "

Outer

Inner

", - ); + let md = convert("

Outer

Inner

"); assert!(md.contains("> Outer")); assert!(md.contains("> > Inner")); } @@ -496,3 +492,202 @@ fn idempotent_double_extract() { let md2 = convert(html); assert_eq!(md1, md2); } + +// ─── Tables Edge Cases ────────────────────────────────────────────── + +#[test] +fn table_no_thead() { + let html = r#" + + + + + +
NameValue
A1
"#; + let md = convert(html); + assert!(md.contains("| Name | Value |")); + assert!(md.contains("| --- | --- |")); + assert!(md.contains("| A | 1 |")); +} + +#[test] +fn table_headers_only() { + let html = r#" + + +
Col1Col2
"#; + let md = convert(html); + assert!(md.contains("| Col1 | Col2 |")); + assert!(md.contains("| --- | --- |")); +} + +#[test] +fn table_mismatched_columns() { + let html = r#" + + + + + +
ABC
12
"#; + let md = convert(html); + assert!(md.contains("| A | B | C |")); + assert!(md.contains("| --- | --- | --- |")); + // Row with fewer cells still renders (padded with empty cells) + assert!(md.contains("| 1 | 2 |")); +} + +#[test] +fn table_empty() { + let html = "
"; + let md = convert(html); + assert_eq!(md, ""); +} + +// ─── Lists Edge Cases ─────────────────────────────────────────────── + +#[test] +fn nested_list_three_levels() { + let html = "
  • L1
    • L2
      • L3
"; + let md = convert(html); + assert!(md.contains("- L1")); + assert!(md.contains(" - L2")); + assert!(md.contains(" - L3")); +} + +#[test] +fn ordered_list_default_start() { + let html = "
  1. Alpha
  2. Beta
  3. Gamma
"; + let md = convert(html); + assert!(md.contains("1. Alpha")); + assert!(md.contains("2. Beta")); + assert!(md.contains("3. Gamma")); +} + +// ─── Renderer Edge Cases ──────────────────────────────────────────── + +#[test] +fn link_empty_url() { + let md = convert(r#"

text

"#); + assert!(md.contains("[text]()")); +} + +#[test] +fn image_empty_src() { + let md = convert(r#"x"#); + assert!(md.contains("![x]()")); +} + +#[test] +fn code_block_with_backticks() { + let html = r#"
let x = `hello`;
"#; + let md = convert(html); + assert!(md.contains("```")); + assert!(md.contains("let x = `hello`;")); +} + +// ─── Config Tests ─────────────────────────────────────────────────── + +#[test] +fn config_max_processing_secs() { + let m = Markkit::builder().max_processing_secs(30).build(); + assert_eq!(m.config().max_processing_secs, 30); +} + +#[test] +fn config_multiple_include_selectors() { + let html = r#"
AAA
BBB
CCC
"#; + let md = convert_with(html, |b| b.include_selector(".a").include_selector(".b")); + assert!(md.contains("AAA")); + assert!(md.contains("BBB")); + assert!(!md.contains("CCC")); +} + +#[test] +fn config_include_and_exclude() { + let html = r#"

Keep

Remove

Also keep

"#; + let md = convert_with(html, |b| { + b.include_selector("article").exclude_selector(".ad") + }); + assert!(md.contains("Keep")); + assert!(md.contains("Also keep")); + assert!(!md.contains("Remove")); +} + +// ─── Format Detection ─────────────────────────────────────────────── + +#[test] +fn format_content_type_with_params() { + use markkit::format::Format; + assert_eq!( + Format::from_content_type("text/html; charset=utf-8; boundary=foo"), + Format::Html + ); +} + +#[test] +fn format_empty_extension() { + use markkit::format::Format; + assert_eq!(Format::from_extension(""), Format::Unknown); +} + +#[test] +fn format_mixed_case_extension() { + use markkit::format::Format; + assert_eq!(Format::from_extension("HtMl"), Format::Html); +} + +// ─── Extract Bytes ────────────────────────────────────────────────── + +#[tokio::test] +async fn extract_bytes_html() { + let m = Markkit::builder().build(); + let html = b"

Hello

World

"; + let md = m.extract_bytes(html, Some("text/html")).await.unwrap(); + assert!(md.contains("# Hello")); + assert!(md.contains("World")); +} + +#[tokio::test] +async fn extract_bytes_unknown_format() { + let m = Markkit::builder().build(); + let data = b"some random bytes"; + let result = m + .extract_bytes(data, Some("application/octet-stream")) + .await; + assert!(result.is_err()); +} + +#[tokio::test] +async fn extract_bytes_no_content_type() { + let m = Markkit::builder().build(); + let data = b"some random bytes"; + let result = m.extract_bytes(data, None).await; + assert!(result.is_err()); +} + +// ─── Parser Edge Cases ────────────────────────────────────────────── + +#[test] +fn html_no_body() { + let md = convert("

Title

Content

"); + assert!(md.contains("# Title")); + assert!(md.contains("Content")); +} + +#[test] +fn empty_list_items_skipped() { + let md = convert("
  • text
"); + assert!(md.contains("- text")); + // Empty list item should be skipped + let dash_count = md.matches("- ").count(); + assert_eq!(dash_count, 1, "Expected only one list item, got: {}", md); +} + +#[test] +fn div_with_multiple_blocks() { + let html = "

First

Second

"; + let md = convert(html); + assert!(md.contains("First")); + assert!(md.contains("Second")); +} diff --git a/crates/markkit/tests/html_security.rs b/crates/markkit/tests/html_security.rs index c68df29..508b068 100644 --- a/crates/markkit/tests/html_security.rs +++ b/crates/markkit/tests/html_security.rs @@ -159,3 +159,46 @@ fn null_bytes() { assert!(md.contains("before")); assert!(md.contains("after")); } + +// ─── Depth Limits ─────────────────────────────────────────────────── + +#[test] +fn depth_limit_200() { + // MAX_DEPTH is 200. With 200 nested divs, the inner

is at depth 200 + // (body=0, div1=1, ..., div200=200). At depth >= 200 the parser + // falls back to text extraction. The content should still appear. + let mut html = String::new(); + for _ in 0..200 { + html.push_str("

"); + } + html.push_str("

Deep content

"); + for _ in 0..200 { + html.push_str("
"); + } + let md = convert(&html); + assert!( + md.contains("Deep content"), + "Content at depth 200 should still be extracted as text: {:?}", + md + ); +} + +#[test] +fn depth_limit_exceeded() { + // 250 nested divs — well past MAX_DEPTH (200). + // Content should still be extracted (flattened to text), not lost. + let mut html = String::new(); + for _ in 0..250 { + html.push_str("
"); + } + html.push_str("

Very deep content

"); + for _ in 0..250 { + html.push_str("
"); + } + let md = convert(&html); + assert!( + md.contains("Very deep content"), + "Content past depth limit should be flattened to text: {:?}", + md + ); +} diff --git a/examples/ir_access.rs b/examples/ir_access.rs index e026500..fc9eb42 100644 --- a/examples/ir_access.rs +++ b/examples/ir_access.rs @@ -28,7 +28,9 @@ fn main() { } } println!("--- Node Counts ---"); - println!("Headings: {headings}, Paragraphs: {paragraphs}, Quotes: {quotes}, Code blocks: {code_blocks}"); + println!( + "Headings: {headings}, Paragraphs: {paragraphs}, Quotes: {quotes}, Code blocks: {code_blocks}" + ); // Serialize to JSON let json = serde_json::to_string_pretty(&doc).expect("json failed");