feat: HTML-to-Markdown pipeline via canonical IR

CHANGELOG.md

@@ -16,3 +16,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 - Node.js bindings (`markkit-js` crate)
 - Benchmark harness (`markkit-bench` crate)
 - LLM evaluation harness (`markkit-eval` crate)
+- **HTML → Markdown pipeline** via canonical IR (`ir.rs`, `html/parser.rs`, `render/markdown.rs`)
+  - Full element mapping: headings, paragraphs, lists (nested), blockquotes, code blocks, tables (GFM), inline formatting, links, images, thematic breaks, line breaks
+  - Configurable CSS include/exclude selectors
+  - Depth limiting (MAX_DEPTH=200) to prevent stack overflow
+  - Whitespace normalization with inter-element space preservation
+  - Non-content element stripping (script, style, iframe, form, svg, etc.)
+  - Resource limit enforcement (max input size)
+- **Canonical IR** with serde JSON serialization support
+- **Sync API**: `extract_html_sync()` and `parse_html()` for non-async usage
+- **CLI stdin support**: `--stdin` flag and `-` source convention
+- **Python bindings**: `extract_html_sync()` and `extract_html_sync_with_options()` with keyword args
+- **JS bindings**: `extract_html_sync()` synchronous function
+- **Specs**: IR specification, threat model, comprehensive HTML extraction spec, examples spec
+- **Examples**: `basic.rs`, `custom_selectors.rs`, `ir_access.rs`, `configuration.rs`
+- **Test fixtures**: 9 paired input/output HTML→Markdown fixture files

crates/markkit-cli/src/main.rs

@@ -1,3 +1,5 @@
+use std::io::Read;
+
 use anyhow::Result;
 use clap::{Parser, Subcommand};
 use markkit::Markkit;
@@ -17,13 +19,17 @@ struct Cli {
 enum Commands {
     /// Extract markdown from a URL or file
     Extract {
-        /// URL or file path to extract from
-        source: String,
+        /// URL or file path to extract from (use "-" to read from stdin)
+        source: Option<String>,
 
         /// Treat source as a file path instead of URL
         #[arg(long)]
         file: bool,
 
+        /// Read HTML from stdin
+        #[arg(long)]
+        stdin: bool,
+
         /// CSS selector to include (can be repeated)
         #[arg(long = "include", short = 'i')]
         include_selectors: Vec<String>,
@@ -50,11 +56,18 @@ async fn main() -> Result<()> {
         Commands::Extract {
             source,
             file,
+            stdin,
             include_selectors,
             exclude_selectors,
             max_size,
             json: output_json,
         } => {
+            let read_stdin = stdin || source.as_deref() == Some("-");
+
+            if !read_stdin && source.is_none() {
+                anyhow::bail!("source is required unless --stdin or \"-\" is used");
+            }
+
             let mut builder = Markkit::builder().max_input_size(max_size);
 
             for sel in &include_selectors {
@@ -66,9 +79,16 @@ async fn main() -> Result<()> {
 
             let markkit = builder.build();
 
-            let markdown = if file {
-                let bytes = tokio::fs::read(&source).await?;
-                let ext = std::path::Path::new(&source)
+            let source_label = source.clone().unwrap_or_else(|| "<stdin>".to_string());
+
+            let markdown = if read_stdin {
+                let mut buf = String::new();
+                std::io::stdin().read_to_string(&mut buf)?;
+                markkit.extract_html_sync(&buf)?
+            } else if file {
+                let src = source.as_deref().unwrap();
+                let bytes = tokio::fs::read(src).await?;
+                let ext = std::path::Path::new(src)
                     .extension()
                     .and_then(|e| e.to_str())
                     .unwrap_or("");
@@ -82,12 +102,13 @@ async fn main() -> Result<()> {
                 };
                 markkit.extract_bytes(&bytes, content_type).await?
             } else {
-                markkit.extract_url(&source).await?
+                let src = source.as_deref().unwrap();
+                markkit.extract_url(src).await?
             };
 
             if output_json {
                 let output = serde_json::json!({
-                    "source": source,
+                    "source": source_label,
                     "markdown": markdown,
                 });
                 println!("{}", serde_json::to_string_pretty(&output)?);

crates/markkit-cli/tests/cli_integration.rs

@@ -0,0 +1,201 @@
+use std::io::Write;
+use std::process::Command;
+
+fn markkit_bin() -> Command {
+    Command::new(env!("CARGO_BIN_EXE_markkit"))
+}
+
+#[test]
+fn help_flag_works() {
+    let output = markkit_bin()
+        .arg("--help")
+        .output()
+        .expect("failed to run markkit");
+    assert!(output.status.success());
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("Extract clean markdown"));
+}
+
+#[test]
+fn extract_help_works() {
+    let output = markkit_bin()
+        .args(["extract", "--help"])
+        .output()
+        .expect("failed to run markkit");
+    assert!(output.status.success());
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("--stdin"));
+}
+
+#[test]
+fn extract_from_stdin_flag() {
+    let mut child = markkit_bin()
+        .args(["extract", "--stdin"])
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .expect("failed to spawn markkit");
+
+    child
+        .stdin
+        .as_mut()
+        .unwrap()
+        .write_all(b"<h1>Hello World</h1><p>Some text here.</p>")
+        .unwrap();
+
+    let output = child.wait_with_output().unwrap();
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("Hello World"), "output: {stdout}");
+    assert!(stdout.contains("Some text here"), "output: {stdout}");
+}
+
+#[test]
+fn extract_from_stdin_dash() {
+    let mut child = markkit_bin()
+        .args(["extract", "-"])
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .expect("failed to spawn markkit");
+
+    child
+        .stdin
+        .as_mut()
+        .unwrap()
+        .write_all(b"<h1>Dash Test</h1>")
+        .unwrap();
+
+    let output = child.wait_with_output().unwrap();
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("Dash Test"), "output: {stdout}");
+}
+
+#[test]
+fn extract_from_file() {
+    let dir = tempfile::tempdir().unwrap();
+    let file_path = dir.path().join("test.html");
+    std::fs::write(&file_path, "<h1>File Test</h1><p>Paragraph.</p>").unwrap();
+
+    let output = markkit_bin()
+        .args(["extract", "--file", file_path.to_str().unwrap()])
+        .output()
+        .expect("failed to run markkit");
+
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("File Test"), "output: {stdout}");
+    assert!(stdout.contains("Paragraph"), "output: {stdout}");
+}
+
+#[test]
+fn extract_json_output() {
+    let mut child = markkit_bin()
+        .args(["extract", "--stdin", "--json"])
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .expect("failed to spawn markkit");
+
+    child
+        .stdin
+        .as_mut()
+        .unwrap()
+        .write_all(b"<h1>JSON Test</h1>")
+        .unwrap();
+
+    let output = child.wait_with_output().unwrap();
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("invalid JSON output");
+    assert_eq!(parsed["source"], "<stdin>");
+    assert!(parsed["markdown"].as_str().unwrap().contains("JSON Test"));
+}
+
+#[test]
+fn extract_with_include_selector() {
+    let mut child = markkit_bin()
+        .args(["extract", "--stdin", "-i", "article"])
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .expect("failed to spawn markkit");
+
+    child
+        .stdin
+        .as_mut()
+        .unwrap()
+        .write_all(b"<div>Ignore this</div><article><p>Keep this</p></article>")
+        .unwrap();
+
+    let output = child.wait_with_output().unwrap();
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("Keep this"), "output: {stdout}");
+}
+
+#[test]
+fn extract_with_exclude_selector() {
+    let mut child = markkit_bin()
+        .args(["extract", "--stdin", "-e", "nav"])
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()
+        .expect("failed to spawn markkit");
+
+    child
+        .stdin
+        .as_mut()
+        .unwrap()
+        .write_all(b"<nav>Menu</nav><p>Content</p>")
+        .unwrap();
+
+    let output = child.wait_with_output().unwrap();
+    assert!(
+        output.status.success(),
+        "stderr: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    assert!(stdout.contains("Content"), "output: {stdout}");
+    assert!(
+        !stdout.contains("Menu"),
+        "nav should be excluded, output: {stdout}"
+    );
+}
+
+#[test]
+fn missing_source_errors() {
+    let output = markkit_bin()
+        .args(["extract"])
+        .output()
+        .expect("failed to run markkit");
+
+    assert!(!output.status.success());
+}

crates/markkit-js/src/lib.rs

@@ -1,6 +1,6 @@
 use napi_derive::napi;
 
-/// Extract markdown from an HTML string.
+/// Extract markdown from an HTML string (async).
 #[napi]
 pub async fn extract_html(html: String) -> napi::Result<String> {
     let m = markkit::Markkit::builder().build();
@@ -9,6 +9,14 @@ pub async fn extract_html(html: String) -> napi::Result<String> {
         .map_err(|e| napi::Error::from_reason(e.to_string()))
 }
 
+/// Extract markdown from an HTML string (synchronous).
+#[napi]
+pub fn extract_html_sync(html: String) -> napi::Result<String> {
+    let m = markkit::Markkit::builder().build();
+    m.extract_html_sync(&html)
+        .map_err(|e| napi::Error::from_reason(e.to_string()))
+}
+
 /// Extract markdown from a URL.
 #[napi]
 pub async fn extract_url(url: String) -> napi::Result<String> {

crates/markkit-python/src/lib.rs

@@ -1,6 +1,6 @@
 use pyo3::prelude::*;
 
-/// Extract markdown from an HTML string.
+/// Extract markdown from an HTML string (async).
 #[pyfunction]
 fn extract_html(py: Python<'_>, html: String) -> PyResult<Bound<'_, PyAny>> {
     pyo3_async_runtimes::tokio::future_into_py(py, async move {
@@ -11,6 +11,47 @@ fn extract_html(py: Python<'_>, html: String) -> PyResult<Bound<'_, PyAny>> {
     })
 }
 
+/// Extract markdown from an HTML string (synchronous).
+#[pyfunction]
+fn extract_html_sync(html: String) -> PyResult<String> {
+    let m = markkit::Markkit::builder().build();
+    m.extract_html_sync(&html)
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
+}
+
+/// Extract markdown from an HTML string with configuration options (synchronous).
+///
+/// Keyword arguments:
+///   include_selectors: list of CSS selectors to include
+///   exclude_selectors: list of CSS selectors to exclude
+///   max_input_size: maximum input size in bytes
+#[pyfunction]
+#[pyo3(signature = (html, *, include_selectors=None, exclude_selectors=None, max_input_size=None))]
+fn extract_html_sync_with_options(
+    html: String,
+    include_selectors: Option<Vec<String>>,
+    exclude_selectors: Option<Vec<String>>,
+    max_input_size: Option<usize>,
+) -> PyResult<String> {
+    let mut builder = markkit::Markkit::builder();
+    if let Some(size) = max_input_size {
+        builder = builder.max_input_size(size);
+    }
+    if let Some(selectors) = include_selectors {
+        for sel in &selectors {
+            builder = builder.include_selector(sel);
+        }
+    }
+    if let Some(selectors) = exclude_selectors {
+        for sel in &selectors {
+            builder = builder.exclude_selector(sel);
+        }
+    }
+    let m = builder.build();
+    m.extract_html_sync(&html)
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
+}
+
 /// Extract markdown from a URL.
 #[pyfunction]
 fn extract_url(py: Python<'_>, url: String) -> PyResult<Bound<'_, PyAny>> {
@@ -40,6 +81,8 @@ fn extract_bytes(
 #[pymodule]
 fn markkit_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(extract_html, m)?)?;
+    m.add_function(wrap_pyfunction!(extract_html_sync, m)?)?;
+    m.add_function(wrap_pyfunction!(extract_html_sync_with_options, m)?)?;
     m.add_function(wrap_pyfunction!(extract_url, m)?)?;
     m.add_function(wrap_pyfunction!(extract_bytes, m)?)?;
     Ok(())