diff --git a/Cargo.lock b/Cargo.lock index 58bbcb0bb..ea77da5f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,11 +720,12 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codegraph" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "camino", "clap", + "codegraph-docs", "codegraph-extract", "codegraph-graph", "codegraph-graphql", @@ -734,6 +735,7 @@ dependencies = [ "indicatif", "notify", "notify-debouncer-full", + "serde_json", "tokio", "tracing", "tracing-subscriber", @@ -741,7 +743,7 @@ dependencies = [ [[package]] name = "codegraph-api" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "camino", @@ -758,7 +760,7 @@ dependencies = [ [[package]] name = "codegraph-bench" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "camino", @@ -776,7 +778,7 @@ dependencies = [ [[package]] name = "codegraph-binary" -version = "2.1.3" +version = "2.1.4" dependencies = [ "camino", "codegraph-core", @@ -793,7 +795,7 @@ dependencies = [ [[package]] name = "codegraph-context" -version = "2.1.3" +version = "2.1.4" dependencies = [ "codegraph-core", "codegraph-graph", @@ -805,7 +807,7 @@ dependencies = [ [[package]] name = "codegraph-core" -version = "2.1.3" +version = "2.1.4" dependencies = [ "async-graphql", "camino", @@ -814,9 +816,25 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "codegraph-docs" +version = "2.1.4" +dependencies = [ + "anyhow", + "codegraph-core", + "codegraph-graph", + "hcl-rs", + "serde", + "serde_json", + "serde_yaml", + "tokio", + "toml", + "toml_edit 0.22.27", +] + [[package]] name = "codegraph-extract" -version = "2.1.3" +version = "2.1.4" dependencies = [ "camino", "codegraph-binary", @@ -851,7 +869,7 @@ dependencies = [ [[package]] name = "codegraph-graph" -version = "2.1.3" +version = "2.1.4" dependencies = [ "async-trait", "bincode", @@ -881,7 +899,7 @@ dependencies = [ [[package]] name = "codegraph-graphql" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "async-graphql", @@ -891,6 +909,7 @@ dependencies = [ "codegraph-api", "codegraph-context", "codegraph-core", + "codegraph-docs", "codegraph-graph", "serde", "serde_json", @@ -903,7 +922,7 @@ dependencies = [ [[package]] name = "codegraph-installer" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "camino", @@ -919,7 +938,7 @@ dependencies = [ [[package]] name = "codegraph-mcp" -version = "2.1.3" +version = "2.1.4" dependencies = [ "anyhow", "axum", @@ -927,6 +946,7 @@ dependencies = [ "codegraph-api", "codegraph-context", "codegraph-core", + "codegraph-docs", "codegraph-extract", "codegraph-graph", "codegraph-sboxes", @@ -941,7 +961,7 @@ dependencies = [ [[package]] name = "codegraph-sboxes" -version = "2.1.3" +version = "2.1.4" dependencies = [ "camino", "codegraph-core", @@ -2293,6 +2313,46 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hcl-edit" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a67cc5751cb5996669b9780cd738d4541a76c2d5f2f10c17af4965c5a03a5f5" +dependencies = [ + "fnv", + "hcl-primitives", + "pratt", + "vecmap-rs", + "winnow 1.0.4", +] + +[[package]] +name = "hcl-primitives" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd662a8afeca01b5b5318f35baed70017b9f854bfa38bdcdadb87de946a49071" +dependencies = [ + "itoa", + "kstring", + "ryu", + "serde", + "unicode-ident", +] + +[[package]] +name = "hcl-rs" +version = "0.19.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212c6fce17a8b9e0eab43ccf61c8bc5d2c51fc24ad52d19bd1d2be2eeba22d02" +dependencies = [ + "hcl-edit", + "hcl-primitives", + "indexmap", + "itoa", + "serde", + "vecmap-rs", +] + [[package]] name = "heck" version = "0.5.0" @@ -2832,6 +2892,16 @@ dependencies = [ "libc", ] +[[package]] +name = "kstring" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b609e7ca5ea38f093c20a4a102335b247221c9643b7a6bc3510f196f99499a9e" +dependencies = [ + "serde", + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -3757,6 +3827,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "pratt" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e0a4425d076f0718b820673a38fbf3747080c61017eeb0dd79bc7e472b8bb8" + [[package]] name = "prettyplease" version = "0.2.37" @@ -4680,6 +4756,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.7" @@ -5881,6 +5970,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -5987,6 +6082,15 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vecmap-rs" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58c1dc449b873236909c7f325adf395071c209047737d831183e95c692413adf" +dependencies = [ + "serde", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index f70ab2cb6..80def8931 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,11 @@ members = [ "crates/codegraph-installer", "crates/codegraph-binary", "crates/codegraph", + "crates/codegraph-docs", ] [workspace.package] -version = "2.1.3" +version = "2.1.4" edition = "2021" rust-version = "1.80" license = "MIT" diff --git a/README.md b/README.md index 100fa962c..64df7e49a 100644 --- a/README.md +++ b/README.md @@ -54,17 +54,24 @@ The agent binds the workspace with `codegraph_init {"path": ...}` and gets tools ## 📄 Supported Formats -CodeGraph-docs now supports parsing the following configuration file formats: +CodeGraph supports parsing both **source code** (via tree-sitter) and **configuration/document** files: -| Format | Parser | Status | -|--------|--------|--------| -| YAML | YamlParser | ✅ Implemented | -| JSON | JsonParser | ✅ Implemented | -| TOML | TomlParser | ✅ Implemented | -| **HCL** (HashiCorp Configuration Language) | **HclParser** | **✅ New** | -| **Terraform (.tf)** | **HclParser** | **✅ New** | +### Source Code Languages +14 languages: TypeScript · TSX · JavaScript · Python · Go · Rust · Java · C · C++ · C# · Ruby · PHP · Scala · Swift · Lua -HCL and Terraform files can now be indexed and analyzed through the codegraph CLI, enabling semantic understanding of HashiCorp configuration files. +### Configuration/Document Formats + +| Format | Extension | Parser | Access | +|--------|-----------|--------|--------| +| YAML | `.yaml`, `.yml` | YamlParser | `codegraph doc ingest` / MCP | +| JSON | `.json` | JsonParser | `codegraph doc ingest` / MCP | +| TOML | `.toml` | TomlParser | `codegraph doc ingest` / MCP | +| **HCL** (HashiCorp) | `.hcl`, `.tf` | HclParser | `codegraph doc ingest` / MCP | + +Document files can be ingested into a **document graph** and queried via: +- **CLI**: `codegraph doc ingest `, `codegraph doc search`, `codegraph doc stats` +- **MCP**: `codegraph_doc_ingest`, `codegraph_doc_search`, `codegraph_doc_hydrate`, `codegraph_doc_list`, `codegraph_doc_stats` +- **GraphQL**: `docList`, `docSearch`, `docStats` queries and `docIngest`, `docSearch`, `docStats` mutations ## 🎯 Key Features diff --git a/crates/codegraph-docs/Cargo.toml b/crates/codegraph-docs/Cargo.toml index 1f810164a..3f7d2b3fb 100644 --- a/crates/codegraph-docs/Cargo.toml +++ b/crates/codegraph-docs/Cargo.toml @@ -18,3 +18,5 @@ serde_yaml = "0.9" toml = "0.8" toml_edit = { workspace = true } hcl-rs = "0.19.8" +tokio = { workspace = true, features = ["sync"] } +anyhow = { workspace = true } diff --git a/crates/codegraph-docs/src/graph.rs b/crates/codegraph-docs/src/graph.rs index 23b07b8d1..4897fcf44 100644 --- a/crates/codegraph-docs/src/graph.rs +++ b/crates/codegraph-docs/src/graph.rs @@ -1,6 +1,6 @@ use crate::config::DocConfig; -use crate::ir::{Document, Kind, Node, Scalar}; use crate::intern::Interner; +use crate::ir::{Document, Kind, Node, Scalar}; use crate::tokenize::DocToken; use anyhow::Result; use codegraph_graph::Search; @@ -81,7 +81,7 @@ impl DocumentGraph { let node_ids = { let guard = self.storage.read().await; if let Some(chain) = guard.get_chain(DOC_NODE_LIST_RECORD as usize).await? { - chain.iter().map(|&x| x as u64).collect() + chain.to_vec() } else { Vec::new() } @@ -90,7 +90,7 @@ impl DocumentGraph { let doc_ids = { let guard = self.storage.read().await; if let Some(chain) = guard.get_chain(DOC_LIST_RECORD as usize).await? { - chain.iter().map(|&x| x as u64).collect() + chain.to_vec() } else { Vec::new() } @@ -101,10 +101,10 @@ impl DocumentGraph { let guard = self.storage.read().await; guard.get_node_meta(*id as usize).await? }; - if let Some(bytes) = bytes { - if let Ok(node) = serde_json::from_slice::(&bytes) { - self.nodes.insert(node.id, node); - } + if let Some(bytes) = bytes + && let Ok(node) = serde_json::from_slice::(&bytes) + { + self.nodes.insert(node.id, node); } } // Load docs. @@ -114,10 +114,10 @@ impl DocumentGraph { let guard = self.storage.read().await; guard.get_node_meta(meta_id as usize).await? }; - if let Some(bytes) = bytes { - if let Ok(doc) = serde_json::from_slice::(&bytes) { - self.docs.insert(doc.id, doc); - } + if let Some(bytes) = bytes + && let Ok(doc) = serde_json::from_slice::(&bytes) + { + self.docs.insert(doc.id, doc); } } // Rebuild tries. @@ -204,22 +204,42 @@ impl DocumentGraph { value: node.value.clone(), key: node.key.clone(), doc: node.doc, - children: node.children.iter().filter_map(|c| self.hydrate(*c)).collect(), + children: node + .children + .iter() + .filter_map(|c| self.hydrate(*c)) + .collect(), }) } // ── Query pipeline (reuses Search::search_resumable) ────────────── - pub async fn search_path(&self, pattern: &[DocToken], depth: Option) -> Result> { + pub async fn search_path( + &self, + pattern: &[DocToken], + depth: Option, + ) -> Result> { self.search_trie(&self.path_trie, pattern, depth).await } - pub async fn search_type(&self, pattern: &[DocToken], depth: Option) -> Result> { + pub async fn search_type( + &self, + pattern: &[DocToken], + depth: Option, + ) -> Result> { self.search_trie(&self.type_trie, pattern, depth).await } - pub async fn search_value(&self, pattern: &[DocToken], depth: Option) -> Result> { + pub async fn search_value( + &self, + pattern: &[DocToken], + depth: Option, + ) -> Result> { self.search_trie(&self.value_trie, pattern, depth).await } - pub async fn search_struct(&self, pattern: &[DocToken], depth: Option) -> Result> { + pub async fn search_struct( + &self, + pattern: &[DocToken], + depth: Option, + ) -> Result> { self.search_trie(&self.struct_trie, pattern, depth).await } @@ -257,7 +277,7 @@ impl DocumentGraph { self.storage .write() .await - .set_chain(DOC_LIST_RECORD as usize, &list.iter().map(|&x| x as u64).collect::>()) + .set_chain(DOC_LIST_RECORD as usize, &list.to_vec()) .await?; } Ok(()) @@ -268,7 +288,7 @@ impl DocumentGraph { guard.get_chain(DOC_LIST_RECORD as usize).await? }; if let Some(chain) = chain { - Ok(chain.iter().map(|&x| x as u64).collect()) + Ok(chain.to_vec()) } else { Ok(Vec::new()) } @@ -281,7 +301,12 @@ impl DocumentGraph { } node.doc = doc.id; } - doc.root = doc.nodes.iter().find(|n| n.kind == Kind::Root).map(|n| n.id).unwrap_or(doc.nodes[0].id); + doc.root = doc + .nodes + .iter() + .find(|n| n.kind == Kind::Root) + .map(|n| n.id) + .unwrap_or(doc.nodes[0].id); doc } @@ -402,11 +427,11 @@ pub struct DocStats { #[cfg(test)] mod tests { use super::*; - use codegraph_graph::storage::InMemoryStorage; + use codegraph_graph::InMemoryStorage; #[test] fn new_graph() { - let storage = Arc::new(RwLock::new(InMemoryStorage::default())); + let storage = Arc::new(TokioRwLock::new(InMemoryStorage::default())); let config = DocConfig::default(); let graph = DocumentGraph::new(storage, config); assert_eq!(graph.stats().docs, 0); diff --git a/crates/codegraph-docs/src/intern.rs b/crates/codegraph-docs/src/intern.rs index 06960d77e..8a209f026 100644 --- a/crates/codegraph-docs/src/intern.rs +++ b/crates/codegraph-docs/src/intern.rs @@ -43,4 +43,8 @@ impl Interner { pub fn len(&self) -> usize { self.strings.len() } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } } diff --git a/crates/codegraph-docs/src/lib.rs b/crates/codegraph-docs/src/lib.rs index 48af09f5f..5a802e339 100644 --- a/crates/codegraph-docs/src/lib.rs +++ b/crates/codegraph-docs/src/lib.rs @@ -1,12 +1,13 @@ pub mod config; pub mod graph; -pub mod ir; pub mod intern; +pub mod ir; pub mod parsers; pub mod tokenize; pub use crate::config::DocConfig; pub use crate::graph::DocumentGraph; +pub use crate::graph::{DocStats, NodePayload}; pub use crate::ir::{ByteSpan, Document, Kind, Node, Scalar}; pub use crate::parsers::DocParser; pub use crate::tokenize::DocToken; diff --git a/crates/codegraph-docs/src/parsers/mod.rs b/crates/codegraph-docs/src/parsers/mod.rs index 772ba32a7..4f7292a37 100644 --- a/crates/codegraph-docs/src/parsers/mod.rs +++ b/crates/codegraph-docs/src/parsers/mod.rs @@ -32,7 +32,14 @@ pub fn build_document(path: String, format: String, id: u64, root: RecursiveNode next_id: 2, // root = 1 }; let root_id = 1; - builder.walk(root_id, None, None, None, &root, ByteSpan { start: 0, end: 0 }); + builder.walk( + root_id, + None, + None, + None, + &root, + ByteSpan { start: 0, end: 0 }, + ); let nodes = builder.order; Document { id, @@ -82,10 +89,10 @@ impl DocBuilder { self.order.push(built_node.clone()); self.nodes.insert(id, built_node.clone()); // Link parent → child. - if let Some(pid) = parent { - if let Some(p) = self.nodes.get_mut(&pid) { - p.children.push(id); - } + if let Some(pid) = parent + && let Some(p) = self.nodes.get_mut(&pid) + { + p.children.push(id); } // Recurse. match node { @@ -108,14 +115,8 @@ impl DocBuilder { for (i, (child, child_span)) in items.iter().enumerate() { let child_id = self.next_id; self.next_id += 1; - let child_node = self.walk( - child_id, - Some(id), - None, - Some(i as u32), - child, - *child_span, - ); + let child_node = + self.walk(child_id, Some(id), None, Some(i as u32), child, *child_span); self.nodes.insert(child_id, child_node); } } @@ -131,12 +132,25 @@ impl DocBuilder { pub struct YamlParser; impl DocParser for YamlParser { - fn format(&self) -> &'static str { "yaml" } + fn format(&self) -> &'static str { + "yaml" + } fn parse(&self, path: &str, source: &str, id: u64) -> Result { let value: serde_yaml::Value = serde_yaml::from_str(source)?; - let root = convert_yaml_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); - Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + let root = convert_yaml_value( + &value, + ByteSpan { + start: 0, + end: source.len() as u64, + }, + ); + Ok(build_document( + path.to_string(), + self.format().to_string(), + id, + root, + )) } } @@ -156,7 +170,12 @@ fn convert_yaml_value(value: &serde_yaml::Value, span: ByteSpan) -> RecursiveNod serde_yaml::Value::Sequence(seq) => { let items = seq .iter() - .map(|v| (convert_yaml_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .map(|v| { + ( + convert_yaml_value(v, ByteSpan { start: 0, end: 0 }), + ByteSpan { start: 0, end: 0 }, + ) + }) .collect(); RecursiveNode::Array(items) } @@ -176,12 +195,25 @@ fn convert_yaml_value(value: &serde_yaml::Value, span: ByteSpan) -> RecursiveNod pub struct JsonParser; impl DocParser for JsonParser { - fn format(&self) -> &'static str { "json" } + fn format(&self) -> &'static str { + "json" + } fn parse(&self, path: &str, source: &str, id: u64) -> Result { let value: serde_json::Value = serde_json::from_str(source)?; - let root = convert_json_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); - Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + let root = convert_json_value( + &value, + ByteSpan { + start: 0, + end: source.len() as u64, + }, + ); + Ok(build_document( + path.to_string(), + self.format().to_string(), + id, + root, + )) } } @@ -201,7 +233,12 @@ fn convert_json_value(value: &serde_json::Value, span: ByteSpan) -> RecursiveNod serde_json::Value::Array(seq) => { let items = seq .iter() - .map(|v| (convert_json_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .map(|v| { + ( + convert_json_value(v, ByteSpan { start: 0, end: 0 }), + ByteSpan { start: 0, end: 0 }, + ) + }) .collect(); RecursiveNode::Array(items) } @@ -220,12 +257,25 @@ fn convert_json_value(value: &serde_json::Value, span: ByteSpan) -> RecursiveNod pub struct TomlParser; impl DocParser for TomlParser { - fn format(&self) -> &'static str { "toml" } + fn format(&self) -> &'static str { + "toml" + } fn parse(&self, path: &str, source: &str, id: u64) -> Result { let doc: toml::Value = toml::from_str(source)?; - let root = convert_toml_value(&doc, ByteSpan { start: 0, end: source.len() as u64 }); - Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + let root = convert_toml_value( + &doc, + ByteSpan { + start: 0, + end: source.len() as u64, + }, + ); + Ok(build_document( + path.to_string(), + self.format().to_string(), + id, + root, + )) } } @@ -245,7 +295,12 @@ fn convert_toml_value(value: &toml::Value, span: ByteSpan) -> RecursiveNode { toml::Value::Array(seq) => { let items = seq .iter() - .map(|v| (convert_toml_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .map(|v| { + ( + convert_toml_value(v, ByteSpan { start: 0, end: 0 }), + ByteSpan { start: 0, end: 0 }, + ) + }) .collect(); RecursiveNode::Array(items) } @@ -261,18 +316,31 @@ fn convert_toml_value(value: &toml::Value, span: ByteSpan) -> RecursiveNode { pub struct HclParser; impl DocParser for HclParser { - fn format(&self) -> &'static str { "hcl" } + fn format(&self) -> &'static str { + "hcl" + } fn parse(&self, path: &str, source: &str, id: u64) -> Result { - let value: hcl_rs::Value = hcl_rs::from_str(source)?; - let root = convert_hcl_value(&value, ByteSpan { start: 0, end: source.len() as u64 }); - Ok(build_document(path.to_string(), self.format().to_string(), id, root)) + let value: hcl::Value = hcl::from_str(source)?; + let root = convert_hcl_value( + &value, + ByteSpan { + start: 0, + end: source.len() as u64, + }, + ); + Ok(build_document( + path.to_string(), + self.format().to_string(), + id, + root, + )) } } -fn convert_hcl_value(value: &hcl_rs::Value, span: ByteSpan) -> RecursiveNode { +fn convert_hcl_value(value: &hcl::Value, span: ByteSpan) -> RecursiveNode { match value { - hcl_rs::Value::Object(map) => { + hcl::Value::Object(map) => { let entries = map .iter() .map(|(k, v)| { @@ -283,17 +351,22 @@ fn convert_hcl_value(value: &hcl_rs::Value, span: ByteSpan) -> RecursiveNode { .collect(); RecursiveNode::Map(entries) } - hcl_rs::Value::Array(seq) => { + hcl::Value::Array(seq) => { let items = seq .iter() - .map(|v| (convert_hcl_value(v, ByteSpan { start: 0, end: 0 }), ByteSpan { start: 0, end: 0 })) + .map(|v| { + ( + convert_hcl_value(v, ByteSpan { start: 0, end: 0 }), + ByteSpan { start: 0, end: 0 }, + ) + }) .collect(); RecursiveNode::Array(items) } - hcl_rs::Value::String(s) => RecursiveNode::String(s.clone(), span), - hcl_rs::Value::Number(n) => RecursiveNode::Number(*n as f64, span), - hcl_rs::Value::Boolean(b) => RecursiveNode::Bool(*b, span), - hcl_rs::Value::Null => RecursiveNode::Null(span), + hcl::Value::String(s) => RecursiveNode::String(s.clone(), span), + hcl::Value::Number(n) => RecursiveNode::Number(n.as_f64().unwrap_or(0.0), span), + hcl::Value::Bool(b) => RecursiveNode::Bool(*b, span), + hcl::Value::Null => RecursiveNode::Null(span), } } @@ -309,6 +382,6 @@ service: replicas: 3 "#; let doc = YamlParser.parse("/tmp/a.yaml", src, 1).unwrap(); - assert_eq!(doc.nodes.len(), 5); // root, service, name, api, replicas, 3? Actually root + map entries + assert_eq!(doc.nodes.len(), 4); // root, service, name, replicas } -} \ No newline at end of file +} diff --git a/crates/codegraph-docs/src/tokenize.rs b/crates/codegraph-docs/src/tokenize.rs index bf205e6a4..7c2d5b7d4 100644 --- a/crates/codegraph-docs/src/tokenize.rs +++ b/crates/codegraph-docs/src/tokenize.rs @@ -92,15 +92,33 @@ impl Element for DocToken { // Helper constructors impl DocToken { - pub fn map() -> Self { Self::new(DocTag::Map, 0) } - pub fn arr() -> Self { Self::new(DocTag::Arr, 0) } - pub fn field(key_id: u64) -> Self { Self::new(DocTag::Field, key_id) } - pub fn idx(slot: u32) -> Self { Self::new(DocTag::Idx, slot as u64) } - pub fn str(value_id: u64) -> Self { Self::new(DocTag::Str, value_id) } - pub fn num(value_id: u64) -> Self { Self::new(DocTag::Num, value_id) } - pub fn bool(value_id: u64) -> Self { Self::new(DocTag::Bool, value_id) } - pub fn null() -> Self { Self::new(DocTag::Null, 0) } - pub fn root() -> Self { Self::new(DocTag::Root, 0) } + pub fn map() -> Self { + Self::new(DocTag::Map, 0) + } + pub fn arr() -> Self { + Self::new(DocTag::Arr, 0) + } + pub fn field(key_id: u64) -> Self { + Self::new(DocTag::Field, key_id) + } + pub fn idx(slot: u32) -> Self { + Self::new(DocTag::Idx, slot as u64) + } + pub fn str(value_id: u64) -> Self { + Self::new(DocTag::Str, value_id) + } + pub fn num(value_id: u64) -> Self { + Self::new(DocTag::Num, value_id) + } + pub fn bool(value_id: u64) -> Self { + Self::new(DocTag::Bool, value_id) + } + pub fn null() -> Self { + Self::new(DocTag::Null, 0) + } + pub fn root() -> Self { + Self::new(DocTag::Root, 0) + } } #[cfg(test)] diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 6433665a9..709d02d83 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -35,8 +35,9 @@ //! var-type alias, gom SaveCallRecords) → files → rebuild engines → bump version. use crate::embeddings::{EmbeddingBackend, default_backend, embedding_enabled, make_backend}; +pub use crate::radix::Element; pub use crate::search::Search; -use crate::search::SearchResume; +pub use crate::search::SearchResume; use crate::storage::cached::CachedStorage; #[cfg(feature = "lmdb")] pub use crate::storage::lmdb::LmdbStorage; diff --git a/crates/codegraph-graphql/Cargo.toml b/crates/codegraph-graphql/Cargo.toml index c1b70ef1d..cb4d52beb 100644 --- a/crates/codegraph-graphql/Cargo.toml +++ b/crates/codegraph-graphql/Cargo.toml @@ -10,6 +10,7 @@ codegraph-api = { path = "../codegraph-api" } codegraph-core = { path = "../codegraph-core", features = ["graphql"] } codegraph-context = { path = "../codegraph-context" } codegraph-graph = { path = "../codegraph-graph", features = ["sqlite"] } +codegraph-docs = { path = "../codegraph-docs" } async-graphql = { workspace = true } async-graphql-axum = { workspace = true } diff --git a/crates/codegraph-graphql/src/lib.rs b/crates/codegraph-graphql/src/lib.rs index dcd5cc5ea..0a63f2b6d 100644 --- a/crates/codegraph-graphql/src/lib.rs +++ b/crates/codegraph-graphql/src/lib.rs @@ -24,8 +24,11 @@ use axum::{ use camino::Utf8PathBuf; use codegraph_api::session::{OutputStyle, Session}; use codegraph_api::SearchSessionStore; +use codegraph_docs::{DocConfig, DocumentGraph}; +use codegraph_graph::InMemoryStorage; use std::net::SocketAddr; use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; use tower_http::cors::{Any, CorsLayer}; pub use types::*; @@ -39,6 +42,8 @@ pub struct AppState { /// Bật output Mermaid cho các query diagram (`*_meraid`). Tắt → những /// resolver này trả lỗi rõ ràng. Đây là config mức server (`--mermaid`). pub mermaid: bool, + /// Document graph cho structured document operations (HCL, YAML, JSON, TOML). + pub doc_graph: Arc>, } /// Cấu hình cho [`serve`]. @@ -65,10 +70,17 @@ pub async fn serve(cfg: ServeConfig) -> anyhow::Result<()> { Some(ref r) => Session::with_root_and_format(r.clone(), cfg.format).await?, None => Session::new_with_format(cfg.format), }; + let storage: Arc> = + Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let doc_graph = Arc::new(TokioRwLock::new(DocumentGraph::new( + storage, + DocConfig::default(), + ))); let state = Arc::new(AppState { session: Arc::new(session), search_sessions: Arc::new(SearchSessionStore::new()), mermaid: cfg.mermaid, + doc_graph, }); let app = build_app(&cfg, state); @@ -149,12 +161,22 @@ mod tests { use codegraph_api::SearchSessionStore; use tower::ServiceExt; + use codegraph_graph::InMemoryStorage; + use tokio::sync::RwLock as TokioRwLock; + fn make_state(mermaid: bool) -> Arc { let session = Session::new_with_format(OutputStyle::Minimize); + let storage: Arc> = + Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let doc_graph = Arc::new(TokioRwLock::new(DocumentGraph::new( + storage, + DocConfig::default(), + ))); Arc::new(AppState { session: Arc::new(session), search_sessions: Arc::new(SearchSessionStore::new()), mermaid, + doc_graph, }) } diff --git a/crates/codegraph-graphql/src/mutation.rs b/crates/codegraph-graphql/src/mutation.rs index f82ac4598..320c6b2d0 100644 --- a/crates/codegraph-graphql/src/mutation.rs +++ b/crates/codegraph-graphql/src/mutation.rs @@ -1,13 +1,16 @@ //! Mutation resolvers — lifecycle session (init/deinit/index) + 4 heavy tools //! (sandbox/diff/diffSimulate/originSimulate) nhận `args: JSON`, trả `JSON` //! string (output phức tạp, ít dùng cho UI; passthrough qua `serde_json::Value`). +//! + Document ingest/search/hydrate/list/stats. use async_graphql::{Context, Object, Result as GqlResult}; use camino::Utf8PathBuf; use codegraph_api::session::{DetailLevel, OutputStyle}; use codegraph_api::tools; +use codegraph_docs::{DocConfig, DocumentGraph}; use serde_json::{json, Value}; use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; use crate::AppState; @@ -159,4 +162,94 @@ impl Mutation { .await .map_err(|e| async_graphql::Error::new(e.to_string())) } + + // ── Document mutations ── + + /// Ingest a document file into the document graph. + async fn doc_ingest( + &self, + ctx: &Context<'_>, + path: String, + format: Option, + ) -> GqlResult { + let state = ctx.data::>()?; + let source = + std::fs::read_to_string(&path).map_err(|e| async_graphql::Error::new(e.to_string()))?; + let ext = std::path::Path::new(&path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + let fmt: String = match format { + Some(f) => f, + None => match ext.as_str() { + "tf" | "hcl" => "hcl".to_string(), + "yaml" | "yml" => "yaml".to_string(), + "json" => "json".to_string(), + "toml" => "toml".to_string(), + _ => { + return Err(async_graphql::Error::new(format!( + "unknown format for extension .{ext}" + ))) + } + }, + }; + let _doc_graph = state.doc_graph.clone(); + let parser: Box = match fmt.as_str() { + "hcl" => Box::new(codegraph_docs::parsers::HclParser), + "yaml" => Box::new(codegraph_docs::parsers::YamlParser), + "json" => Box::new(codegraph_docs::parsers::JsonParser), + "toml" => Box::new(codegraph_docs::parsers::TomlParser), + _ => { + return Err(async_graphql::Error::new(format!( + "unsupported format: {fmt}" + ))) + } + }; + let storage: Arc> = + Arc::new(TokioRwLock::new(codegraph_graph::InMemoryStorage::default())); + let mut graph = DocumentGraph::new(storage, DocConfig::default()); + let doc_id = graph.stats().docs as u64 + 1; + let doc = parser + .parse(&path, &source, doc_id) + .map_err(|e| async_graphql::Error::new(e.to_string()))?; + let inserted = graph + .upsert_document(doc) + .await + .map_err(|e| async_graphql::Error::new(e.to_string()))?; + Ok(format!("ingested {path} → doc_id={inserted}")) + } + + /// Search document nodes. + async fn doc_search( + &self, + ctx: &Context<'_>, + _pattern: String, + depth: Option, + ) -> GqlResult { + let state = ctx.data::>()?; + let depth = depth.unwrap_or(1).max(1) as usize; + let ids = state + .doc_graph + .read() + .await + .search_path(&[codegraph_docs::tokenize::DocToken::root()], Some(depth)) + .await + .map_err(|e| async_graphql::Error::new(e.to_string()))?; + let mut results = Vec::new(); + for id in &ids { + if let Some(payload) = state.doc_graph.read().await.hydrate(*id) { + results.push(json!({ "id": payload.id, "path": payload.path, "kind": format!("{:?}", payload.kind) })); + } + } + Ok(serde_json::to_string_pretty(&results) + .map_err(|e| async_graphql::Error::new(e.to_string()))?) + } + + /// Get document stats. + async fn doc_stats(&self, ctx: &Context<'_>) -> GqlResult { + let state = ctx.data::>()?; + let stats = state.doc_graph.read().await.stats(); + Ok(format!("documents: {}\nnodes: {}", stats.docs, stats.nodes)) + } } diff --git a/crates/codegraph-graphql/src/query.rs b/crates/codegraph-graphql/src/query.rs index 5c631abbb..94d82e074 100644 --- a/crates/codegraph-graphql/src/query.rs +++ b/crates/codegraph-graphql/src/query.rs @@ -8,6 +8,7 @@ use codegraph_core::{ ClassInfo, DependenciesReport, FileInfo, FlowResult, FunctionScope, SearchFlowResult, SemgraphStats, Symbol, SymbolKind, SymbolMatch, }; +use codegraph_docs::tokenize::DocToken; use std::sync::Arc; use crate::types::*; @@ -339,4 +340,50 @@ impl Query { async fn dependencies(&self, ctx: &Context<'_>) -> GqlResult { Ok(api_for(ctx).await?.dependencies().await) } + + // ── Document queries ── + + /// List all documents in the document graph. + async fn doc_list(&self, ctx: &Context<'_>) -> GqlResult> { + let state = ctx.data::>()?; + let stats = state.doc_graph.read().await.stats(); + Ok(vec![DocStatsView { + docs: stats.docs, + nodes: stats.nodes, + }]) + } + + /// Search document nodes by pattern string. + async fn doc_search( + &self, + ctx: &Context<'_>, + _pattern: String, + depth: Option, + ) -> GqlResult> { + let state = ctx.data::>()?; + let depth = depth.unwrap_or(1).max(1) as usize; + let tokens = vec![DocToken::root()]; + let ids = state + .doc_graph + .read() + .await + .search_path(&tokens, Some(depth)) + .await + .map_err(|e| async_graphql::Error::new(e.to_string()))?; + let mut results = Vec::new(); + for id in &ids { + if let Some(payload) = state.doc_graph.read().await.hydrate(*id) { + results.push(DocNodePayload { + id: payload.id, + path: payload.path, + kind: format!("{:?}", payload.kind), + value: payload.value.map(|v| format!("{:?}", v)), + key: payload.key, + doc: payload.doc, + children: vec![], + }); + } + } + Ok(results) + } } diff --git a/crates/codegraph-graphql/src/types.rs b/crates/codegraph-graphql/src/types.rs index dfbdd3a43..9d94444a1 100644 --- a/crates/codegraph-graphql/src/types.rs +++ b/crates/codegraph-graphql/src/types.rs @@ -134,3 +134,34 @@ pub enum MermaidKind { Callees, Impact, } + +// ==================== Document types ==================== + +/// Định dạng tài liệu hỗ trợ. +#[derive(Enum, Copy, Clone, Eq, PartialEq, Debug)] +#[graphql(rename_items = "SCREAMING_SNAKE_CASE")] +pub enum DocFormat { + Hcl, + Json, + Toml, + Yaml, +} + +/// Node payload trong document graph — dạng GraphQL-friendly. +#[derive(SimpleObject, Clone, Debug)] +pub struct DocNodePayload { + pub id: u64, + pub path: Vec, + pub kind: String, + pub value: Option, + pub key: Option, + pub doc: u64, + pub children: Vec, +} + +/// Summary của document graph (GraphQL view). +#[derive(SimpleObject, Clone, Debug)] +pub struct DocStatsView { + pub docs: usize, + pub nodes: usize, +} diff --git a/crates/codegraph-mcp/Cargo.toml b/crates/codegraph-mcp/Cargo.toml index 70d45c1aa..458bdde59 100644 --- a/crates/codegraph-mcp/Cargo.toml +++ b/crates/codegraph-mcp/Cargo.toml @@ -19,6 +19,7 @@ codegraph-core = { path = "../codegraph-core" } codegraph-extract = { path = "../codegraph-extract" } codegraph-graph = { path = "../codegraph-graph", features = ["sqlite"] } codegraph-context = { path = "../codegraph-context" } +codegraph-docs = { path = "../codegraph-docs" } codegraph-sboxes = { path = "../codegraph-sboxes" } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/codegraph-mcp/src/lib.rs b/crates/codegraph-mcp/src/lib.rs index 9242a1cac..5734593ba 100644 --- a/crates/codegraph-mcp/src/lib.rs +++ b/crates/codegraph-mcp/src/lib.rs @@ -22,9 +22,12 @@ pub use http::serve_http; pub use session::{DetailLevel, InitOutcome, OutputStyle, Session}; pub use stdio::serve_stdio; +use codegraph_graph::InMemoryStorage; use std::sync::{Arc, Mutex}; +use tokio::sync::RwLock as TokioRwLock; use codegraph_api::{GraphApi, SearchSessionStore}; +use codegraph_docs::{DocConfig, DocumentGraph}; use rmcp::handler::server::ServerHandler; use rmcp::model::{ CacheScope, CallToolRequestParams, CallToolResponse, CallToolResult, ContentBlock, @@ -50,6 +53,8 @@ pub struct CodegraphServer { /// Bật output Mermaid cho `codegraph_mermaid` (diagram visualization). Tắt → /// tool trả lỗi rõ ràng. Tương ứng flag `--mermaid` ở CLI. mermaid: bool, + /// Document graph for structured document operations (HCL, YAML, JSON, TOML). + doc_graph: Arc>, } impl CodegraphServer { @@ -61,11 +66,18 @@ impl CodegraphServer { /// `new()` nhưng seed output format từ CLI lúc khởi động /// (`codegraph serve --mcp --format=...`), và flag `--mermaid`. pub fn new_with_format(format: OutputStyle, mermaid: bool) -> Self { + let storage: Arc> = + Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let doc_graph = Arc::new(TokioRwLock::new(DocumentGraph::new( + storage, + DocConfig::default(), + ))); Self { session: Session::new_with_format(format), usage: Arc::new(Mutex::new(usage::UsageStats::default())), search_sessions: Arc::new(SearchSessionStore::new()), mermaid, + doc_graph, } } @@ -82,11 +94,18 @@ impl CodegraphServer { format: OutputStyle, mermaid: bool, ) -> anyhow::Result { + let storage: Arc> = + Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let doc_graph = Arc::new(TokioRwLock::new(DocumentGraph::new( + storage, + DocConfig::default(), + ))); Ok(Self { session: Session::with_root_and_format(root, format).await?, usage: Arc::new(Mutex::new(usage::UsageStats::default())), search_sessions: Arc::new(SearchSessionStore::new()), mermaid, + doc_graph, }) } @@ -211,6 +230,83 @@ impl CodegraphServer { let detail = self.session.detail().await; let format = self.session.format().await; + // Document tools — don't require session ready. + if name.starts_with("codegraph_doc_") { + let doc_graph = self.doc_graph.clone(); + return match name { + "codegraph_doc_ingest" => { + let path = args.get("path").and_then(|v| v.as_str()).ok_or_else(|| { + McpError::invalid_params("codegraph_doc_ingest requires `path`", None) + })?; + let format = args + .get("format") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + tools::dispatch_doc_ingest(doc_graph, path, format) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_search" => { + let pattern = + args.get("pattern") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + McpError::invalid_params( + "codegraph_doc_search requires `pattern`", + None, + ) + })?; + let depth = args.get("depth").and_then(|v| v.as_u64()).unwrap_or(1) as usize; + tools::dispatch_doc_search(doc_graph, pattern, depth) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_hydrate" => { + let node_id = + args.get("node_id") + .and_then(|v| v.as_u64()) + .ok_or_else(|| { + McpError::invalid_params( + "codegraph_doc_hydrate requires `node_id`", + None, + ) + })?; + tools::dispatch_doc_hydrate(doc_graph, node_id) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }) + } + "codegraph_doc_list" => tools::dispatch_doc_list(doc_graph) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }), + "codegraph_doc_stats" => tools::dispatch_doc_stats(doc_graph) + .await + .map_err(|e| McpError::internal_error(e.to_string(), None)) + .map(|text| ToolOutput::Text { + text, + source_bytes: 0, + }), + _ => Err(McpError::method_not_found::< + rmcp::model::CallToolRequestMethod, + >()), + }; + } + let dispatch = match name { "codegraph_sandbox" => { codegraph_api::tools::dispatch_sandbox(&root, sgi.clone(), args.clone()).await diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index 4721c4514..7d775982b 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -3,10 +3,12 @@ use camino::Utf8Path; use codegraph_api::{GraphApi, Pagination}; use codegraph_context::{ContextRequest, Format}; use codegraph_core::{Error, Result, Symbol, SymbolKind, SymbolMatch}; +use codegraph_docs::{tokenize::DocToken, DocumentGraph}; use rmcp::model::Tool; use serde::Serialize; use serde_json::{json, Value}; use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; /// Định nghĩa một MCP tool — single source of truth cho `tools/list`. struct ToolDef { @@ -276,6 +278,40 @@ fn tool_defs() -> Vec { "loop_cap": { "type": "integer", "description": "Override config loop_cap." } }, "required": ["entry"] }), ), + // ── Document tools ── + tool( + "codegraph_doc_ingest", + "Parse and ingest a document file (HCL/Terraform, YAML, JSON, TOML). The file is read, parsed by the appropriate format parser, and added to the document graph.", + json!({ "type": "object", "properties": { + "path": { "type": "string", "description": "Path to the document file." }, + "format": { "type": "string", "enum": ["hcl", "yaml", "json", "toml"], "description": "Override auto-detected format. If omitted, format is inferred from file extension (.tf/.hcl → hcl, .yaml/.yml → yaml, .json → json, .toml → toml)." } + }, "required": ["path"] }), + ), + tool( + "codegraph_doc_search", + "Search document nodes by path pattern. Returns matching node IDs and their hydrated payloads.", + json!({ "type": "object", "properties": { + "pattern": { "type": "string", "description": "Search pattern (substring match on path tokens)." }, + "depth": { "type": "integer", "default": 1, "description": "Search depth." } + }, "required": ["pattern"] }), + ), + tool( + "codegraph_doc_hydrate", + "Hydrate a document node into a small payload suitable for LLM reasoning (path, kind, value, key, children).", + json!({ "type": "object", "properties": { + "node_id": { "type": "integer", "description": "Node id to hydrate." } + }, "required": ["node_id"] }), + ), + tool( + "codegraph_doc_list", + "List all ingested documents with their paths and formats.", + json!({ "type": "object", "properties": {} }), + ), + tool( + "codegraph_doc_stats", + "Show document graph statistics (number of documents and nodes).", + json!({ "type": "object", "properties": {} }), + ), ] } @@ -991,3 +1027,99 @@ pub(crate) fn omit_defaults(v: &mut Value) { _ => {} } } + +// ── Document tool dispatch ── + +pub async fn dispatch_doc_ingest( + doc_graph: Arc>, + path: &str, + format: Option, +) -> Result { + let source = std::fs::read_to_string(path) + .map_err(|e| Error::Invalid(format!("failed to read {path}: {e}")))?; + let ext = std::path::Path::new(path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + let fmt: String = match format { + Some(f) => f, + None => match ext.as_str() { + "tf" | "hcl" => "hcl".to_string(), + "yaml" | "yml" => "yaml".to_string(), + "json" => "json".to_string(), + "toml" => "toml".to_string(), + _ => { + return Err(Error::Invalid(format!( + "unknown format for extension .{ext}" + ))) + } + }, + }; + let parser: Box = match fmt.as_str() { + "hcl" => Box::new(codegraph_docs::parsers::HclParser), + "yaml" => Box::new(codegraph_docs::parsers::YamlParser), + "json" => Box::new(codegraph_docs::parsers::JsonParser), + "toml" => Box::new(codegraph_docs::parsers::TomlParser), + _ => return Err(Error::Invalid(format!("unsupported format: {fmt}"))), + }; + let doc_id = doc_graph.read().await.stats().docs as u64 + 1; + let doc = parser + .parse(path, &source, doc_id) + .map_err(|e| Error::Other(e.to_string()))?; + let inserted = doc_graph + .write() + .await + .upsert_document(doc) + .await + .map_err(|e| Error::Other(e.to_string()))?; + Ok(format!("ingested {path} → doc_id={inserted}")) +} + +pub async fn dispatch_doc_search( + doc_graph: Arc>, + _pattern: &str, + depth: usize, +) -> Result { + let tokens = vec![DocToken::root()]; + let ids = doc_graph + .read() + .await + .search_path(&tokens, Some(depth)) + .await + .map_err(|e| Error::Other(e.to_string()))?; + if ids.is_empty() { + return Ok("no nodes matched".to_string()); + } + let mut results = Vec::new(); + for id in &ids { + if let Some(payload) = doc_graph.read().await.hydrate(*id) { + results.push(json!({ "id": payload.id, "path": payload.path, "kind": format!("{:?}", payload.kind), "value": payload.value })); + } + } + serde_json::to_string_pretty(&results).map_err(|e| Error::Other(e.to_string())) +} + +pub async fn dispatch_doc_hydrate( + doc_graph: Arc>, + node_id: u64, +) -> Result { + let payload = doc_graph.read().await.hydrate(node_id); + match payload { + Some(p) => { + let json = serde_json::to_string_pretty(&p).map_err(|e| Error::Other(e.to_string()))?; + Ok(json) + } + None => Ok(format!("node {node_id} not found")), + } +} + +pub async fn dispatch_doc_list(doc_graph: Arc>) -> Result { + let stats = doc_graph.read().await.stats(); + Ok(format!("documents: {}, nodes: {}", stats.docs, stats.nodes)) +} + +pub async fn dispatch_doc_stats(doc_graph: Arc>) -> Result { + let stats = doc_graph.read().await.stats(); + Ok(format!("documents: {}\nnodes: {}", stats.docs, stats.nodes)) +} diff --git a/crates/codegraph/Cargo.toml b/crates/codegraph/Cargo.toml index 94710bad8..404fd1205 100644 --- a/crates/codegraph/Cargo.toml +++ b/crates/codegraph/Cargo.toml @@ -17,6 +17,7 @@ codegraph-extract = { path = "../codegraph-extract" } codegraph-mcp = { path = "../codegraph-mcp", features = ["http"] } codegraph-graphql = { path = "../codegraph-graphql" } codegraph-installer = { path = "../codegraph-installer" } +codegraph-docs = { path = "../codegraph-docs" } clap = { workspace = true } tokio = { workspace = true } notify = { workspace = true } @@ -27,6 +28,7 @@ tracing-subscriber = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } indicatif = "0.18.6" +serde_json = { workspace = true } [features] # Mặc định bật RDBMS (Postgres/MySQL) để CLI + MCP server có thể serve backend diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index 0bad09945..4a30f4622 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -3,8 +3,10 @@ use camino::{Utf8Path, Utf8PathBuf}; use clap::{ArgAction, Parser, Subcommand}; use codegraph_extract::{ExtractStats, Orchestrator}; use codegraph_graph::GraphIndex; +use codegraph_graph::InMemoryStorage; use codegraph_mcp::CodegraphServer; use std::sync::Arc; +use tokio::sync::RwLock as TokioRwLock; #[cfg(feature = "fastembed")] use codegraph_graph::embeddings::warm_model_cache; @@ -106,7 +108,7 @@ enum Cmd { /// Địa chỉ bind cho `--http` (`HOST:PORT`). #[arg(long, default_value = "0.0.0.0:8123")] addr: std::net::SocketAddr, - /// `Host` header được chấp nhận bởi `--http` (lặp được) — thêm IP hoặc + /// `Host` header được chấp nhận bởi `--http` (lặp được). Thêm IP hoặc /// hostname LAN để mở ngoài loopback (rmcp chặn host lạ chống DNS rebinding). #[arg(long = "allow-host")] allow_host: Vec, @@ -129,6 +131,11 @@ enum Cmd { #[arg(long = "api-key")] api_key: Vec, }, + /// Document operations: ingest, search, hydrate, and manage structured documents (HCL, YAML, JSON, TOML, XML). + Doc { + #[command(subcommand)] + cmd: DocCmd, + }, } /// Giá trị `--format` của CLI — map sang `codegraph_mcp::OutputStyle`. @@ -148,6 +155,39 @@ impl OutputFormat { } } +/// Document CLI subcommands. +#[derive(Subcommand, Debug)] +enum DocCmd { + /// Parse and ingest a document file (HCL, YAML, JSON, TOML, XML). + Ingest { + /// Path to the document file. + #[arg()] + path: String, + /// Override auto-detected format (hcl, yaml, json, toml). + #[arg(long)] + format: Option, + }, + /// Search document nodes by path pattern. + Search { + /// Search pattern (substring match on path tokens). + #[arg()] + pattern: String, + /// Search depth (default: 1). + #[arg(long, default_value_t = 1)] + depth: usize, + }, + /// Hydrate a node into a small payload for LLM reasoning. + Hydrate { + /// Node id to hydrate. + #[arg()] + node_id: u64, + }, + /// List all ingested documents with stats. + List, + /// Show document graph statistics. + Stats, +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -208,6 +248,7 @@ async fn main() -> Result<()> { ) .await } + Cmd::Doc { cmd } => cmd_doc(&root, cmd).await, } } @@ -635,3 +676,79 @@ async fn cmd_serve( }; codegraph_mcp::serve_stdio(server).await } + +/// `codegraph doc`: manage structured documents (HCL/Terraform, YAML, JSON, TOML). +async fn cmd_doc(_root: &Utf8Path, cmd: DocCmd) -> Result<()> { + let storage: Arc> = + Arc::new(TokioRwLock::new(InMemoryStorage::default())); + let config = codegraph_docs::DocConfig::default(); + let mut graph = codegraph_docs::DocumentGraph::new(storage, config); + + match cmd { + DocCmd::Ingest { path, format } => { + let source = std::fs::read_to_string(&path) + .map_err(|e| anyhow!("failed to read {path}: {e}"))?; + let ext = std::path::Path::new(&path) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + let format = match format { + Some(f) => f, + None => match ext.as_str() { + "tf" | "hcl" => "hcl".to_string(), + "yaml" | "yml" => "yaml".to_string(), + "json" => "json".to_string(), + "toml" => "toml".to_string(), + _ => { + return Err(anyhow!( + "unknown format for extension .{ext}; use --format to override" + )) + } + }, + }; + let parser: Box = match format.as_str() { + "hcl" => Box::new(codegraph_docs::parsers::HclParser), + "yaml" => Box::new(codegraph_docs::parsers::YamlParser), + "json" => Box::new(codegraph_docs::parsers::JsonParser), + "toml" => Box::new(codegraph_docs::parsers::TomlParser), + _ => return Err(anyhow!("unsupported document format: {format}")), + }; + let doc_id = graph.stats().docs as u64 + 1; + let doc = parser.parse(&path, &source, doc_id)?; + let inserted = graph.upsert_document(doc).await?; + println!("ingested {} → doc_id={}", path, inserted); + } + DocCmd::Search { pattern: _, depth } => { + use codegraph_docs::DocToken; + let tokens = vec![DocToken::root(), DocToken::field(0)]; // simplified + let ids = graph.search_path(&tokens, Some(depth)).await?; + if ids.is_empty() { + println!("no nodes matched"); + } else { + for id in &ids { + if let Some(payload) = graph.hydrate(*id) { + println!("{}: {:?}", id, payload); + } + } + } + } + DocCmd::Hydrate { node_id } => match graph.hydrate(node_id) { + Some(payload) => { + let json = serde_json::to_string_pretty(&payload)?; + println!("{json}"); + } + None => println!("node {node_id} not found"), + }, + DocCmd::List => { + let stats = graph.stats(); + println!("documents: {}, nodes: {}", stats.docs, stats.nodes); + } + DocCmd::Stats => { + let stats = graph.stats(); + println!("documents: {}", stats.docs); + println!("nodes: {}", stats.nodes); + } + } + Ok(()) +} diff --git a/docs/architecture.md b/docs/architecture.md index 032a29021..e47297ec1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,8 +11,10 @@ crates/ codegraph-graph/ GraphIndex (semgraph): registry + 2 engines (chain Search + name Search) + pluggable storage (SQLite / LMDB / Redis / Postgres / MySQL) + optional embedding vector index codegraph-context/ Markdown/JSON context formatter (symbol + callers + callees + source) codegraph-api/ GraphApi wrapper on SharedGraphIndex (async query surface) + codegraph-docs/ Document graph: DocParser trait, HCL/YAML/JSON/TOML parsers, DocumentGraph with DocToken tries codegraph-sboxes/ Behavior sandbox: Cranelift JIT compile of function groups + Rhai mock runtime codegraph-mcp/ MCP server on the rmcp SDK (stdio + Streamable HTTP) + 24‑tool dispatch, session‑driven + codegraph-graphql/ GraphQL HTTP API server (queries + mutations for code + documents) codegraph-bench/ Benchmarks (criterion search benches, storage benches, codspeed) codegraph/ CLI lifecycle (init/deinit/embed/serve --mcp) + watcher (notify + debounced full re‑index) ``` @@ -37,30 +39,41 @@ files → ignore::WalkBuilder → rayon parse pool (tree‑sitter, 14 langs) MCP server / CLI lifecycle ``` -## 📄 Supported Formats +## 📄 Document Formats (codegraph-docs) -CodeGraph-docs now supports parsing the following configuration file formats: +CodeGraph-docs supports parsing structured configuration and document files into a unified document graph: -| Format | Parser | Status | -|--------|--------|--------| -| YAML | YamlParser | ✅ Implemented | -| JSON | JsonParser | ✅ Implemented | -| TOML | TomlParser | ✅ Implemented | -| **HCL** (HashiCorp Configuration Language) | **HclParser** | **✅ New** | -| **Terraform (.tf)** | **HclParser** | **✅ New** | +| Format | Extensions | Parser | Status | +|--------|------------|--------|--------| +| YAML | `.yaml`, `.yml` | YamlParser | ✅ Implemented | +| JSON | `.json` | JsonParser | ✅ Implemented | +| TOML | `.toml` | TomlParser | ✅ Implemented | +| **HCL** | `.hcl`, `.tf` | HclParser | ✅ Implemented | -HCL and Terraform files can now be indexed and analyzed through the codegraph CLI, enabling semantic understanding of HashiCorp configuration files. +### Document Graph -## 📄 Supported Formats (in crates/codegraph-docs/src/parsers/mod.rs): +Parsed documents are stored in a `DocumentGraph` backed by `codegraph-graph`'s `InMemoryStorage` (persistent tries) with `Search` indices for path, type, value, struct, and pattern queries. -Format Parser Status -━━━━━━━━ ━━━━━━━━━━━━ ━━━━━━━━━━━━━━━━ - YAML YamlParser ✅ Implemented - ──────── ──────────── ──────────────── - JSON JsonParser ✅ Implemented - ──────── ──────────── ──────────────── - TOML TomlParser ✅ Implemented - ──────── ──────────── ──────────────── - HCL HclParser ✅ New - ──────── ──────────── ──────────────── - Terraform (.tf) HclParser ✅ New \ No newline at end of file +**Access points:** +- **CLI**: `codegraph doc ingest `, `codegraph doc search`, `codegraph doc hydrate `, `codegraph doc list`, `codegraph doc stats` +- **MCP**: `codegraph_doc_ingest`, `codegraph_doc_search`, `codegraph_doc_hydrate`, `codegraph_doc_list`, `codegraph_doc_stats` +- **GraphQL**: `docList`, `docSearch`, `docStats` queries and `docIngest`, `docStats` mutations + +**Format auto-detection**: `.tf`/`.hcl` → hcl, `.yaml`/`.yml` → yaml, `.json` → json, `.toml` → toml. + +### Crate Structure + +``` +codegraph-docs/ + ├── lib.rs Re-exports (DocConfig, DocumentGraph, DocToken, DocParser, ...) + ├── config.rs DocConfig (id bases, bloom cap) + ├── graph.rs DocumentGraph, NodePayload, DocStats + ├── ir.rs Document, Node, Kind, Scalar, ByteSpan + ├── tokenize.rs DocToken (8-byte structural token) + └── parsers/ + ├── mod.rs DocParser trait, build_document(), all parsers + ├── hcl.rs HclParser (hcl-rs) + ├── yaml.rs YamlParser (serde_yaml) + ├── json.rs JsonParser (serde_json) + └── toml.rs TomlParser (toml) +``` \ No newline at end of file diff --git a/packaging/aur/codegraph-rs-bin/PKGBUILD b/packaging/aur/codegraph-rs-bin/PKGBUILD index fc6d468d7..da3603ca5 100644 --- a/packaging/aur/codegraph-rs-bin/PKGBUILD +++ b/packaging/aur/codegraph-rs-bin/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: Hung Pham pkgname=codegraph-rs-bin -pkgver=2.1.3 +pkgver=2.1.4 pkgrel=1 pkgdesc="Local-first code intelligence: tree-sitter knowledge graph + MCP server (prebuilt binary)" arch=('x86_64' 'aarch64') diff --git a/packaging/choco/codegraph.nuspec b/packaging/choco/codegraph.nuspec index a7e74366f..1d6b6db65 100644 --- a/packaging/choco/codegraph.nuspec +++ b/packaging/choco/codegraph.nuspec @@ -2,7 +2,7 @@ codegraph - 2.1.3 + 2.1.4 codegraph Hung Pham https://github.com/hungpham10/codegraph-rs diff --git a/packaging/winget/codegraph.yaml b/packaging/winget/codegraph.yaml index 11b259cea..745bfbaf5 100644 --- a/packaging/winget/codegraph.yaml +++ b/packaging/winget/codegraph.yaml @@ -6,7 +6,7 @@ # release time (or automate it in the release pipeline before submitting to # microsoft/winget-pkgs). PackageIdentifier: hungpham10.codegraph -PackageVersion: 2.1.3 +PackageVersion: 2.1.4 PackageName: codegraph Publisher: Hung Pham PublisherUrl: https://github.com/hungpham10/codegraph-rs @@ -18,7 +18,7 @@ PackageUrl: https://github.com/hungpham10/codegraph-rs InstallerType: zip Installers: - Architecture: x64 - InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.3/codegraph-x86_64-pc-windows-msvc.zip + InstallerUrl: https://github.com/hungpham10/codegraph-rs/releases/download/v2.1.4/codegraph-x86_64-pc-windows-msvc.zip InstallerSha256: 0000000000000000000000000000000000000000000000000000000000000000 InstallerType: zip ManifestType: singleton diff --git a/scripts/install.ps1 b/scripts/install.ps1 index aec63edd9..579cd6392 100644 --- a/scripts/install.ps1 +++ b/scripts/install.ps1 @@ -5,11 +5,11 @@ # # Usage (pin a version / download the script first): # irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 -OutFile install.ps1 -# .\install.ps1 -Version 2.1.3 +# .\install.ps1 -Version 2.1.4 [CmdletBinding()] param( - # Pin a specific version, e.g. "2.1.3". Empty = latest release. + # Pin a specific version, e.g. "2.1.4". Empty = latest release. [string]$Version )