From 6e7c7e026ddc5564455d65e14b7ef639416c6ec1 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:53 +0530 Subject: [PATCH 01/10] Add first_non_trivia_token and last_non_trivia_token --- src/api.rs | 25 +++++++++++++++++++++++-- src/cursor.rs | 45 ++++++++++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/api.rs b/src/api.rs index 14ffdf7a..fd0f747b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -184,15 +184,22 @@ impl SyntaxNode { self.raw.prev_sibling_or_token().map(NodeOrToken::from) } - /// Return the leftmost token in the subtree of this node. pub fn first_token(&self) -> Option> { self.raw.first_token().map(SyntaxToken::from) } - /// Return the rightmost token in the subtree of this node. + pub fn last_token(&self) -> Option> { self.raw.last_token().map(SyntaxToken::from) } + pub fn first_non_trivia_token(&self) -> Option> { + self.raw.first_non_trivia_token().map(SyntaxToken::from) + } + + pub fn last_non_trivia_token(&self) -> Option> { + self.raw.last_non_trivia_token().map(SyntaxToken::from) + } + pub fn siblings(&self, direction: Direction) -> impl Iterator> { self.raw.siblings(direction).map(SyntaxNode::from) } @@ -355,6 +362,20 @@ impl SyntaxElement { } } + pub fn first_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.first_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + + pub fn last_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.last_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + pub fn index(&self) -> usize { match self { NodeOrToken::Node(it) => it.index(), diff --git a/src/cursor.rs b/src/cursor.rs index d837be60..2bcd1ae8 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -441,10 +441,26 @@ impl SyntaxNode { } pub fn first_token(&self) -> Option { - self.first_child_or_token()?.first_token() + self.first_non_trivia_token().map(SyntaxToken::first_token_including_trivia) } + pub fn last_token(&self) -> Option { - self.last_child_or_token()?.last_token() + self.last_non_trivia_token().map(SyntaxToken::last_token_including_trivia) + } + + pub fn first_non_trivia_token(&self) -> Option { + self.children_with_tokens().find_map(|child| child.first_non_trivia_token()) + } + + pub fn last_non_trivia_token(&self) -> Option { + let mut child = self.last_child_or_token(); + while let Some(element) = child { + if let Some(token) = element.last_non_trivia_token() { + return Some(token); + } + child = element.prev_sibling_or_token(); + } + None } #[inline] @@ -726,20 +742,20 @@ impl SyntaxToken { fn next_non_trivia_token(&self) -> Option { match self.next_sibling_or_token() { - Some(element) => element.first_token(), + Some(element) => element.first_non_trivia_token(), None => self .ancestors() .find_map(|it| it.next_sibling_or_token()) - .and_then(|element| element.first_token()), + .and_then(|element| element.first_non_trivia_token()), } } fn prev_non_trivia_token(&self) -> Option { match self.prev_sibling_or_token() { - Some(element) => element.last_token(), + Some(element) => element.last_non_trivia_token(), None => self .ancestors() .find_map(|it| it.prev_sibling_or_token()) - .and_then(|element| element.last_token()), + .and_then(|element| element.last_non_trivia_token()), } } @@ -886,12 +902,27 @@ impl SyntaxElement { pub fn first_token(&self) -> Option { match self { NodeOrToken::Node(it) => it.first_token(), - NodeOrToken::Token(it) => Some(it.clone()), + NodeOrToken::Token(it) => Some(it.clone().first_token_including_trivia()), } } + pub fn last_token(&self) -> Option { match self { NodeOrToken::Node(it) => it.last_token(), + NodeOrToken::Token(it) => Some(it.clone().last_token_including_trivia()), + } + } + + pub fn first_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.first_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + + pub fn last_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.last_non_trivia_token(), NodeOrToken::Token(it) => Some(it.clone()), } } From 3ba38990b7ad0658e058eb44b639473993f4f661 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:54 +0530 Subject: [PATCH 02/10] Add next_non_trivia_token and prev_non_trivia_token --- src/api.rs | 30 ++++++++++++ src/cursor.rs | 125 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 127 insertions(+), 28 deletions(-) diff --git a/src/api.rs b/src/api.rs index fd0f747b..3266a50d 100644 --- a/src/api.rs +++ b/src/api.rs @@ -200,6 +200,14 @@ impl SyntaxNode { self.raw.last_non_trivia_token().map(SyntaxToken::from) } + pub fn next_non_trivia_token(&self) -> Option> { + self.raw.next_non_trivia_token().map(SyntaxToken::from) + } + + pub fn prev_non_trivia_token(&self) -> Option> { + self.raw.prev_non_trivia_token().map(SyntaxToken::from) + } + pub fn siblings(&self, direction: Direction) -> impl Iterator> { self.raw.siblings(direction).map(SyntaxNode::from) } @@ -352,6 +360,14 @@ impl SyntaxToken { pub fn prev_token(&self) -> Option> { self.raw.prev_token().map(SyntaxToken::from) } + + pub fn next_non_trivia_token(&self) -> Option> { + self.raw.next_non_trivia_token().map(SyntaxToken::from) + } + + pub fn prev_non_trivia_token(&self) -> Option> { + self.raw.prev_non_trivia_token().map(SyntaxToken::from) + } } impl SyntaxElement { @@ -424,6 +440,20 @@ impl SyntaxElement { NodeOrToken::Token(it) => it.prev_sibling_or_token(), } } + + pub fn next_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.next_non_trivia_token(), + NodeOrToken::Token(it) => it.next_non_trivia_token(), + } + } + + pub fn prev_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.prev_non_trivia_token(), + NodeOrToken::Token(it) => it.prev_non_trivia_token(), + } + } } #[derive(Debug, Clone)] diff --git a/src/cursor.rs b/src/cursor.rs index 2bcd1ae8..6f4da39e 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -200,6 +200,14 @@ impl NodeData { self.parent.map(|it| unsafe { &*it.as_ptr() }) } + #[inline] + fn in_leading_trivia(&self) -> Option { + let parent = self.parent()?; + let Green::Token { ptr } = parent.green else { return None }; + let start = parent.offset() + unsafe { ptr.as_ref() }.leading_trivia_len(); + Some(self.offset() < start) + } + #[inline] fn green(&self) -> GreenElementRef<'_> { match &self.green { @@ -284,6 +292,54 @@ impl NodeData { Some(SyntaxElement::new(child.as_ref(), parent, index as u32, offset)) }) } + + fn next_non_trivia_token(&self) -> Option { + if let Some(leading) = self.in_leading_trivia() { + let owner = self.parent_token()?; + return if leading { Some(owner) } else { owner.data().next_non_trivia_token() }; + } + let mut parent = self.parent_node(); + let mut element = self.next_sibling_or_token(); + loop { + match element { + Some(current) => { + if let Some(token) = current.first_non_trivia_token() { + return Some(token); + } + element = current.next_sibling_or_token(); + } + None => { + let node = parent?; + element = node.next_sibling_or_token(); + parent = node.parent(); + } + } + } + } + + fn prev_non_trivia_token(&self) -> Option { + if let Some(leading) = self.in_leading_trivia() { + let owner = self.parent_token()?; + return if leading { owner.data().prev_non_trivia_token() } else { Some(owner) }; + } + let mut parent = self.parent_node(); + let mut element = self.prev_sibling_or_token(); + loop { + match element { + Some(current) => { + if let Some(token) = current.last_non_trivia_token() { + return Some(token); + } + element = current.prev_sibling_or_token(); + } + None => { + let node = parent?; + element = node.prev_sibling_or_token(); + parent = node.parent(); + } + } + } + } } impl SyntaxNode { @@ -463,6 +519,13 @@ impl SyntaxNode { None } + pub fn next_non_trivia_token(&self) -> Option { + self.data().next_non_trivia_token() + } + pub fn prev_non_trivia_token(&self) -> Option { + self.data().prev_non_trivia_token() + } + #[inline] pub fn siblings(&self, direction: Direction) -> impl Iterator { iter::successors(Some(self.clone()), move |node| match direction { @@ -710,13 +773,15 @@ impl SyntaxToken { } pub fn next_token(&self) -> Option { - if let Some(parent) = self.data().parent_token() { - let index = self.index() + 1; - return if self.is_leading_trivia(&parent) { - parent.leading_trivia().nth(index).or(Some(parent)) + if let (Some(leading), Some(owner)) = + (self.data().in_leading_trivia(), self.data().parent_token()) + { + let index = self.data().index() as usize; + return if leading { + owner.leading_trivia().nth(index + 1).or(Some(owner)) } else { - parent.trailing_trivia().nth(index).or_else(|| { - parent.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia) + owner.trailing_trivia().nth(index + 1).or_else(|| { + owner.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia) }) }; } @@ -725,14 +790,16 @@ impl SyntaxToken { .or_else(|| self.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia)) } pub fn prev_token(&self) -> Option { - if let Some(parent) = self.data().parent_token() { - let index = self.index().checked_sub(1); - return if self.is_leading_trivia(&parent) { - index.and_then(|it| parent.leading_trivia().nth(it)).or_else(|| { - parent.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia) + if let (Some(leading), Some(owner)) = + (self.data().in_leading_trivia(), self.data().parent_token()) + { + let index = (self.data().index() as usize).checked_sub(1); + return if leading { + index.and_then(|it| owner.leading_trivia().nth(it)).or_else(|| { + owner.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia) }) } else { - index.and_then(|it| parent.trailing_trivia().nth(it)).or(Some(parent)) + index.and_then(|it| owner.trailing_trivia().nth(it)).or(Some(owner)) }; } self.leading_trivia() @@ -740,23 +807,12 @@ impl SyntaxToken { .or_else(|| self.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia)) } - fn next_non_trivia_token(&self) -> Option { - match self.next_sibling_or_token() { - Some(element) => element.first_non_trivia_token(), - None => self - .ancestors() - .find_map(|it| it.next_sibling_or_token()) - .and_then(|element| element.first_non_trivia_token()), - } + pub fn next_non_trivia_token(&self) -> Option { + self.data().next_non_trivia_token() } - fn prev_non_trivia_token(&self) -> Option { - match self.prev_sibling_or_token() { - Some(element) => element.last_non_trivia_token(), - None => self - .ancestors() - .find_map(|it| it.prev_sibling_or_token()) - .and_then(|element| element.last_non_trivia_token()), - } + + pub fn prev_non_trivia_token(&self) -> Option { + self.data().prev_non_trivia_token() } fn first_token_including_trivia(self) -> SyntaxToken { @@ -940,6 +996,19 @@ impl SyntaxElement { } } + pub fn next_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.next_non_trivia_token(), + NodeOrToken::Token(it) => it.next_non_trivia_token(), + } + } + pub fn prev_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.prev_non_trivia_token(), + NodeOrToken::Token(it) => it.prev_non_trivia_token(), + } + } + fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { let range = self.text_range_including_trivia(); assert!(range.start() <= offset && offset <= range.end()); From b511ac0bf6ca287b88f8efa1272cd8964bb3fd1a Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:55 +0530 Subject: [PATCH 03/10] Make token index optional and expose is_trivia --- src/api.rs | 17 ++++++++++++++--- src/cursor.rs | 20 ++++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/api.rs b/src/api.rs index 3266a50d..847311ff 100644 --- a/src/api.rs +++ b/src/api.rs @@ -291,10 +291,14 @@ impl SyntaxToken { self.raw.text_range_including_trivia() } - pub fn index(&self) -> usize { + pub fn index(&self) -> Option { self.raw.index() } + pub fn is_trivia(&self) -> bool { + self.raw.is_trivia() + } + pub fn text(&self) -> &str { self.raw.text() } @@ -392,9 +396,9 @@ impl SyntaxElement { } } - pub fn index(&self) -> usize { + pub fn index(&self) -> Option { match self { - NodeOrToken::Node(it) => it.index(), + NodeOrToken::Node(it) => Some(it.index()), NodeOrToken::Token(it) => it.index(), } } @@ -454,6 +458,13 @@ impl SyntaxElement { NodeOrToken::Token(it) => it.prev_non_trivia_token(), } } + + pub fn is_trivia(&self) -> bool { + match self { + NodeOrToken::Node(_) => false, + NodeOrToken::Token(it) => it.is_trivia(), + } + } } #[derive(Debug, Clone)] diff --git a/src/cursor.rs b/src/cursor.rs index 6f4da39e..5d8c7568 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -694,6 +694,11 @@ impl SyntaxToken { parent.replace_with(new_parent) } + #[inline] + pub fn is_trivia(&self) -> bool { + self.data().parent().is_some_and(|parent| matches!(parent.green, Green::Token { .. })) + } + #[inline] pub fn kind(&self) -> SyntaxKind { self.data().kind() @@ -710,8 +715,8 @@ impl SyntaxToken { } #[inline] - pub fn index(&self) -> usize { - self.data().index() as usize + pub fn index(&self) -> Option { + (!self.is_trivia()).then(|| self.data().index() as usize) } #[inline] @@ -915,9 +920,9 @@ impl SyntaxElement { } #[inline] - pub fn index(&self) -> usize { + pub fn index(&self) -> Option { match self { - NodeOrToken::Node(it) => it.index(), + NodeOrToken::Node(it) => Some(it.index()), NodeOrToken::Token(it) => it.index(), } } @@ -1009,6 +1014,13 @@ impl SyntaxElement { } } + pub fn is_trivia(&self) -> bool { + match self { + NodeOrToken::Node(_) => false, + NodeOrToken::Token(it) => it.is_trivia(), + } + } + fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { let range = self.text_range_including_trivia(); assert!(range.start() <= offset && offset <= range.end()); From 08d5aefa5b830ae71498724d1fe6be7ffc779079 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:56 +0530 Subject: [PATCH 04/10] Reject trivia views in SyntaxToken::replace_with --- src/cursor.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/cursor.rs b/src/cursor.rs index 5d8c7568..9535c503 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -674,19 +674,7 @@ impl SyntaxToken { pub fn replace_with(&self, replacement: GreenToken) -> GreenNode { assert_eq!(self.kind(), replacement.kind()); - if let Some(owner) = self.data().parent_token() { - let index = self.data().index() as usize; - let green = owner.green(); - let mut leading = green.leading_trivia().to_vec(); - let mut trailing = green.trailing_trivia().to_vec(); - if self.is_leading_trivia(&owner) { - leading[index] = replacement; - } else { - trailing[index] = replacement; - } - let new_owner = GreenToken::with_trivia(green.kind(), green.text(), leading, trailing); - return owner.replace_with(new_owner); - } + assert!(!self.is_trivia(), "cannot replace a trivia view; replace its owning token"); let parent = self.parent().unwrap(); let me: u32 = self.data().index(); @@ -829,10 +817,6 @@ impl SyntaxToken { last.unwrap_or(self) } - fn is_leading_trivia(&self, parent: &SyntaxToken) -> bool { - self.data().offset() < parent.text_range().start() - } - pub(crate) fn with_trivia(&self) -> impl Iterator { self.leading_trivia().chain(iter::once(self.clone())).chain(self.trailing_trivia()) } From 5025d1092b7858a8812316b0b380ef15cad023ed Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:57 +0530 Subject: [PATCH 05/10] Add node ranges and text without outer trivia --- src/api.rs | 22 ++++++++++++++++++++++ src/cursor.rs | 23 ++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/api.rs b/src/api.rs index 847311ff..735eaa93 100644 --- a/src/api.rs +++ b/src/api.rs @@ -124,6 +124,10 @@ impl SyntaxNode { self.raw.text_range() } + pub fn text_range_without_outer_trivia(&self) -> TextRange { + self.raw.text_range_without_outer_trivia() + } + pub fn index(&self) -> usize { self.raw.index() } @@ -132,6 +136,10 @@ impl SyntaxNode { self.raw.text() } + pub fn text_without_outer_trivia(&self) -> SyntaxText { + self.raw.text_without_outer_trivia() + } + pub fn green(&self) -> &GreenNodeData { self.raw.green() } @@ -382,6 +390,20 @@ impl SyntaxElement { } } + pub fn text_range_including_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range(), + NodeOrToken::Token(it) => it.text_range_including_trivia(), + } + } + + pub fn text_range_without_outer_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range_without_outer_trivia(), + NodeOrToken::Token(it) => it.text_range(), + } + } + pub fn first_non_trivia_token(&self) -> Option> { match self { NodeOrToken::Node(it) => it.first_non_trivia_token(), diff --git a/src/cursor.rs b/src/cursor.rs index 9535c503..76c19bfa 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -397,6 +397,15 @@ impl SyntaxNode { self.data().text_range() } + pub fn text_range_without_outer_trivia(&self) -> TextRange { + match (self.first_non_trivia_token(), self.last_non_trivia_token()) { + (Some(first), Some(last)) => { + TextRange::new(first.text_range().start(), last.text_range().end()) + } + _ => TextRange::empty(self.offset()), + } + } + #[inline] pub fn index(&self) -> usize { self.data().index() as usize @@ -407,6 +416,10 @@ impl SyntaxNode { SyntaxText::new(self.clone()) } + pub fn text_without_outer_trivia(&self) -> SyntaxText { + self.text().slice(self.text_range_without_outer_trivia() - self.text_range().start()) + } + #[inline] pub fn green(&self) -> &GreenNodeData { self.green_ref() @@ -896,13 +909,21 @@ impl SyntaxElement { } #[inline] - fn text_range_including_trivia(&self) -> TextRange { + pub fn text_range_including_trivia(&self) -> TextRange { match self { NodeOrToken::Node(it) => it.text_range(), NodeOrToken::Token(it) => it.text_range_including_trivia(), } } + #[inline] + pub fn text_range_without_outer_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range_without_outer_trivia(), + NodeOrToken::Token(it) => it.text_range(), + } + } + #[inline] pub fn index(&self) -> Option { match self { From 06ca547662c0b66231c64274024dca3865823e1d Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:58 +0530 Subject: [PATCH 06/10] Add trivia_before and trivia_after --- src/api.rs | 24 ++++++++++++++++++++++++ src/cursor.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/api.rs b/src/api.rs index 735eaa93..c917e653 100644 --- a/src/api.rs +++ b/src/api.rs @@ -216,6 +216,14 @@ impl SyntaxNode { self.raw.prev_non_trivia_token().map(SyntaxToken::from) } + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_before().map(SyntaxToken::from) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_after().map(SyntaxToken::from) + } + pub fn siblings(&self, direction: Direction) -> impl Iterator> { self.raw.siblings(direction).map(SyntaxNode::from) } @@ -380,6 +388,14 @@ impl SyntaxToken { pub fn prev_non_trivia_token(&self) -> Option> { self.raw.prev_non_trivia_token().map(SyntaxToken::from) } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_before().map(SyntaxToken::from) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_after().map(SyntaxToken::from) + } } impl SyntaxElement { @@ -487,6 +503,14 @@ impl SyntaxElement { NodeOrToken::Token(it) => it.is_trivia(), } } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() + } } #[derive(Debug, Clone)] diff --git a/src/cursor.rs b/src/cursor.rs index 76c19bfa..f3897ede 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -539,6 +539,14 @@ impl SyntaxNode { self.data().prev_non_trivia_token() } + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() + } + #[inline] pub fn siblings(&self, direction: Direction) -> impl Iterator { iter::successors(Some(self.clone()), move |node| match direction { @@ -821,6 +829,16 @@ impl SyntaxToken { self.data().prev_non_trivia_token() } + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + let prev = self.prev_non_trivia_token().map(|it| it.trailing_trivia()); + prev.into_iter().flatten().chain(self.leading_trivia()) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + let next = self.next_non_trivia_token().map(|it| it.leading_trivia()); + self.trailing_trivia().chain(next.into_iter().flatten()) + } + fn first_token_including_trivia(self) -> SyntaxToken { let first = self.leading_trivia().next(); first.unwrap_or(self) @@ -1026,6 +1044,14 @@ impl SyntaxElement { } } + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() + } + fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { let range = self.text_range_including_trivia(); assert!(range.start() <= offset && offset <= range.end()); From fec9bb30e70b28926ea675b6e0c9b7bf89305cd5 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:51:59 +0530 Subject: [PATCH 07/10] Inline the trivia iterators and rename with_trivia --- src/api.rs | 8 ++++---- src/cursor.rs | 40 +++++++++++++++------------------------- src/syntax_text.rs | 2 +- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/api.rs b/src/api.rs index c917e653..d1e0035f 100644 --- a/src/api.rs +++ b/src/api.rs @@ -70,18 +70,18 @@ impl fmt::Debug for SyntaxToken { write!(f, " {:?}", format!("{} ...", &text[..idx]))?; } write!(f, " [")?; - for (idx, piece) in self.leading_trivia().enumerate() { + for (idx, trivia) in self.leading_trivia().enumerate() { if idx > 0 { write!(f, ", ")?; } - write!(f, "{:?}({:?})", piece.kind(), piece.text())?; + write!(f, "{:?}({:?})", trivia.kind(), trivia.text())?; } write!(f, "] [")?; - for (idx, piece) in self.trailing_trivia().enumerate() { + for (idx, trivia) in self.trailing_trivia().enumerate() { if idx > 0 { write!(f, ", ")?; } - write!(f, "{:?}({:?})", piece.kind(), piece.text())?; + write!(f, "{:?}({:?})", trivia.kind(), trivia.text())?; } write!(f, "]") } diff --git a/src/cursor.rs b/src/cursor.rs index f3897ede..23331679 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -744,7 +744,7 @@ impl SyntaxToken { } pub fn text_including_trivia(&self) -> String { - self.with_trivia().map(|token| token.text().to_owned()).collect() + self.tokens_including_trivia().map(|trivia| trivia.text().to_owned()).collect() } #[inline] @@ -848,12 +848,12 @@ impl SyntaxToken { last.unwrap_or(self) } - pub(crate) fn with_trivia(&self) -> impl Iterator { + pub(crate) fn tokens_including_trivia(&self) -> impl DoubleEndedIterator { self.leading_trivia().chain(iter::once(self.clone())).chain(self.trailing_trivia()) } fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { - let mut tokens = self.with_trivia().filter(|token| { + let mut tokens = self.tokens_including_trivia().filter(|token| { let range = token.text_range(); !range.is_empty() && range.start() <= offset && offset <= range.end() }); @@ -869,34 +869,24 @@ impl SyntaxToken { pub fn leading_trivia( &self, ) -> impl DoubleEndedIterator + ExactSizeIterator { - self.trivia(true) + let owner = self.clone(); + let start = self.data().offset(); + (0..self.green().leading_trivia().len()).map(move |index| { + let trivia = owner.green().leading_trivia(); + let offset = start + trivia[..index].iter().map(|it| it.text_len()).sum::(); + SyntaxToken::new_trivia(&trivia[index], owner.clone(), index as u32, offset) + }) } pub fn trailing_trivia( &self, ) -> impl DoubleEndedIterator + ExactSizeIterator { - self.trivia(false) - } - - fn trivia( - &self, - leading: bool, - ) -> impl DoubleEndedIterator + ExactSizeIterator { - let green = self.green(); - let (start, len) = if leading { - (self.data().offset(), green.leading_trivia().len()) - } else { - (self.text_range().end(), green.trailing_trivia().len()) - }; - let token = self.clone(); - (0..len).map(move |index| { - let trivia = if leading { - token.green().leading_trivia() - } else { - token.green().trailing_trivia() - }; + let owner = self.clone(); + let start = self.text_range().end(); + (0..self.green().trailing_trivia().len()).map(move |index| { + let trivia = owner.green().trailing_trivia(); let offset = start + trivia[..index].iter().map(|it| it.text_len()).sum::(); - SyntaxToken::new_trivia(&trivia[index], token.clone(), index as u32, offset) + SyntaxToken::new_trivia(&trivia[index], owner.clone(), index as u32, offset) }) } } diff --git a/src/syntax_text.rs b/src/syntax_text.rs index abe6d49c..1aac20bc 100644 --- a/src/syntax_text.rs +++ b/src/syntax_text.rs @@ -108,7 +108,7 @@ impl SyntaxText { self.node .descendants_with_tokens() .filter_map(|element| element.into_token()) - .flat_map(|token| token.with_trivia()) + .flat_map(|token| token.tokens_including_trivia()) .filter_map(move |token| { let token_range = token.text_range(); let range = text_range.intersect(token_range)?; From 42cb7d73f066ed6d345e9f39706e977bfe1bb646 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Thu, 10 Sep 2026 23:52:00 +0530 Subject: [PATCH 08/10] Omit empty trivia from token Debug output --- src/api.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api.rs b/src/api.rs index d1e0035f..7e1aec18 100644 --- a/src/api.rs +++ b/src/api.rs @@ -69,6 +69,9 @@ impl fmt::Debug for SyntaxToken { let idx = (21..25).find(|&idx| text.is_char_boundary(idx)).unwrap(); write!(f, " {:?}", format!("{} ...", &text[..idx]))?; } + if self.leading_trivia().len() == 0 && self.trailing_trivia().len() == 0 { + return Ok(()); + } write!(f, " [")?; for (idx, trivia) in self.leading_trivia().enumerate() { if idx > 0 { From a0b80bf169e2a0fce718b6af65a40711cbbf6038 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Fri, 11 Sep 2026 11:55:52 +0530 Subject: [PATCH 09/10] Add SyntaxToken::owning_node --- src/api.rs | 4 ++++ src/cursor.rs | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/api.rs b/src/api.rs index 7e1aec18..27fb14b1 100644 --- a/src/api.rs +++ b/src/api.rs @@ -346,6 +346,10 @@ impl SyntaxToken { self.raw.parent().map(SyntaxNode::from) } + pub fn owning_node(&self) -> Option> { + self.raw.owning_node().map(SyntaxNode::from) + } + /// Iterator over all the ancestors of this token excluding itself. #[deprecated = "use `SyntaxToken::parent_ancestors` instead"] pub fn ancestors(&self) -> impl Iterator> { diff --git a/src/cursor.rs b/src/cursor.rs index 23331679..8a39875b 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -762,6 +762,19 @@ impl SyntaxToken { std::iter::successors(self.parent(), SyntaxNode::parent) } + pub fn owning_node(&self) -> Option { + if !self.is_trivia() { + return self.parent(); + } + match (self.prev_non_trivia_token(), self.next_non_trivia_token()) { + (Some(prev), Some(next)) => { + prev.ancestors().find(|it| it.text_range().contains_range(next.text_range())) + } + (Some(other), None) | (None, Some(other)) => other.ancestors().last(), + (None, None) => None, + } + } + #[inline] pub fn tree_top(&self) -> SyntaxNode { self.ancestors().last().unwrap() From 98f180c582fa349df1cb9134761c2fcbbc6a3203 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Fri, 11 Sep 2026 14:52:05 +0530 Subject: [PATCH 10/10] Assert trivia_before and trivia_after are called on non-trivia tokens --- src/cursor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cursor.rs b/src/cursor.rs index 8a39875b..a4834a42 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -843,11 +843,13 @@ impl SyntaxToken { } pub fn trivia_before(&self) -> impl DoubleEndedIterator { + debug_assert!(!self.is_trivia(), "trivia_before is defined for non-trivia tokens"); let prev = self.prev_non_trivia_token().map(|it| it.trailing_trivia()); prev.into_iter().flatten().chain(self.leading_trivia()) } pub fn trivia_after(&self) -> impl DoubleEndedIterator { + debug_assert!(!self.is_trivia(), "trivia_after is defined for non-trivia tokens"); let next = self.next_non_trivia_token().map(|it| it.leading_trivia()); self.trailing_trivia().chain(next.into_iter().flatten()) }