diff --git a/src/api.rs b/src/api.rs index 14ffdf7a..27fb14b1 100644 --- a/src/api.rs +++ b/src/api.rs @@ -69,19 +69,22 @@ impl fmt::Debug for SyntaxToken { let idx = (21..25).find(|&idx| text.is_char_boundary(idx)).unwrap(); write!(f, " {:?}", format!("{} ...", &text[..idx]))?; } + if self.leading_trivia().len() == 0 && self.trailing_trivia().len() == 0 { + return Ok(()); + } write!(f, " [")?; - for (idx, piece) in self.leading_trivia().enumerate() { + for (idx, trivia) in self.leading_trivia().enumerate() { if idx > 0 { write!(f, ", ")?; } - write!(f, "{:?}({:?})", piece.kind(), piece.text())?; + write!(f, "{:?}({:?})", trivia.kind(), trivia.text())?; } write!(f, "] [")?; - for (idx, piece) in self.trailing_trivia().enumerate() { + for (idx, trivia) in self.trailing_trivia().enumerate() { if idx > 0 { write!(f, ", ")?; } - write!(f, "{:?}({:?})", piece.kind(), piece.text())?; + write!(f, "{:?}({:?})", trivia.kind(), trivia.text())?; } write!(f, "]") } @@ -124,6 +127,10 @@ impl SyntaxNode { self.raw.text_range() } + pub fn text_range_without_outer_trivia(&self) -> TextRange { + self.raw.text_range_without_outer_trivia() + } + pub fn index(&self) -> usize { self.raw.index() } @@ -132,6 +139,10 @@ impl SyntaxNode { self.raw.text() } + pub fn text_without_outer_trivia(&self) -> SyntaxText { + self.raw.text_without_outer_trivia() + } + pub fn green(&self) -> &GreenNodeData { self.raw.green() } @@ -184,15 +195,38 @@ impl SyntaxNode { self.raw.prev_sibling_or_token().map(NodeOrToken::from) } - /// Return the leftmost token in the subtree of this node. pub fn first_token(&self) -> Option> { self.raw.first_token().map(SyntaxToken::from) } - /// Return the rightmost token in the subtree of this node. + pub fn last_token(&self) -> Option> { self.raw.last_token().map(SyntaxToken::from) } + pub fn first_non_trivia_token(&self) -> Option> { + self.raw.first_non_trivia_token().map(SyntaxToken::from) + } + + pub fn last_non_trivia_token(&self) -> Option> { + self.raw.last_non_trivia_token().map(SyntaxToken::from) + } + + pub fn next_non_trivia_token(&self) -> Option> { + self.raw.next_non_trivia_token().map(SyntaxToken::from) + } + + pub fn prev_non_trivia_token(&self) -> Option> { + self.raw.prev_non_trivia_token().map(SyntaxToken::from) + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_before().map(SyntaxToken::from) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_after().map(SyntaxToken::from) + } + pub fn siblings(&self, direction: Direction) -> impl Iterator> { self.raw.siblings(direction).map(SyntaxNode::from) } @@ -276,10 +310,14 @@ impl SyntaxToken { self.raw.text_range_including_trivia() } - pub fn index(&self) -> usize { + pub fn index(&self) -> Option { self.raw.index() } + pub fn is_trivia(&self) -> bool { + self.raw.is_trivia() + } + pub fn text(&self) -> &str { self.raw.text() } @@ -308,6 +346,10 @@ impl SyntaxToken { self.raw.parent().map(SyntaxNode::from) } + pub fn owning_node(&self) -> Option> { + self.raw.owning_node().map(SyntaxNode::from) + } + /// Iterator over all the ancestors of this token excluding itself. #[deprecated = "use `SyntaxToken::parent_ancestors` instead"] pub fn ancestors(&self) -> impl Iterator> { @@ -345,6 +387,22 @@ impl SyntaxToken { pub fn prev_token(&self) -> Option> { self.raw.prev_token().map(SyntaxToken::from) } + + pub fn next_non_trivia_token(&self) -> Option> { + self.raw.next_non_trivia_token().map(SyntaxToken::from) + } + + pub fn prev_non_trivia_token(&self) -> Option> { + self.raw.prev_non_trivia_token().map(SyntaxToken::from) + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_before().map(SyntaxToken::from) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.raw.trivia_after().map(SyntaxToken::from) + } } impl SyntaxElement { @@ -355,9 +413,37 @@ impl SyntaxElement { } } - pub fn index(&self) -> usize { + pub fn text_range_including_trivia(&self) -> TextRange { match self { - NodeOrToken::Node(it) => it.index(), + NodeOrToken::Node(it) => it.text_range(), + NodeOrToken::Token(it) => it.text_range_including_trivia(), + } + } + + pub fn text_range_without_outer_trivia(&self) -> TextRange { + match self { + NodeOrToken::Node(it) => it.text_range_without_outer_trivia(), + NodeOrToken::Token(it) => it.text_range(), + } + } + + pub fn first_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.first_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + + pub fn last_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.last_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + + pub fn index(&self) -> Option { + match self { + NodeOrToken::Node(it) => Some(it.index()), NodeOrToken::Token(it) => it.index(), } } @@ -403,6 +489,35 @@ impl SyntaxElement { NodeOrToken::Token(it) => it.prev_sibling_or_token(), } } + + pub fn next_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.next_non_trivia_token(), + NodeOrToken::Token(it) => it.next_non_trivia_token(), + } + } + + pub fn prev_non_trivia_token(&self) -> Option> { + match self { + NodeOrToken::Node(it) => it.prev_non_trivia_token(), + NodeOrToken::Token(it) => it.prev_non_trivia_token(), + } + } + + pub fn is_trivia(&self) -> bool { + match self { + NodeOrToken::Node(_) => false, + NodeOrToken::Token(it) => it.is_trivia(), + } + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator> { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator> { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() + } } #[derive(Debug, Clone)] diff --git a/src/cursor.rs b/src/cursor.rs index d837be60..a4834a42 100644 --- a/src/cursor.rs +++ b/src/cursor.rs @@ -200,6 +200,14 @@ impl NodeData { self.parent.map(|it| unsafe { &*it.as_ptr() }) } + #[inline] + fn in_leading_trivia(&self) -> Option { + let parent = self.parent()?; + let Green::Token { ptr } = parent.green else { return None }; + let start = parent.offset() + unsafe { ptr.as_ref() }.leading_trivia_len(); + Some(self.offset() < start) + } + #[inline] fn green(&self) -> GreenElementRef<'_> { match &self.green { @@ -284,6 +292,54 @@ impl NodeData { Some(SyntaxElement::new(child.as_ref(), parent, index as u32, offset)) }) } + + fn next_non_trivia_token(&self) -> Option { + if let Some(leading) = self.in_leading_trivia() { + let owner = self.parent_token()?; + return if leading { Some(owner) } else { owner.data().next_non_trivia_token() }; + } + let mut parent = self.parent_node(); + let mut element = self.next_sibling_or_token(); + loop { + match element { + Some(current) => { + if let Some(token) = current.first_non_trivia_token() { + return Some(token); + } + element = current.next_sibling_or_token(); + } + None => { + let node = parent?; + element = node.next_sibling_or_token(); + parent = node.parent(); + } + } + } + } + + fn prev_non_trivia_token(&self) -> Option { + if let Some(leading) = self.in_leading_trivia() { + let owner = self.parent_token()?; + return if leading { owner.data().prev_non_trivia_token() } else { Some(owner) }; + } + let mut parent = self.parent_node(); + let mut element = self.prev_sibling_or_token(); + loop { + match element { + Some(current) => { + if let Some(token) = current.last_non_trivia_token() { + return Some(token); + } + element = current.prev_sibling_or_token(); + } + None => { + let node = parent?; + element = node.prev_sibling_or_token(); + parent = node.parent(); + } + } + } + } } impl SyntaxNode { @@ -341,6 +397,15 @@ impl SyntaxNode { self.data().text_range() } + pub fn text_range_without_outer_trivia(&self) -> TextRange { + match (self.first_non_trivia_token(), self.last_non_trivia_token()) { + (Some(first), Some(last)) => { + TextRange::new(first.text_range().start(), last.text_range().end()) + } + _ => TextRange::empty(self.offset()), + } + } + #[inline] pub fn index(&self) -> usize { self.data().index() as usize @@ -351,6 +416,10 @@ impl SyntaxNode { SyntaxText::new(self.clone()) } + pub fn text_without_outer_trivia(&self) -> SyntaxText { + self.text().slice(self.text_range_without_outer_trivia() - self.text_range().start()) + } + #[inline] pub fn green(&self) -> &GreenNodeData { self.green_ref() @@ -441,10 +510,41 @@ impl SyntaxNode { } pub fn first_token(&self) -> Option { - self.first_child_or_token()?.first_token() + self.first_non_trivia_token().map(SyntaxToken::first_token_including_trivia) } + pub fn last_token(&self) -> Option { - self.last_child_or_token()?.last_token() + self.last_non_trivia_token().map(SyntaxToken::last_token_including_trivia) + } + + pub fn first_non_trivia_token(&self) -> Option { + self.children_with_tokens().find_map(|child| child.first_non_trivia_token()) + } + + pub fn last_non_trivia_token(&self) -> Option { + let mut child = self.last_child_or_token(); + while let Some(element) = child { + if let Some(token) = element.last_non_trivia_token() { + return Some(token); + } + child = element.prev_sibling_or_token(); + } + None + } + + pub fn next_non_trivia_token(&self) -> Option { + self.data().next_non_trivia_token() + } + pub fn prev_non_trivia_token(&self) -> Option { + self.data().prev_non_trivia_token() + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() } #[inline] @@ -595,19 +695,7 @@ impl SyntaxToken { pub fn replace_with(&self, replacement: GreenToken) -> GreenNode { assert_eq!(self.kind(), replacement.kind()); - if let Some(owner) = self.data().parent_token() { - let index = self.data().index() as usize; - let green = owner.green(); - let mut leading = green.leading_trivia().to_vec(); - let mut trailing = green.trailing_trivia().to_vec(); - if self.is_leading_trivia(&owner) { - leading[index] = replacement; - } else { - trailing[index] = replacement; - } - let new_owner = GreenToken::with_trivia(green.kind(), green.text(), leading, trailing); - return owner.replace_with(new_owner); - } + assert!(!self.is_trivia(), "cannot replace a trivia view; replace its owning token"); let parent = self.parent().unwrap(); let me: u32 = self.data().index(); @@ -615,6 +703,11 @@ impl SyntaxToken { parent.replace_with(new_parent) } + #[inline] + pub fn is_trivia(&self) -> bool { + self.data().parent().is_some_and(|parent| matches!(parent.green, Green::Token { .. })) + } + #[inline] pub fn kind(&self) -> SyntaxKind { self.data().kind() @@ -631,8 +724,8 @@ impl SyntaxToken { } #[inline] - pub fn index(&self) -> usize { - self.data().index() as usize + pub fn index(&self) -> Option { + (!self.is_trivia()).then(|| self.data().index() as usize) } #[inline] @@ -651,7 +744,7 @@ impl SyntaxToken { } pub fn text_including_trivia(&self) -> String { - self.with_trivia().map(|token| token.text().to_owned()).collect() + self.tokens_including_trivia().map(|trivia| trivia.text().to_owned()).collect() } #[inline] @@ -669,6 +762,19 @@ impl SyntaxToken { std::iter::successors(self.parent(), SyntaxNode::parent) } + pub fn owning_node(&self) -> Option { + if !self.is_trivia() { + return self.parent(); + } + match (self.prev_non_trivia_token(), self.next_non_trivia_token()) { + (Some(prev), Some(next)) => { + prev.ancestors().find(|it| it.text_range().contains_range(next.text_range())) + } + (Some(other), None) | (None, Some(other)) => other.ancestors().last(), + (None, None) => None, + } + } + #[inline] pub fn tree_top(&self) -> SyntaxNode { self.ancestors().last().unwrap() @@ -694,13 +800,15 @@ impl SyntaxToken { } pub fn next_token(&self) -> Option { - if let Some(parent) = self.data().parent_token() { - let index = self.index() + 1; - return if self.is_leading_trivia(&parent) { - parent.leading_trivia().nth(index).or(Some(parent)) + if let (Some(leading), Some(owner)) = + (self.data().in_leading_trivia(), self.data().parent_token()) + { + let index = self.data().index() as usize; + return if leading { + owner.leading_trivia().nth(index + 1).or(Some(owner)) } else { - parent.trailing_trivia().nth(index).or_else(|| { - parent.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia) + owner.trailing_trivia().nth(index + 1).or_else(|| { + owner.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia) }) }; } @@ -709,14 +817,16 @@ impl SyntaxToken { .or_else(|| self.next_non_trivia_token().map(SyntaxToken::first_token_including_trivia)) } pub fn prev_token(&self) -> Option { - if let Some(parent) = self.data().parent_token() { - let index = self.index().checked_sub(1); - return if self.is_leading_trivia(&parent) { - index.and_then(|it| parent.leading_trivia().nth(it)).or_else(|| { - parent.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia) + if let (Some(leading), Some(owner)) = + (self.data().in_leading_trivia(), self.data().parent_token()) + { + let index = (self.data().index() as usize).checked_sub(1); + return if leading { + index.and_then(|it| owner.leading_trivia().nth(it)).or_else(|| { + owner.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia) }) } else { - index.and_then(|it| parent.trailing_trivia().nth(it)).or(Some(parent)) + index.and_then(|it| owner.trailing_trivia().nth(it)).or(Some(owner)) }; } self.leading_trivia() @@ -724,23 +834,24 @@ impl SyntaxToken { .or_else(|| self.prev_non_trivia_token().map(SyntaxToken::last_token_including_trivia)) } - fn next_non_trivia_token(&self) -> Option { - match self.next_sibling_or_token() { - Some(element) => element.first_token(), - None => self - .ancestors() - .find_map(|it| it.next_sibling_or_token()) - .and_then(|element| element.first_token()), - } + pub fn next_non_trivia_token(&self) -> Option { + self.data().next_non_trivia_token() } - fn prev_non_trivia_token(&self) -> Option { - match self.prev_sibling_or_token() { - Some(element) => element.last_token(), - None => self - .ancestors() - .find_map(|it| it.prev_sibling_or_token()) - .and_then(|element| element.last_token()), - } + + pub fn prev_non_trivia_token(&self) -> Option { + self.data().prev_non_trivia_token() + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + debug_assert!(!self.is_trivia(), "trivia_before is defined for non-trivia tokens"); + let prev = self.prev_non_trivia_token().map(|it| it.trailing_trivia()); + prev.into_iter().flatten().chain(self.leading_trivia()) + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + debug_assert!(!self.is_trivia(), "trivia_after is defined for non-trivia tokens"); + let next = self.next_non_trivia_token().map(|it| it.leading_trivia()); + self.trailing_trivia().chain(next.into_iter().flatten()) } fn first_token_including_trivia(self) -> SyntaxToken { @@ -752,16 +863,12 @@ impl SyntaxToken { last.unwrap_or(self) } - fn is_leading_trivia(&self, parent: &SyntaxToken) -> bool { - self.data().offset() < parent.text_range().start() - } - - pub(crate) fn with_trivia(&self) -> impl Iterator { + pub(crate) fn tokens_including_trivia(&self) -> impl DoubleEndedIterator { self.leading_trivia().chain(iter::once(self.clone())).chain(self.trailing_trivia()) } fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { - let mut tokens = self.with_trivia().filter(|token| { + let mut tokens = self.tokens_including_trivia().filter(|token| { let range = token.text_range(); !range.is_empty() && range.start() <= offset && offset <= range.end() }); @@ -777,34 +884,24 @@ impl SyntaxToken { pub fn leading_trivia( &self, ) -> impl DoubleEndedIterator + ExactSizeIterator { - self.trivia(true) + let owner = self.clone(); + let start = self.data().offset(); + (0..self.green().leading_trivia().len()).map(move |index| { + let trivia = owner.green().leading_trivia(); + let offset = start + trivia[..index].iter().map(|it| it.text_len()).sum::(); + SyntaxToken::new_trivia(&trivia[index], owner.clone(), index as u32, offset) + }) } pub fn trailing_trivia( &self, ) -> impl DoubleEndedIterator + ExactSizeIterator { - self.trivia(false) - } - - fn trivia( - &self, - leading: bool, - ) -> impl DoubleEndedIterator + ExactSizeIterator { - let green = self.green(); - let (start, len) = if leading { - (self.data().offset(), green.leading_trivia().len()) - } else { - (self.text_range().end(), green.trailing_trivia().len()) - }; - let token = self.clone(); - (0..len).map(move |index| { - let trivia = if leading { - token.green().leading_trivia() - } else { - token.green().trailing_trivia() - }; + let owner = self.clone(); + let start = self.text_range().end(); + (0..self.green().trailing_trivia().len()).map(move |index| { + let trivia = owner.green().trailing_trivia(); let offset = start + trivia[..index].iter().map(|it| it.text_len()).sum::(); - SyntaxToken::new_trivia(&trivia[index], token.clone(), index as u32, offset) + SyntaxToken::new_trivia(&trivia[index], owner.clone(), index as u32, offset) }) } } @@ -835,7 +932,7 @@ impl SyntaxElement { } #[inline] - fn text_range_including_trivia(&self) -> TextRange { + pub fn text_range_including_trivia(&self) -> TextRange { match self { NodeOrToken::Node(it) => it.text_range(), NodeOrToken::Token(it) => it.text_range_including_trivia(), @@ -843,9 +940,17 @@ impl SyntaxElement { } #[inline] - pub fn index(&self) -> usize { + pub fn text_range_without_outer_trivia(&self) -> TextRange { match self { - NodeOrToken::Node(it) => it.index(), + NodeOrToken::Node(it) => it.text_range_without_outer_trivia(), + NodeOrToken::Token(it) => it.text_range(), + } + } + + #[inline] + pub fn index(&self) -> Option { + match self { + NodeOrToken::Node(it) => Some(it.index()), NodeOrToken::Token(it) => it.index(), } } @@ -886,12 +991,27 @@ impl SyntaxElement { pub fn first_token(&self) -> Option { match self { NodeOrToken::Node(it) => it.first_token(), - NodeOrToken::Token(it) => Some(it.clone()), + NodeOrToken::Token(it) => Some(it.clone().first_token_including_trivia()), } } + pub fn last_token(&self) -> Option { match self { NodeOrToken::Node(it) => it.last_token(), + NodeOrToken::Token(it) => Some(it.clone().last_token_including_trivia()), + } + } + + pub fn first_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.first_non_trivia_token(), + NodeOrToken::Token(it) => Some(it.clone()), + } + } + + pub fn last_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.last_non_trivia_token(), NodeOrToken::Token(it) => Some(it.clone()), } } @@ -909,6 +1029,34 @@ impl SyntaxElement { } } + pub fn next_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.next_non_trivia_token(), + NodeOrToken::Token(it) => it.next_non_trivia_token(), + } + } + pub fn prev_non_trivia_token(&self) -> Option { + match self { + NodeOrToken::Node(it) => it.prev_non_trivia_token(), + NodeOrToken::Token(it) => it.prev_non_trivia_token(), + } + } + + pub fn is_trivia(&self) -> bool { + match self { + NodeOrToken::Node(_) => false, + NodeOrToken::Token(it) => it.is_trivia(), + } + } + + pub fn trivia_before(&self) -> impl DoubleEndedIterator { + self.first_non_trivia_token().map(|it| it.trivia_before()).into_iter().flatten() + } + + pub fn trivia_after(&self) -> impl DoubleEndedIterator { + self.last_non_trivia_token().map(|it| it.trivia_after()).into_iter().flatten() + } + fn token_at_offset(&self, offset: TextSize) -> TokenAtOffset { let range = self.text_range_including_trivia(); assert!(range.start() <= offset && offset <= range.end()); diff --git a/src/syntax_text.rs b/src/syntax_text.rs index abe6d49c..1aac20bc 100644 --- a/src/syntax_text.rs +++ b/src/syntax_text.rs @@ -108,7 +108,7 @@ impl SyntaxText { self.node .descendants_with_tokens() .filter_map(|element| element.into_token()) - .flat_map(|token| token.with_trivia()) + .flat_map(|token| token.tokens_including_trivia()) .filter_map(move |token| { let token_range = token.text_range(); let range = text_range.intersect(token_range)?;