From e608b73d1a8635564daa5605564a33ee1cee2e31 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Fri, 14 Jul 2023 20:45:31 -0400 Subject: [PATCH] Implement all-token iteration. Radio targets are now being properly detected and they trigger re-parses but the tests do not yet pass. --- src/parser/document.rs | 24 +---------- src/parser/token.rs | 91 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 22 deletions(-) diff --git a/src/parser/document.rs b/src/parser/document.rs index 719b53b..9a08331 100644 --- a/src/parser/document.rs +++ b/src/parser/document.rs @@ -18,6 +18,7 @@ use super::element::Element; use super::object::Object; use super::parser_with_context::parser_with_context; use super::source::Source; +use super::token::AllTokensIterator; use super::token::Token; use super::util::exit_matcher_parser; use super::util::get_consumed; @@ -286,27 +287,6 @@ fn headline_end<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s st impl<'s> Document<'s> { pub fn iter_tokens<'r>(&'r self) -> impl Iterator> { - self.zeroth_section - .iter() - .map(Token::Section) - .chain(self.children.iter().map(Token::Heading)) - } -} - -impl<'s> Heading<'s> { - pub fn iter_tokens<'r>(&'r self) -> impl Iterator> { - self.title - .iter() - .map(Token::Object) - .chain(self.children.iter().map(|de| match de { - DocumentElement::Heading(obj) => Token::Heading(obj), - DocumentElement::Section(obj) => Token::Section(obj), - })) - } -} - -impl<'s> Section<'s> { - pub fn iter_tokens<'r>(&'r self) -> impl Iterator> { - self.children.iter().map(Token::Element) + AllTokensIterator::new(Token::Document(self)) } } diff --git a/src/parser/token.rs b/src/parser/token.rs index 5dc7e20..940697e 100644 --- a/src/parser/token.rs +++ b/src/parser/token.rs @@ -1,3 +1,5 @@ +use std::collections::VecDeque; + use super::Document; use super::Element; use super::Heading; @@ -83,4 +85,93 @@ impl<'r, 's> Token<'r, 's> { Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)), } } + + pub fn all_tokens_no_order(&self) -> Box> + '_> { + match self { + Token::Document(document) => Box::new( + document + .zeroth_section + .iter() + .map(Token::Section) + .chain(document.children.iter().map(Token::Heading)), + ), + Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain( + heading.children.iter().map(|de| match de { + DocumentElement::Heading(ref obj) => Token::Heading(obj), + DocumentElement::Section(ref obj) => Token::Section(obj), + }), + )), + Token::Section(section) => Box::new(section.children.iter().map(Token::Element)), + Token::Object(obj) => match obj { + Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Code(_) => Box::new(std::iter::empty()), + Object::Verbatim(_) => Box::new(std::iter::empty()), + Object::PlainText(_) => Box::new(std::iter::empty()), + Object::RegularLink(_) => Box::new(std::iter::empty()), + Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::PlainLink(_) => Box::new(std::iter::empty()), + Object::AngleLink(_) => Box::new(std::iter::empty()), + Object::OrgMacro(_) => Box::new(std::iter::empty()), + }, + Token::Element(elem) => match elem { + Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::PlainList(inner) => { + Box::new(inner.children.iter().map(Token::PlainListItem)) + } + Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::FootnoteDefinition(inner) => { + Box::new(inner.children.iter().map(Token::Element)) + } + Element::Comment(_) => Box::new(std::iter::empty()), + Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::PropertyDrawer(_) => Box::new(std::iter::empty()), + Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)), + Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::CommentBlock(_) => Box::new(std::iter::empty()), + Element::ExampleBlock(_) => Box::new(std::iter::empty()), + Element::ExportBlock(_) => Box::new(std::iter::empty()), + Element::SrcBlock(_) => Box::new(std::iter::empty()), + Element::Clock(_) => Box::new(std::iter::empty()), + Element::DiarySexp(_) => Box::new(std::iter::empty()), + Element::Planning(_) => Box::new(std::iter::empty()), + Element::FixedWidthArea(_) => Box::new(std::iter::empty()), + Element::HorizontalRule(_) => Box::new(std::iter::empty()), + Element::Keyword(_) => Box::new(std::iter::empty()), + Element::LatexEnvironment(_) => Box::new(std::iter::empty()), + }, + Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)), + Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)), + Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)), + } + } +} + +pub struct AllTokensIterator<'r, 's> { + queued_tokens: VecDeque>, +} + +impl<'r, 's> AllTokensIterator<'r, 's> { + pub fn new(tkn: Token<'r, 's>) -> Self { + let mut queued_tokens = VecDeque::new(); + queued_tokens.push_back(tkn); + AllTokensIterator { queued_tokens } + } +} + +impl<'r, 's> Iterator for AllTokensIterator<'r, 's> { + type Item = Token<'r, 's>; + + fn next(&mut self) -> Option { + let next_token = match self.queued_tokens.pop_front() { + Some(tkn) => tkn, + None => return None, + }; + self.queued_tokens.extend(next_token.iter_tokens()); + Some(next_token) + } }