Implement all-token iteration.

Radio targets are now being properly detected and they trigger re-parses but the tests do not yet pass.

src/parser/document.rs

@@ -18,6 +18,7 @@ use super::element::Element;
 use super::object::Object;
 use super::parser_with_context::parser_with_context;
 use super::source::Source;
+use super::token::AllTokensIterator;
 use super::token::Token;
 use super::util::exit_matcher_parser;
 use super::util::get_consumed;
@@ -286,27 +287,6 @@ fn headline_end<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s st
 
 impl<'s> Document<'s> {
     pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
-        self.zeroth_section
+        AllTokensIterator::new(Token::Document(self))
-            .iter()
-            .map(Token::Section)
-            .chain(self.children.iter().map(Token::Heading))
-    }
-}
-
-impl<'s> Heading<'s> {
-    pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
-        self.title
-            .iter()
-            .map(Token::Object)
-            .chain(self.children.iter().map(|de| match de {
-                DocumentElement::Heading(obj) => Token::Heading(obj),
-                DocumentElement::Section(obj) => Token::Section(obj),
-            }))
-    }
-}
-
-impl<'s> Section<'s> {
-    pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
-        self.children.iter().map(Token::Element)
     }
 }

src/parser/token.rs

@@ -1,3 +1,5 @@
+use std::collections::VecDeque;
+
 use super::Document;
 use super::Element;
 use super::Heading;
@@ -83,4 +85,93 @@ impl<'r, 's> Token<'r, 's> {
             Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
         }
     }
+
+    pub fn all_tokens_no_order(&self) -> Box<dyn Iterator<Item = Token<'r, 's>> + '_> {
+        match self {
+            Token::Document(document) => Box::new(
+                document
+                    .zeroth_section
+                    .iter()
+                    .map(Token::Section)
+                    .chain(document.children.iter().map(Token::Heading)),
+            ),
+            Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain(
+                heading.children.iter().map(|de| match de {
+                    DocumentElement::Heading(ref obj) => Token::Heading(obj),
+                    DocumentElement::Section(ref obj) => Token::Section(obj),
+                }),
+            )),
+            Token::Section(section) => Box::new(section.children.iter().map(Token::Element)),
+            Token::Object(obj) => match obj {
+                Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Code(_) => Box::new(std::iter::empty()),
+                Object::Verbatim(_) => Box::new(std::iter::empty()),
+                Object::PlainText(_) => Box::new(std::iter::empty()),
+                Object::RegularLink(_) => Box::new(std::iter::empty()),
+                Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::PlainLink(_) => Box::new(std::iter::empty()),
+                Object::AngleLink(_) => Box::new(std::iter::empty()),
+                Object::OrgMacro(_) => Box::new(std::iter::empty()),
+            },
+            Token::Element(elem) => match elem {
+                Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::PlainList(inner) => {
+                    Box::new(inner.children.iter().map(Token::PlainListItem))
+                }
+                Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::FootnoteDefinition(inner) => {
+                    Box::new(inner.children.iter().map(Token::Element))
+                }
+                Element::Comment(_) => Box::new(std::iter::empty()),
+                Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
+                Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
+                Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::CommentBlock(_) => Box::new(std::iter::empty()),
+                Element::ExampleBlock(_) => Box::new(std::iter::empty()),
+                Element::ExportBlock(_) => Box::new(std::iter::empty()),
+                Element::SrcBlock(_) => Box::new(std::iter::empty()),
+                Element::Clock(_) => Box::new(std::iter::empty()),
+                Element::DiarySexp(_) => Box::new(std::iter::empty()),
+                Element::Planning(_) => Box::new(std::iter::empty()),
+                Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
+                Element::HorizontalRule(_) => Box::new(std::iter::empty()),
+                Element::Keyword(_) => Box::new(std::iter::empty()),
+                Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
+            },
+            Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
+            Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
+            Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
+        }
+    }
+}
+
+pub struct AllTokensIterator<'r, 's> {
+    queued_tokens: VecDeque<Token<'r, 's>>,
+}
+
+impl<'r, 's> AllTokensIterator<'r, 's> {
+    pub fn new(tkn: Token<'r, 's>) -> Self {
+        let mut queued_tokens = VecDeque::new();
+        queued_tokens.push_back(tkn);
+        AllTokensIterator { queued_tokens }
+    }
+}
+
+impl<'r, 's> Iterator for AllTokensIterator<'r, 's> {
+    type Item = Token<'r, 's>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next_token = match self.queued_tokens.pop_front() {
+            Some(tkn) => tkn,
+            None => return None,
+        };
+        self.queued_tokens.extend(next_token.iter_tokens());
+        Some(next_token)
+    }
 }