Merge branch 'radio_link'

build.rs

@@ -79,8 +79,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "element_container_priority_greater_block_greater_block" => Some("Need to implement subscript."),
         "element_container_priority_section_greater_block" => Some("Need to implement subscript."),
         "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "radio_link_before_and_after" => Some("Matching the contents of radio targets not yet implemented."),
-        "radio_link_simple" => Some("Matching the contents of radio targets not yet implemented."),
         _ => None,
     }
 }

org_mode_samples/radio_link/identical_or_semantically_identical.org

@@ -0,0 +1 @@
+alpha *bar* baz foo <<<*bar*  baz>>> lorem ipsum *bar* baz dolar.

src/parser/document.rs

@@ -18,6 +18,8 @@ use super::element::Element;
 use super::object::Object;
 use super::parser_with_context::parser_with_context;
 use super::source::Source;
+use super::token::AllTokensIterator;
+use super::token::Token;
 use super::util::exit_matcher_parser;
 use super::util::get_consumed;
 use super::util::start_of_line;
@@ -95,8 +97,35 @@ pub fn document(input: &str) -> Res<&str, Document> {
     let initial_context: ContextTree<'_, '_> = ContextTree::new();
     let document_context =
         initial_context.with_additional_node(ContextElement::DocumentRoot(input));
-    let zeroth_section_matcher = parser_with_context!(zeroth_section)(&document_context);
-    let heading_matcher = parser_with_context!(heading)(&document_context);
+    let (remaining, document) = _document(&document_context, input)?;
+    {
+        // If there are radio targets in this document then we need to parse the entire document again with the knowledge of the radio targets.
+        let all_radio_targets: Vec<&Vec<Object<'_>>> = document
+            .iter_tokens()
+            .filter_map(|tkn| match tkn {
+                Token::Object(obj) => Some(obj),
+                _ => None,
+            })
+            .filter_map(|obj| match obj {
+                Object::RadioTarget(rt) => Some(rt),
+                _ => None,
+            })
+            .map(|rt| &rt.children)
+            .collect();
+        if !all_radio_targets.is_empty() {
+            let document_context = document_context
+                .with_additional_node(ContextElement::RadioTarget(all_radio_targets));
+            let (remaining, document) = _document(&document_context, input)?;
+            return Ok((remaining, document));
+        }
+    }
+    Ok((remaining, document))
+}
+
+#[tracing::instrument(ret, level = "debug")]
+pub fn _document<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Document<'s>> {
+    let zeroth_section_matcher = parser_with_context!(zeroth_section)(context);
+    let heading_matcher = parser_with_context!(heading)(context);
     let (remaining, _blank_lines) = many0(blank_line)(input)?;
     let (remaining, zeroth_section) = opt(zeroth_section_matcher)(remaining)?;
     let (remaining, children) = many0(heading_matcher)(remaining)?;
@@ -255,3 +284,9 @@ fn headline<'r, 's>(
 fn headline_end<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     line_ending(input)
 }
+
+impl<'s> Document<'s> {
+    pub fn iter_tokens<'r>(&'r self) -> impl Iterator<Item = Token<'r, 's>> {
+        AllTokensIterator::new(Token::Document(self))
+    }
+}

src/parser/element.rs

@@ -20,6 +20,7 @@ use super::lesser_element::SrcBlock;
 use super::lesser_element::VerseBlock;
 use super::source::SetSource;
 use super::source::Source;
+use super::token::Token;
 use super::Drawer;
 
 #[derive(Debug)]

src/parser/mod.rs

@@ -35,6 +35,7 @@ pub mod sexp;
 mod source;
 mod table;
 mod text_markup;
+mod token;
 mod util;
 pub use document::document;
 pub use document::Document;

src/parser/parser_context.rs

@@ -6,6 +6,7 @@ use nom::IResult;
 use super::list::List;
 use super::list::Node;
 use super::Context;
+use super::Object;
 use crate::error::CustomError;
 use crate::error::MyError;
 use crate::error::Res;
@@ -133,6 +134,13 @@ pub enum ContextElement<'r, 's> {
 
     /// Indicates if elements should consume the whitespace after them.
     ConsumeTrailingWhitespace(bool),
+
+    /// The contents of a radio target.
+    ///
+    /// If any are found, this will force a 2nd parse through the
+    /// org-mode document since text needs to be re-parsed to look for
+    /// radio links matching the contents of radio targets.
+    RadioTarget(Vec<&'r Vec<Object<'s>>>),
 }
 
 pub struct ExitMatcherNode<'r> {

src/parser/plain_text.rs

@@ -1,12 +1,16 @@
 use nom::branch::alt;
+use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
+use nom::combinator::map;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::object::PlainText;
+use super::radio_link::RematchObject;
 use super::Context;
+use super::Object;
 use crate::error::Res;
 use crate::parser::object_parser::any_object_except_plain_text;
 use crate::parser::parser_with_context::parser_with_context;
@@ -33,12 +37,24 @@ fn plain_text_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
     recognize(parser_with_context!(any_object_except_plain_text)(context))(input)
 }
 
+impl<'x> RematchObject<'x> for PlainText<'x> {
+    #[tracing::instrument(ret, level = "debug")]
+    fn rematch_object<'r, 's>(
+        &'x self,
+        _context: Context<'r, 's>,
+        input: &'s str,
+    ) -> Res<&'s str, Object<'s>> {
+        map(tag(self.source), |s| {
+            Object::PlainText(PlainText { source: s })
+        })(input)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use nom::combinator::map;
 
     use super::*;
-    use crate::parser::object::Object;
     use crate::parser::parser_context::ContextElement;
     use crate::parser::parser_context::ContextTree;
     use crate::parser::parser_with_context::parser_with_context;

src/parser/radio_link.rs

@@ -6,6 +6,9 @@ use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::Context;
+use super::Object;
+use crate::error::CustomError;
+use crate::error::MyError;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
 use crate::parser::object_parser::minimal_set_object;
@@ -14,14 +17,66 @@ use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
-use crate::parser::util::not_yet_implemented;
 use crate::parser::RadioLink;
 use crate::parser::RadioTarget;
 
 #[tracing::instrument(ret, level = "debug")]
 pub fn radio_link<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, RadioLink<'s>> {
-    not_yet_implemented()?;
-    todo!();
+    let radio_targets = context
+        .iter()
+        .filter_map(|context_element| match context_element.get_data() {
+            ContextElement::RadioTarget(targets) => Some(targets),
+            _ => None,
+        })
+        .flatten();
+    for radio_target in radio_targets {
+        let rematched_target = rematch_target(context, radio_target, input);
+        if let Ok((remaining, rematched_target)) = rematched_target {
+            let (remaining, _) = space0(remaining)?;
+            let source = get_consumed(input, remaining);
+            return Ok((
+                remaining,
+                RadioLink {
+                    source,
+                    children: rematched_target,
+                },
+            ));
+        }
+    }
+    Err(nom::Err::Error(CustomError::MyError(MyError(
+        "NoRadioLink",
+    ))))
+}
+
+#[tracing::instrument(ret, level = "debug")]
+pub fn rematch_target<'x, 'r, 's>(
+    context: Context<'r, 's>,
+    target: &'x Vec<Object<'x>>,
+    input: &'s str,
+) -> Res<&'s str, Vec<Object<'s>>> {
+    let mut remaining = input;
+    let mut new_matches = Vec::with_capacity(target.len());
+    for original_object in target {
+        match original_object {
+            // TODO: The rest of the minimal set of objects.
+            Object::Bold(bold) => {
+                let (new_remaining, new_match) = bold.rematch_object(context, remaining)?;
+                remaining = new_remaining;
+                new_matches.push(new_match);
+            }
+            Object::PlainText(plaintext) => {
+                let (new_remaining, new_match) = plaintext.rematch_object(context, remaining)?;
+                remaining = new_remaining;
+                new_matches.push(new_match);
+            }
+            _ => {
+                return Err(nom::Err::Error(CustomError::MyError(MyError(
+                    "OnlyMinimalSetObjectsAllowed",
+                ))));
+            }
+        };
+    }
+    Ok((remaining, new_matches))
 }
 
 #[tracing::instrument(ret, level = "debug")]
@@ -54,3 +109,87 @@ pub fn radio_target<'r, 's>(
 fn radio_target_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     alt((tag("<"), tag(">"), line_ending))(input)
 }
+
+pub trait RematchObject<'x> {
+    fn rematch_object<'r, 's>(
+        &'x self,
+        context: Context<'r, 's>,
+        input: &'s str,
+    ) -> Res<&'s str, Object<'s>>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::parser::element_parser::element;
+    use crate::parser::parser_context::ContextElement;
+    use crate::parser::parser_context::ContextTree;
+    use crate::parser::parser_with_context::parser_with_context;
+    use crate::parser::source::Source;
+    use crate::parser::Bold;
+    use crate::parser::PlainText;
+
+    #[test]
+    fn plain_text_radio_target() {
+        let input = "foo bar baz";
+        let radio_target_match = vec![Object::PlainText(PlainText { source: "bar" })];
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let document_context = initial_context
+            .with_additional_node(ContextElement::DocumentRoot(input))
+            .with_additional_node(ContextElement::RadioTarget(vec![&radio_target_match]));
+        let paragraph_matcher = parser_with_context!(element(true))(&document_context);
+        let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph");
+        let first_paragraph = match first_paragraph {
+            crate::parser::Element::Paragraph(paragraph) => paragraph,
+            _ => panic!("Should be a paragraph!"),
+        };
+        assert_eq!(remaining, "");
+        assert_eq!(first_paragraph.get_source(), "foo bar baz");
+        assert_eq!(first_paragraph.children.len(), 3);
+        assert_eq!(
+            first_paragraph
+                .children
+                .get(1)
+                .expect("Len already asserted to be 3"),
+            &Object::RadioLink(RadioLink {
+                source: "bar ",
+                children: vec![Object::PlainText(PlainText { source: "bar" })]
+            })
+        );
+    }
+
+    #[test]
+    fn bold_radio_target() {
+        let input = "foo *bar* baz";
+        let radio_target_match = vec![Object::Bold(Bold {
+            source: "*bar*",
+            children: vec![Object::PlainText(PlainText { source: "bar" })],
+        })];
+        let initial_context: ContextTree<'_, '_> = ContextTree::new();
+        let document_context = initial_context
+            .with_additional_node(ContextElement::DocumentRoot(input))
+            .with_additional_node(ContextElement::RadioTarget(vec![&radio_target_match]));
+        let paragraph_matcher = parser_with_context!(element(true))(&document_context);
+        let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph");
+        let first_paragraph = match first_paragraph {
+            crate::parser::Element::Paragraph(paragraph) => paragraph,
+            _ => panic!("Should be a paragraph!"),
+        };
+        assert_eq!(remaining, "");
+        assert_eq!(first_paragraph.get_source(), "foo *bar* baz");
+        assert_eq!(first_paragraph.children.len(), 3);
+        assert_eq!(
+            first_paragraph
+                .children
+                .get(1)
+                .expect("Len already asserted to be 3"),
+            &Object::RadioLink(RadioLink {
+                source: "*bar* ",
+                children: vec![Object::Bold(Bold {
+                    source: "*bar* ",
+                    children: vec![Object::PlainText(PlainText { source: "bar" })]
+                })]
+            })
+        );
+    }
+}

src/parser/text_markup.rs

@@ -12,8 +12,10 @@ use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many_till;
 use nom::sequence::terminated;
+use nom::sequence::tuple;
 use tracing::span;
 
+use super::radio_link::RematchObject;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -23,6 +25,7 @@ use crate::parser::object_parser::standard_set_object;
 use crate::parser::parser_context::ContextElement;
 use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
+use crate::parser::radio_link::rematch_target;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
 use crate::parser::util::get_one_before;
@@ -142,7 +145,6 @@ fn _text_markup_object<'r, 's, 'x>(
         }
     }
 
-    // TODO: Sometimes its plain text, not objects
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
     let (remaining, _trailing_whitespace) = space0(remaining)?;
     Ok((remaining, children))
@@ -189,7 +191,6 @@ fn _text_markup_string<'r, 's, 'x>(
         }
     }
 
-    // TODO: Sometimes its plain text, not objects
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
     let (remaining, _trailing_whitespace) = space0(remaining)?;
     Ok((remaining, contents))
@@ -204,7 +205,7 @@ pub fn pre<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()>
     match preceding_character {
         // If None, we are at the start of the file which is technically the beginning of a line.
         None | Some('\r') | Some('\n') | Some(' ') | Some('\t') | Some('-') | Some('(')
-        | Some('{') | Some('\'') | Some('"') => {}
+        | Some('{') | Some('\'') | Some('"') | Some('<') => {}
         Some(_) => {
             // Not at start of line, cannot be a heading
             return Err(nom::Err::Error(CustomError::MyError(MyError(
@@ -217,7 +218,7 @@ pub fn pre<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()>
 
 #[tracing::instrument(ret, level = "debug")]
 pub fn post<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> {
-    let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\"")), line_ending))(input)?;
+    let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\">")), line_ending))(input)?;
     Ok((remaining, ()))
 }
 
@@ -242,3 +243,53 @@ fn _text_markup_end<'r, 's, 'x>(
     let source = get_consumed(input, remaining);
     Ok((remaining, source))
 }
+
+impl<'x> RematchObject<'x> for Bold<'x> {
+    #[tracing::instrument(ret, level = "debug")]
+    fn rematch_object<'r, 's>(
+        &'x self,
+        _context: Context<'r, 's>,
+        input: &'s str,
+    ) -> Res<&'s str, Object<'s>> {
+        let (remaining, children) =
+            _rematch_text_markup_object(_context, input, "*", &self.children)?;
+        let source = get_consumed(input, remaining);
+        Ok((remaining, Object::Bold(Bold { source, children })))
+    }
+}
+
+#[tracing::instrument(ret, level = "debug")]
+fn _rematch_text_markup_object<'r, 's, 'x>(
+    context: Context<'r, 's>,
+    input: &'s str,
+    marker_symbol: &'static str,
+    original_match_children: &'x Vec<Object<'x>>,
+) -> Res<&'s str, Vec<Object<'s>>> {
+    let (remaining, _) = pre(context, input)?;
+    let (remaining, open) = tag(marker_symbol)(remaining)?;
+    let (remaining, _peek_not_whitespace) = peek(not(multispace1))(remaining)?;
+    let text_markup_end_specialized = text_markup_end(open);
+    let parser_context =
+        context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+            class: ExitClass::Beta,
+            exit_matcher: &text_markup_end_specialized,
+        }));
+
+    let (remaining, children) =
+        // TODO: This doesn't really check the exit matcher between each object. I think it may be possible to construct an org document that parses incorrectly with the current code.
+        rematch_target(&parser_context, original_match_children, remaining)?;
+
+    {
+        let span = span!(tracing::Level::DEBUG, "Checking parent exit.");
+        let _enter = span.enter();
+        if exit_matcher_parser(context, remaining).is_ok() {
+            return Err(nom::Err::Error(CustomError::MyError(MyError(
+                "Parent exit matcher is triggering.",
+            ))));
+        }
+    }
+
+    let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
+    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    Ok((remaining, children))
+}

src/parser/token.rs

@@ -0,0 +1,177 @@
+use std::collections::VecDeque;
+
+use super::Document;
+use super::Element;
+use super::Heading;
+use super::Object;
+use super::PlainListItem;
+use super::Section;
+use super::TableCell;
+use super::TableRow;
+use crate::parser::DocumentElement;
+
+pub enum Token<'r, 's> {
+    Document(&'r Document<'s>),
+    Heading(&'r Heading<'s>),
+    Section(&'r Section<'s>),
+    Object(&'r Object<'s>),
+    Element(&'r Element<'s>),
+    PlainListItem(&'r PlainListItem<'s>),
+    TableRow(&'r TableRow<'s>),
+    TableCell(&'r TableCell<'s>),
+}
+
+impl<'r, 's> Token<'r, 's> {
+    pub fn iter_tokens(&self) -> Box<dyn Iterator<Item = Token<'r, 's>> + '_> {
+        match self {
+            Token::Document(document) => Box::new(
+                document
+                    .zeroth_section
+                    .iter()
+                    .map(Token::Section)
+                    .chain(document.children.iter().map(Token::Heading)),
+            ),
+            Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain(
+                heading.children.iter().map(|de| match de {
+                    DocumentElement::Heading(ref obj) => Token::Heading(obj),
+                    DocumentElement::Section(ref obj) => Token::Section(obj),
+                }),
+            )),
+            Token::Section(section) => Box::new(section.children.iter().map(Token::Element)),
+            Token::Object(obj) => match obj {
+                Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Code(_) => Box::new(std::iter::empty()),
+                Object::Verbatim(_) => Box::new(std::iter::empty()),
+                Object::PlainText(_) => Box::new(std::iter::empty()),
+                Object::RegularLink(_) => Box::new(std::iter::empty()),
+                Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::PlainLink(_) => Box::new(std::iter::empty()),
+                Object::AngleLink(_) => Box::new(std::iter::empty()),
+                Object::OrgMacro(_) => Box::new(std::iter::empty()),
+            },
+            Token::Element(elem) => match elem {
+                Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::PlainList(inner) => {
+                    Box::new(inner.children.iter().map(Token::PlainListItem))
+                }
+                Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::FootnoteDefinition(inner) => {
+                    Box::new(inner.children.iter().map(Token::Element))
+                }
+                Element::Comment(_) => Box::new(std::iter::empty()),
+                Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
+                Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
+                Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::CommentBlock(_) => Box::new(std::iter::empty()),
+                Element::ExampleBlock(_) => Box::new(std::iter::empty()),
+                Element::ExportBlock(_) => Box::new(std::iter::empty()),
+                Element::SrcBlock(_) => Box::new(std::iter::empty()),
+                Element::Clock(_) => Box::new(std::iter::empty()),
+                Element::DiarySexp(_) => Box::new(std::iter::empty()),
+                Element::Planning(_) => Box::new(std::iter::empty()),
+                Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
+                Element::HorizontalRule(_) => Box::new(std::iter::empty()),
+                Element::Keyword(_) => Box::new(std::iter::empty()),
+                Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
+            },
+            Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
+            Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
+            Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
+        }
+    }
+
+    pub fn all_tokens_no_order(&self) -> Box<dyn Iterator<Item = Token<'r, 's>> + '_> {
+        match self {
+            Token::Document(document) => Box::new(
+                document
+                    .zeroth_section
+                    .iter()
+                    .map(Token::Section)
+                    .chain(document.children.iter().map(Token::Heading)),
+            ),
+            Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain(
+                heading.children.iter().map(|de| match de {
+                    DocumentElement::Heading(ref obj) => Token::Heading(obj),
+                    DocumentElement::Section(ref obj) => Token::Section(obj),
+                }),
+            )),
+            Token::Section(section) => Box::new(section.children.iter().map(Token::Element)),
+            Token::Object(obj) => match obj {
+                Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Code(_) => Box::new(std::iter::empty()),
+                Object::Verbatim(_) => Box::new(std::iter::empty()),
+                Object::PlainText(_) => Box::new(std::iter::empty()),
+                Object::RegularLink(_) => Box::new(std::iter::empty()),
+                Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::PlainLink(_) => Box::new(std::iter::empty()),
+                Object::AngleLink(_) => Box::new(std::iter::empty()),
+                Object::OrgMacro(_) => Box::new(std::iter::empty()),
+            },
+            Token::Element(elem) => match elem {
+                Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::PlainList(inner) => {
+                    Box::new(inner.children.iter().map(Token::PlainListItem))
+                }
+                Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::FootnoteDefinition(inner) => {
+                    Box::new(inner.children.iter().map(Token::Element))
+                }
+                Element::Comment(_) => Box::new(std::iter::empty()),
+                Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
+                Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
+                Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::CommentBlock(_) => Box::new(std::iter::empty()),
+                Element::ExampleBlock(_) => Box::new(std::iter::empty()),
+                Element::ExportBlock(_) => Box::new(std::iter::empty()),
+                Element::SrcBlock(_) => Box::new(std::iter::empty()),
+                Element::Clock(_) => Box::new(std::iter::empty()),
+                Element::DiarySexp(_) => Box::new(std::iter::empty()),
+                Element::Planning(_) => Box::new(std::iter::empty()),
+                Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
+                Element::HorizontalRule(_) => Box::new(std::iter::empty()),
+                Element::Keyword(_) => Box::new(std::iter::empty()),
+                Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
+            },
+            Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
+            Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
+            Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
+        }
+    }
+}
+
+pub struct AllTokensIterator<'r, 's> {
+    queued_tokens: VecDeque<Token<'r, 's>>,
+}
+
+impl<'r, 's> AllTokensIterator<'r, 's> {
+    pub fn new(tkn: Token<'r, 's>) -> Self {
+        let mut queued_tokens = VecDeque::new();
+        queued_tokens.push_back(tkn);
+        AllTokensIterator { queued_tokens }
+    }
+}
+
+impl<'r, 's> Iterator for AllTokensIterator<'r, 's> {
+    type Item = Token<'r, 's>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next_token = match self.queued_tokens.pop_front() {
+            Some(tkn) => tkn,
+            None => return None,
+        };
+        self.queued_tokens.extend(next_token.iter_tokens());
+        Some(next_token)
+    }
+}

toy_language.txt

@@ -1,3 +1 @@
-foo *bar /baz *lorem* ipsum/ dolar* alpha
-
-foo *bar /baz _lorem_ ipsum/ dolar* alpha
+foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar.