diff --git a/build.rs b/build.rs index 0d45c7cd..bb99b667 100644 --- a/build.rs +++ b/build.rs @@ -79,8 +79,6 @@ fn is_expect_fail(name: &str) -> Option<&str> { "element_container_priority_greater_block_greater_block" => Some("Need to implement subscript."), "element_container_priority_section_greater_block" => Some("Need to implement subscript."), "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."), - "radio_link_before_and_after" => Some("Matching the contents of radio targets not yet implemented."), - "radio_link_simple" => Some("Matching the contents of radio targets not yet implemented."), _ => None, } } diff --git a/org_mode_samples/radio_link/identical_or_semantically_identical.org b/org_mode_samples/radio_link/identical_or_semantically_identical.org new file mode 100644 index 00000000..a99a24d2 --- /dev/null +++ b/org_mode_samples/radio_link/identical_or_semantically_identical.org @@ -0,0 +1 @@ +alpha *bar* baz foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar. diff --git a/src/parser/document.rs b/src/parser/document.rs index 9954fab2..9a083319 100644 --- a/src/parser/document.rs +++ b/src/parser/document.rs @@ -18,6 +18,8 @@ use super::element::Element; use super::object::Object; use super::parser_with_context::parser_with_context; use super::source::Source; +use super::token::AllTokensIterator; +use super::token::Token; use super::util::exit_matcher_parser; use super::util::get_consumed; use super::util::start_of_line; @@ -95,8 +97,35 @@ pub fn document(input: &str) -> Res<&str, Document> { let initial_context: ContextTree<'_, '_> = ContextTree::new(); let document_context = initial_context.with_additional_node(ContextElement::DocumentRoot(input)); - let zeroth_section_matcher = parser_with_context!(zeroth_section)(&document_context); - let heading_matcher = parser_with_context!(heading)(&document_context); + let (remaining, document) = _document(&document_context, input)?; + { + // If there are radio targets in this document then we need to parse the entire document again with the knowledge of the radio targets. + let all_radio_targets: Vec<&Vec>> = document + .iter_tokens() + .filter_map(|tkn| match tkn { + Token::Object(obj) => Some(obj), + _ => None, + }) + .filter_map(|obj| match obj { + Object::RadioTarget(rt) => Some(rt), + _ => None, + }) + .map(|rt| &rt.children) + .collect(); + if !all_radio_targets.is_empty() { + let document_context = document_context + .with_additional_node(ContextElement::RadioTarget(all_radio_targets)); + let (remaining, document) = _document(&document_context, input)?; + return Ok((remaining, document)); + } + } + Ok((remaining, document)) +} + +#[tracing::instrument(ret, level = "debug")] +pub fn _document<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Document<'s>> { + let zeroth_section_matcher = parser_with_context!(zeroth_section)(context); + let heading_matcher = parser_with_context!(heading)(context); let (remaining, _blank_lines) = many0(blank_line)(input)?; let (remaining, zeroth_section) = opt(zeroth_section_matcher)(remaining)?; let (remaining, children) = many0(heading_matcher)(remaining)?; @@ -255,3 +284,9 @@ fn headline<'r, 's>( fn headline_end<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { line_ending(input) } + +impl<'s> Document<'s> { + pub fn iter_tokens<'r>(&'r self) -> impl Iterator> { + AllTokensIterator::new(Token::Document(self)) + } +} diff --git a/src/parser/element.rs b/src/parser/element.rs index 3052ecef..27efa21c 100644 --- a/src/parser/element.rs +++ b/src/parser/element.rs @@ -20,6 +20,7 @@ use super::lesser_element::SrcBlock; use super::lesser_element::VerseBlock; use super::source::SetSource; use super::source::Source; +use super::token::Token; use super::Drawer; #[derive(Debug)] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index df096538..dcd61c69 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -35,6 +35,7 @@ pub mod sexp; mod source; mod table; mod text_markup; +mod token; mod util; pub use document::document; pub use document::Document; diff --git a/src/parser/parser_context.rs b/src/parser/parser_context.rs index 232db316..c49e0997 100644 --- a/src/parser/parser_context.rs +++ b/src/parser/parser_context.rs @@ -6,6 +6,7 @@ use nom::IResult; use super::list::List; use super::list::Node; use super::Context; +use super::Object; use crate::error::CustomError; use crate::error::MyError; use crate::error::Res; @@ -133,6 +134,13 @@ pub enum ContextElement<'r, 's> { /// Indicates if elements should consume the whitespace after them. ConsumeTrailingWhitespace(bool), + + /// The contents of a radio target. + /// + /// If any are found, this will force a 2nd parse through the + /// org-mode document since text needs to be re-parsed to look for + /// radio links matching the contents of radio targets. + RadioTarget(Vec<&'r Vec>>), } pub struct ExitMatcherNode<'r> { diff --git a/src/parser/plain_text.rs b/src/parser/plain_text.rs index 71e2745d..497535c1 100644 --- a/src/parser/plain_text.rs +++ b/src/parser/plain_text.rs @@ -1,12 +1,16 @@ use nom::branch::alt; +use nom::bytes::complete::tag; use nom::character::complete::anychar; +use nom::combinator::map; use nom::combinator::peek; use nom::combinator::recognize; use nom::combinator::verify; use nom::multi::many_till; use super::object::PlainText; +use super::radio_link::RematchObject; use super::Context; +use super::Object; use crate::error::Res; use crate::parser::object_parser::any_object_except_plain_text; use crate::parser::parser_with_context::parser_with_context; @@ -33,12 +37,24 @@ fn plain_text_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s recognize(parser_with_context!(any_object_except_plain_text)(context))(input) } +impl<'x> RematchObject<'x> for PlainText<'x> { + #[tracing::instrument(ret, level = "debug")] + fn rematch_object<'r, 's>( + &'x self, + _context: Context<'r, 's>, + input: &'s str, + ) -> Res<&'s str, Object<'s>> { + map(tag(self.source), |s| { + Object::PlainText(PlainText { source: s }) + })(input) + } +} + #[cfg(test)] mod tests { use nom::combinator::map; use super::*; - use crate::parser::object::Object; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ContextTree; use crate::parser::parser_with_context::parser_with_context; diff --git a/src/parser/radio_link.rs b/src/parser/radio_link.rs index 890f408e..57fe3f80 100644 --- a/src/parser/radio_link.rs +++ b/src/parser/radio_link.rs @@ -6,6 +6,9 @@ use nom::combinator::verify; use nom::multi::many_till; use super::Context; +use super::Object; +use crate::error::CustomError; +use crate::error::MyError; use crate::error::Res; use crate::parser::exiting::ExitClass; use crate::parser::object_parser::minimal_set_object; @@ -14,14 +17,66 @@ use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; -use crate::parser::util::not_yet_implemented; use crate::parser::RadioLink; use crate::parser::RadioTarget; #[tracing::instrument(ret, level = "debug")] pub fn radio_link<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, RadioLink<'s>> { - not_yet_implemented()?; - todo!(); + let radio_targets = context + .iter() + .filter_map(|context_element| match context_element.get_data() { + ContextElement::RadioTarget(targets) => Some(targets), + _ => None, + }) + .flatten(); + for radio_target in radio_targets { + let rematched_target = rematch_target(context, radio_target, input); + if let Ok((remaining, rematched_target)) = rematched_target { + let (remaining, _) = space0(remaining)?; + let source = get_consumed(input, remaining); + return Ok(( + remaining, + RadioLink { + source, + children: rematched_target, + }, + )); + } + } + Err(nom::Err::Error(CustomError::MyError(MyError( + "NoRadioLink", + )))) +} + +#[tracing::instrument(ret, level = "debug")] +pub fn rematch_target<'x, 'r, 's>( + context: Context<'r, 's>, + target: &'x Vec>, + input: &'s str, +) -> Res<&'s str, Vec>> { + let mut remaining = input; + let mut new_matches = Vec::with_capacity(target.len()); + for original_object in target { + match original_object { + // TODO: The rest of the minimal set of objects. + Object::Bold(bold) => { + let (new_remaining, new_match) = bold.rematch_object(context, remaining)?; + remaining = new_remaining; + new_matches.push(new_match); + } + Object::PlainText(plaintext) => { + let (new_remaining, new_match) = plaintext.rematch_object(context, remaining)?; + remaining = new_remaining; + new_matches.push(new_match); + } + _ => { + return Err(nom::Err::Error(CustomError::MyError(MyError( + "OnlyMinimalSetObjectsAllowed", + )))); + } + }; + } + Ok((remaining, new_matches)) } #[tracing::instrument(ret, level = "debug")] @@ -54,3 +109,87 @@ pub fn radio_target<'r, 's>( fn radio_target_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { alt((tag("<"), tag(">"), line_ending))(input) } + +pub trait RematchObject<'x> { + fn rematch_object<'r, 's>( + &'x self, + context: Context<'r, 's>, + input: &'s str, + ) -> Res<&'s str, Object<'s>>; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::element_parser::element; + use crate::parser::parser_context::ContextElement; + use crate::parser::parser_context::ContextTree; + use crate::parser::parser_with_context::parser_with_context; + use crate::parser::source::Source; + use crate::parser::Bold; + use crate::parser::PlainText; + + #[test] + fn plain_text_radio_target() { + let input = "foo bar baz"; + let radio_target_match = vec![Object::PlainText(PlainText { source: "bar" })]; + let initial_context: ContextTree<'_, '_> = ContextTree::new(); + let document_context = initial_context + .with_additional_node(ContextElement::DocumentRoot(input)) + .with_additional_node(ContextElement::RadioTarget(vec![&radio_target_match])); + let paragraph_matcher = parser_with_context!(element(true))(&document_context); + let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph"); + let first_paragraph = match first_paragraph { + crate::parser::Element::Paragraph(paragraph) => paragraph, + _ => panic!("Should be a paragraph!"), + }; + assert_eq!(remaining, ""); + assert_eq!(first_paragraph.get_source(), "foo bar baz"); + assert_eq!(first_paragraph.children.len(), 3); + assert_eq!( + first_paragraph + .children + .get(1) + .expect("Len already asserted to be 3"), + &Object::RadioLink(RadioLink { + source: "bar ", + children: vec![Object::PlainText(PlainText { source: "bar" })] + }) + ); + } + + #[test] + fn bold_radio_target() { + let input = "foo *bar* baz"; + let radio_target_match = vec![Object::Bold(Bold { + source: "*bar*", + children: vec![Object::PlainText(PlainText { source: "bar" })], + })]; + let initial_context: ContextTree<'_, '_> = ContextTree::new(); + let document_context = initial_context + .with_additional_node(ContextElement::DocumentRoot(input)) + .with_additional_node(ContextElement::RadioTarget(vec![&radio_target_match])); + let paragraph_matcher = parser_with_context!(element(true))(&document_context); + let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph"); + let first_paragraph = match first_paragraph { + crate::parser::Element::Paragraph(paragraph) => paragraph, + _ => panic!("Should be a paragraph!"), + }; + assert_eq!(remaining, ""); + assert_eq!(first_paragraph.get_source(), "foo *bar* baz"); + assert_eq!(first_paragraph.children.len(), 3); + assert_eq!( + first_paragraph + .children + .get(1) + .expect("Len already asserted to be 3"), + &Object::RadioLink(RadioLink { + source: "*bar* ", + children: vec![Object::Bold(Bold { + source: "*bar* ", + children: vec![Object::PlainText(PlainText { source: "bar" })] + })] + }) + ); + } +} diff --git a/src/parser/text_markup.rs b/src/parser/text_markup.rs index a7a94971..f6fb816c 100644 --- a/src/parser/text_markup.rs +++ b/src/parser/text_markup.rs @@ -12,8 +12,10 @@ use nom::combinator::recognize; use nom::combinator::verify; use nom::multi::many_till; use nom::sequence::terminated; +use nom::sequence::tuple; use tracing::span; +use super::radio_link::RematchObject; use super::Context; use crate::error::CustomError; use crate::error::MyError; @@ -23,6 +25,7 @@ use crate::parser::object_parser::standard_set_object; use crate::parser::parser_context::ContextElement; use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; +use crate::parser::radio_link::rematch_target; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; use crate::parser::util::get_one_before; @@ -142,7 +145,6 @@ fn _text_markup_object<'r, 's, 'x>( } } - // TODO: Sometimes its plain text, not objects let (remaining, _close) = text_markup_end_specialized(context, remaining)?; let (remaining, _trailing_whitespace) = space0(remaining)?; Ok((remaining, children)) @@ -189,7 +191,6 @@ fn _text_markup_string<'r, 's, 'x>( } } - // TODO: Sometimes its plain text, not objects let (remaining, _close) = text_markup_end_specialized(context, remaining)?; let (remaining, _trailing_whitespace) = space0(remaining)?; Ok((remaining, contents)) @@ -204,7 +205,7 @@ pub fn pre<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> match preceding_character { // If None, we are at the start of the file which is technically the beginning of a line. None | Some('\r') | Some('\n') | Some(' ') | Some('\t') | Some('-') | Some('(') - | Some('{') | Some('\'') | Some('"') => {} + | Some('{') | Some('\'') | Some('"') | Some('<') => {} Some(_) => { // Not at start of line, cannot be a heading return Err(nom::Err::Error(CustomError::MyError(MyError( @@ -217,7 +218,7 @@ pub fn pre<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> #[tracing::instrument(ret, level = "debug")] pub fn post<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> { - let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\"")), line_ending))(input)?; + let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\">")), line_ending))(input)?; Ok((remaining, ())) } @@ -242,3 +243,53 @@ fn _text_markup_end<'r, 's, 'x>( let source = get_consumed(input, remaining); Ok((remaining, source)) } + +impl<'x> RematchObject<'x> for Bold<'x> { + #[tracing::instrument(ret, level = "debug")] + fn rematch_object<'r, 's>( + &'x self, + _context: Context<'r, 's>, + input: &'s str, + ) -> Res<&'s str, Object<'s>> { + let (remaining, children) = + _rematch_text_markup_object(_context, input, "*", &self.children)?; + let source = get_consumed(input, remaining); + Ok((remaining, Object::Bold(Bold { source, children }))) + } +} + +#[tracing::instrument(ret, level = "debug")] +fn _rematch_text_markup_object<'r, 's, 'x>( + context: Context<'r, 's>, + input: &'s str, + marker_symbol: &'static str, + original_match_children: &'x Vec>, +) -> Res<&'s str, Vec>> { + let (remaining, _) = pre(context, input)?; + let (remaining, open) = tag(marker_symbol)(remaining)?; + let (remaining, _peek_not_whitespace) = peek(not(multispace1))(remaining)?; + let text_markup_end_specialized = text_markup_end(open); + let parser_context = + context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + class: ExitClass::Beta, + exit_matcher: &text_markup_end_specialized, + })); + + let (remaining, children) = + // TODO: This doesn't really check the exit matcher between each object. I think it may be possible to construct an org document that parses incorrectly with the current code. + rematch_target(&parser_context, original_match_children, remaining)?; + + { + let span = span!(tracing::Level::DEBUG, "Checking parent exit."); + let _enter = span.enter(); + if exit_matcher_parser(context, remaining).is_ok() { + return Err(nom::Err::Error(CustomError::MyError(MyError( + "Parent exit matcher is triggering.", + )))); + } + } + + let (remaining, _close) = text_markup_end_specialized(context, remaining)?; + let (remaining, _trailing_whitespace) = space0(remaining)?; + Ok((remaining, children)) +} diff --git a/src/parser/token.rs b/src/parser/token.rs new file mode 100644 index 00000000..940697ec --- /dev/null +++ b/src/parser/token.rs @@ -0,0 +1,177 @@ +use std::collections::VecDeque; + +use super::Document; +use super::Element; +use super::Heading; +use super::Object; +use super::PlainListItem; +use super::Section; +use super::TableCell; +use super::TableRow; +use crate::parser::DocumentElement; + +pub enum Token<'r, 's> { + Document(&'r Document<'s>), + Heading(&'r Heading<'s>), + Section(&'r Section<'s>), + Object(&'r Object<'s>), + Element(&'r Element<'s>), + PlainListItem(&'r PlainListItem<'s>), + TableRow(&'r TableRow<'s>), + TableCell(&'r TableCell<'s>), +} + +impl<'r, 's> Token<'r, 's> { + pub fn iter_tokens(&self) -> Box> + '_> { + match self { + Token::Document(document) => Box::new( + document + .zeroth_section + .iter() + .map(Token::Section) + .chain(document.children.iter().map(Token::Heading)), + ), + Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain( + heading.children.iter().map(|de| match de { + DocumentElement::Heading(ref obj) => Token::Heading(obj), + DocumentElement::Section(ref obj) => Token::Section(obj), + }), + )), + Token::Section(section) => Box::new(section.children.iter().map(Token::Element)), + Token::Object(obj) => match obj { + Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Code(_) => Box::new(std::iter::empty()), + Object::Verbatim(_) => Box::new(std::iter::empty()), + Object::PlainText(_) => Box::new(std::iter::empty()), + Object::RegularLink(_) => Box::new(std::iter::empty()), + Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::PlainLink(_) => Box::new(std::iter::empty()), + Object::AngleLink(_) => Box::new(std::iter::empty()), + Object::OrgMacro(_) => Box::new(std::iter::empty()), + }, + Token::Element(elem) => match elem { + Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::PlainList(inner) => { + Box::new(inner.children.iter().map(Token::PlainListItem)) + } + Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::FootnoteDefinition(inner) => { + Box::new(inner.children.iter().map(Token::Element)) + } + Element::Comment(_) => Box::new(std::iter::empty()), + Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::PropertyDrawer(_) => Box::new(std::iter::empty()), + Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)), + Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::CommentBlock(_) => Box::new(std::iter::empty()), + Element::ExampleBlock(_) => Box::new(std::iter::empty()), + Element::ExportBlock(_) => Box::new(std::iter::empty()), + Element::SrcBlock(_) => Box::new(std::iter::empty()), + Element::Clock(_) => Box::new(std::iter::empty()), + Element::DiarySexp(_) => Box::new(std::iter::empty()), + Element::Planning(_) => Box::new(std::iter::empty()), + Element::FixedWidthArea(_) => Box::new(std::iter::empty()), + Element::HorizontalRule(_) => Box::new(std::iter::empty()), + Element::Keyword(_) => Box::new(std::iter::empty()), + Element::LatexEnvironment(_) => Box::new(std::iter::empty()), + }, + Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)), + Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)), + Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)), + } + } + + pub fn all_tokens_no_order(&self) -> Box> + '_> { + match self { + Token::Document(document) => Box::new( + document + .zeroth_section + .iter() + .map(Token::Section) + .chain(document.children.iter().map(Token::Heading)), + ), + Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain( + heading.children.iter().map(|de| match de { + DocumentElement::Heading(ref obj) => Token::Heading(obj), + DocumentElement::Section(ref obj) => Token::Section(obj), + }), + )), + Token::Section(section) => Box::new(section.children.iter().map(Token::Element)), + Token::Object(obj) => match obj { + Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::Code(_) => Box::new(std::iter::empty()), + Object::Verbatim(_) => Box::new(std::iter::empty()), + Object::PlainText(_) => Box::new(std::iter::empty()), + Object::RegularLink(_) => Box::new(std::iter::empty()), + Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)), + Object::PlainLink(_) => Box::new(std::iter::empty()), + Object::AngleLink(_) => Box::new(std::iter::empty()), + Object::OrgMacro(_) => Box::new(std::iter::empty()), + }, + Token::Element(elem) => match elem { + Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::PlainList(inner) => { + Box::new(inner.children.iter().map(Token::PlainListItem)) + } + Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::FootnoteDefinition(inner) => { + Box::new(inner.children.iter().map(Token::Element)) + } + Element::Comment(_) => Box::new(std::iter::empty()), + Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)), + Element::PropertyDrawer(_) => Box::new(std::iter::empty()), + Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)), + Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)), + Element::CommentBlock(_) => Box::new(std::iter::empty()), + Element::ExampleBlock(_) => Box::new(std::iter::empty()), + Element::ExportBlock(_) => Box::new(std::iter::empty()), + Element::SrcBlock(_) => Box::new(std::iter::empty()), + Element::Clock(_) => Box::new(std::iter::empty()), + Element::DiarySexp(_) => Box::new(std::iter::empty()), + Element::Planning(_) => Box::new(std::iter::empty()), + Element::FixedWidthArea(_) => Box::new(std::iter::empty()), + Element::HorizontalRule(_) => Box::new(std::iter::empty()), + Element::Keyword(_) => Box::new(std::iter::empty()), + Element::LatexEnvironment(_) => Box::new(std::iter::empty()), + }, + Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)), + Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)), + Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)), + } + } +} + +pub struct AllTokensIterator<'r, 's> { + queued_tokens: VecDeque>, +} + +impl<'r, 's> AllTokensIterator<'r, 's> { + pub fn new(tkn: Token<'r, 's>) -> Self { + let mut queued_tokens = VecDeque::new(); + queued_tokens.push_back(tkn); + AllTokensIterator { queued_tokens } + } +} + +impl<'r, 's> Iterator for AllTokensIterator<'r, 's> { + type Item = Token<'r, 's>; + + fn next(&mut self) -> Option { + let next_token = match self.queued_tokens.pop_front() { + Some(tkn) => tkn, + None => return None, + }; + self.queued_tokens.extend(next_token.iter_tokens()); + Some(next_token) + } +} diff --git a/toy_language.txt b/toy_language.txt index 64ee8714..a29169b5 100644 --- a/toy_language.txt +++ b/toy_language.txt @@ -1,3 +1 @@ -foo *bar /baz *lorem* ipsum/ dolar* alpha - -foo *bar /baz _lorem_ ipsum/ dolar* alpha +foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar.