build.rs

@@ -79,7 +79,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "element_container_priority_greater_block_greater_block" => Some("Need to implement subscript."),
         "element_container_priority_section_greater_block" => Some("Need to implement subscript."),
         "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "entity_simple" => Some("Need to implement LaTeX fragments."),
         _ => None,
     }
 }

org_mode_samples/entity/simple.org

@@ -1,3 +0,0 @@
-foo \Delta bar
-foo \pibar
-foo \pi{}bar

src/compare/diff.rs

@@ -13,7 +13,6 @@ use crate::parser::DocumentElement;
 use crate::parser::Drawer;
 use crate::parser::DynamicBlock;
 use crate::parser::Element;
-use crate::parser::Entity;
 use crate::parser::ExampleBlock;
 use crate::parser::ExportBlock;
 use crate::parser::FixedWidthArea;
@@ -155,7 +154,6 @@ fn compare_object<'s>(
         Object::PlainLink(obj) => compare_plain_link(source, emacs, obj),
         Object::AngleLink(obj) => compare_angle_link(source, emacs, obj),
         Object::OrgMacro(obj) => compare_org_macro(source, emacs, obj),
-        Object::Entity(obj) => compare_entity(source, emacs, obj),
     }
 }
 
@@ -1240,26 +1238,3 @@ fn compare_org_macro<'s>(
         children: Vec::new(),
     })
 }
-
-fn compare_entity<'s>(
-    source: &'s str,
-    emacs: &'s Token<'s>,
-    rust: &'s Entity<'s>,
-) -> Result<DiffResult, Box<dyn std::error::Error>> {
-    let mut this_status = DiffStatus::Good;
-    let emacs_name = "entity";
-    if assert_name(emacs, emacs_name).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    if assert_bounds(source, emacs, rust).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    Ok(DiffResult {
-        status: this_status,
-        name: emacs_name.to_owned(),
-        message: None,
-        children: Vec::new(),
-    })
-}

src/parser/entity.rs

@@ -1,50 +0,0 @@
-use nom::branch::alt;
-use nom::bytes::complete::tag;
-use nom::bytes::complete::tag_no_case;
-use nom::character::complete::satisfy;
-use nom::character::complete::space0;
-use nom::combinator::eof;
-use nom::combinator::peek;
-use nom::combinator::recognize;
-
-use super::Context;
-use crate::error::Res;
-use crate::parser::object::Entity;
-use crate::parser::parser_with_context::parser_with_context;
-use crate::parser::util::get_consumed;
-
-#[tracing::instrument(ret, level = "debug")]
-pub fn entity<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Entity<'s>> {
-    let (remaining, _) = tag("\\")(input)?;
-    let (remaining, entity_name) = name(context, remaining)?;
-    let (remaining, _) = alt((
-        tag("{}"),
-        peek(recognize(parser_with_context!(entity_end)(context))),
-    ))(remaining)?;
-    let (remaining, _) = space0(remaining)?;
-
-    let source = get_consumed(input, remaining);
-    Ok((
-        remaining,
-        Entity {
-            source,
-            entity_name,
-        },
-    ))
-}
-
-#[tracing::instrument(ret, level = "debug")]
-fn name<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
-    // TODO: This should be defined by org-entities and optionally org-entities-user
-
-    // TODO: Add the rest of the entities, this is a very incomplete list
-    let (remaining, proto) = alt((alt((tag_no_case("delta"), tag_no_case("pi"))),))(input)?;
-    Ok((remaining, proto))
-}
-
-#[tracing::instrument(ret, level = "debug")]
-fn entity_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, ()> {
-    let (remaining, _) = alt((eof, recognize(satisfy(|c| !c.is_alphabetic()))))(input)?;
-
-    Ok((remaining, ()))
-}

src/parser/mod.rs

@@ -7,7 +7,6 @@ mod drawer;
 mod dynamic_block;
 mod element;
 mod element_parser;
-mod entity;
 mod exiting;
 mod fixed_width_area;
 mod footnote_definition;
@@ -71,7 +70,6 @@ pub use lesser_element::VerseBlock;
 pub use object::AngleLink;
 pub use object::Bold;
 pub use object::Code;
-pub use object::Entity;
 pub use object::Italic;
 pub use object::Object;
 pub use object::OrgMacro;

src/parser/object.rs

@@ -15,7 +15,6 @@ pub enum Object<'s> {
     PlainLink(PlainLink<'s>),
     AngleLink(AngleLink<'s>),
     OrgMacro(OrgMacro<'s>),
-    Entity(Entity<'s>),
 }
 
 #[derive(Debug, PartialEq)]
@@ -97,12 +96,6 @@ pub struct OrgMacro<'s> {
     pub macro_args: Vec<&'s str>,
 }
 
-#[derive(Debug, PartialEq)]
-pub struct Entity<'s> {
-    pub source: &'s str,
-    pub entity_name: &'s str,
-}
-
 impl<'s> Source<'s> for Object<'s> {
     fn get_source(&'s self) -> &'s str {
         match self {
@@ -119,7 +112,6 @@ impl<'s> Source<'s> for Object<'s> {
             Object::PlainLink(obj) => obj.source,
             Object::AngleLink(obj) => obj.source,
             Object::OrgMacro(obj) => obj.source,
-            Object::Entity(obj) => obj.source,
         }
     }
 }
@@ -195,9 +187,3 @@ impl<'s> Source<'s> for OrgMacro<'s> {
         self.source
     }
 }
-
-impl<'s> Source<'s> for Entity<'s> {
-    fn get_source(&'s self) -> &'s str {
-        self.source
-    }
-}

src/parser/object_parser.rs

@@ -8,7 +8,6 @@ use super::regular_link::regular_link;
 use super::Context;
 use crate::error::Res;
 use crate::parser::angle_link::angle_link;
-use crate::parser::entity::entity;
 use crate::parser::object::Object;
 use crate::parser::org_macro::org_macro;
 use crate::parser::plain_link::plain_link;
@@ -21,11 +20,10 @@ pub fn standard_set_object<'r, 's>(
     context: Context<'r, 's>,
     input: &'s str,
 ) -> Res<&'s str, Object<'s>> {
-    // TODO: LaTeX fragments, export snippets, footnote references, citations (NOT citation references), inline babel calls, inline source blocks, line breaks, links, macros, targets and radio targets, statistics cookies, subscript and superscript, timestamps, and text markup.
+    // TODO: add entities, LaTeX fragments, export snippets, footnote references, citations (NOT citation references), inline babel calls, inline source blocks, line breaks, links, macros, targets and radio targets, statistics cookies, subscript and superscript, timestamps, and text markup.
     not(|i| context.check_exit_matcher(i))(input)?;
 
     alt((
-        map(parser_with_context!(entity)(context), Object::Entity),
         map(parser_with_context!(radio_link)(context), Object::RadioLink),
         map(
             parser_with_context!(radio_target)(context),
@@ -48,11 +46,10 @@ pub fn minimal_set_object<'r, 's>(
     context: Context<'r, 's>,
     input: &'s str,
 ) -> Res<&'s str, Object<'s>> {
-    // TODO: LaTeX fragments, superscripts and subscripts
+    // TODO: add  entities, LaTeX fragments, superscripts and subscripts
     not(|i| context.check_exit_matcher(i))(input)?;
 
     alt((
-        map(parser_with_context!(entity)(context), Object::Entity),
         parser_with_context!(text_markup)(context),
         map(parser_with_context!(plain_text)(context), Object::PlainText),
     ))(input)
@@ -65,7 +62,6 @@ pub fn any_object_except_plain_text<'r, 's>(
 ) -> Res<&'s str, Object<'s>> {
     // Used for exit matchers so this does not check exit matcher condition.
     alt((
-        map(parser_with_context!(entity)(context), Object::Entity),
         map(parser_with_context!(radio_link)(context), Object::RadioLink),
         map(
             parser_with_context!(radio_target)(context),

src/parser/token.rs

@@ -52,7 +52,70 @@ impl<'r, 's> Token<'r, 's> {
                 Object::PlainLink(_) => Box::new(std::iter::empty()),
                 Object::AngleLink(_) => Box::new(std::iter::empty()),
                 Object::OrgMacro(_) => Box::new(std::iter::empty()),
-                Object::Entity(_) => Box::new(std::iter::empty()),
+            },
+            Token::Element(elem) => match elem {
+                Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::PlainList(inner) => {
+                    Box::new(inner.children.iter().map(Token::PlainListItem))
+                }
+                Element::GreaterBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::DynamicBlock(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::FootnoteDefinition(inner) => {
+                    Box::new(inner.children.iter().map(Token::Element))
+                }
+                Element::Comment(_) => Box::new(std::iter::empty()),
+                Element::Drawer(inner) => Box::new(inner.children.iter().map(Token::Element)),
+                Element::PropertyDrawer(_) => Box::new(std::iter::empty()),
+                Element::Table(inner) => Box::new(inner.children.iter().map(Token::TableRow)),
+                Element::VerseBlock(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Element::CommentBlock(_) => Box::new(std::iter::empty()),
+                Element::ExampleBlock(_) => Box::new(std::iter::empty()),
+                Element::ExportBlock(_) => Box::new(std::iter::empty()),
+                Element::SrcBlock(_) => Box::new(std::iter::empty()),
+                Element::Clock(_) => Box::new(std::iter::empty()),
+                Element::DiarySexp(_) => Box::new(std::iter::empty()),
+                Element::Planning(_) => Box::new(std::iter::empty()),
+                Element::FixedWidthArea(_) => Box::new(std::iter::empty()),
+                Element::HorizontalRule(_) => Box::new(std::iter::empty()),
+                Element::Keyword(_) => Box::new(std::iter::empty()),
+                Element::LatexEnvironment(_) => Box::new(std::iter::empty()),
+            },
+            Token::PlainListItem(elem) => Box::new(elem.children.iter().map(Token::Element)),
+            Token::TableRow(elem) => Box::new(elem.children.iter().map(Token::TableCell)),
+            Token::TableCell(elem) => Box::new(elem.children.iter().map(Token::Object)),
+        }
+    }
+
+    pub fn all_tokens_no_order(&self) -> Box<dyn Iterator<Item = Token<'r, 's>> + '_> {
+        match self {
+            Token::Document(document) => Box::new(
+                document
+                    .zeroth_section
+                    .iter()
+                    .map(Token::Section)
+                    .chain(document.children.iter().map(Token::Heading)),
+            ),
+            Token::Heading(heading) => Box::new(heading.title.iter().map(Token::Object).chain(
+                heading.children.iter().map(|de| match de {
+                    DocumentElement::Heading(ref obj) => Token::Heading(obj),
+                    DocumentElement::Section(ref obj) => Token::Section(obj),
+                }),
+            )),
+            Token::Section(section) => Box::new(section.children.iter().map(Token::Element)),
+            Token::Object(obj) => match obj {
+                Object::Bold(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Italic(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Underline(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::StrikeThrough(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::Code(_) => Box::new(std::iter::empty()),
+                Object::Verbatim(_) => Box::new(std::iter::empty()),
+                Object::PlainText(_) => Box::new(std::iter::empty()),
+                Object::RegularLink(_) => Box::new(std::iter::empty()),
+                Object::RadioLink(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::RadioTarget(inner) => Box::new(inner.children.iter().map(Token::Object)),
+                Object::PlainLink(_) => Box::new(std::iter::empty()),
+                Object::AngleLink(_) => Box::new(std::iter::empty()),
+                Object::OrgMacro(_) => Box::new(std::iter::empty()),
             },
             Token::Element(elem) => match elem {
                 Element::Paragraph(inner) => Box::new(inner.children.iter().map(Token::Object)),

toy_language.txt

@@ -1,3 +1 @@
-foo \Delta bar
-foo \pibar
-foo \pi{}bar
+foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar.