org_mode_samples/radio_link/before_and_after.org

@@ -1 +0,0 @@
-alpha *bar* baz foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar.

org_mode_samples/radio_link/simple.org

@@ -1 +0,0 @@
-foo <<<*bar* baz>>> lorem ipsum *bar* baz dolar.

org_mode_samples/text_markup/deep_nesting.org

@@ -1,3 +0,0 @@
-foo *bar /baz *lorem* ipsum/ dolar* alpha
-
-foo *bar /baz _lorem_ ipsum/ dolar* alpha

src/compare/diff.rs

@@ -29,8 +29,6 @@ use crate::parser::PlainListItem;
 use crate::parser::PlainText;
 use crate::parser::Planning;
 use crate::parser::PropertyDrawer;
-use crate::parser::RadioLink;
-use crate::parser::RadioTarget;
 use crate::parser::RegularLink;
 use crate::parser::Section;
 use crate::parser::SrcBlock;
@@ -146,8 +144,6 @@ fn compare_object<'s>(
         Object::StrikeThrough(obj) => compare_strike_through(source, emacs, obj),
         Object::PlainText(obj) => compare_plain_text(source, emacs, obj),
         Object::RegularLink(obj) => compare_regular_link(source, emacs, obj),
-        Object::RadioLink(obj) => compare_radio_link(source, emacs, obj),
-        Object::RadioTarget(obj) => compare_radio_target(source, emacs, obj),
     }
 }
 
@@ -919,27 +915,6 @@ fn compare_plain_text<'s>(
     let mut this_status = DiffStatus::Good;
     let mut message = None;
     let text = emacs.as_text()?;
-    let start_ind: usize = text
-        .properties
-        .get(0)
-        .expect("Should have start index.")
-        .as_atom()?
-        .parse()?;
-    let end_ind: usize = text
-        .properties
-        .get(1)
-        .expect("Should have end index.")
-        .as_atom()?
-        .parse()?;
-    let emacs_text_length = end_ind - start_ind;
-    if rust.source.len() != emacs_text_length {
-        this_status = DiffStatus::Bad;
-        message = Some(format!(
-            "(emacs len != rust len) {:?} != {:?}",
-            emacs_text_length,
-            rust.source.len()
-        ));
-    }
     let unquoted_text = text.unquote()?;
     if unquoted_text != rust.source {
         this_status = DiffStatus::Bad;
@@ -1117,49 +1092,3 @@ fn compare_regular_link<'s>(
         children: Vec::new(),
     })
 }
-
-fn compare_radio_link<'s>(
-    source: &'s str,
-    emacs: &'s Token<'s>,
-    rust: &'s RadioLink<'s>,
-) -> Result<DiffResult, Box<dyn std::error::Error>> {
-    let mut this_status = DiffStatus::Good;
-    let emacs_name = "link";
-    if assert_name(emacs, emacs_name).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    if assert_bounds(source, emacs, rust).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    Ok(DiffResult {
-        status: this_status,
-        name: emacs_name.to_owned(),
-        message: None,
-        children: Vec::new(),
-    })
-}
-
-fn compare_radio_target<'s>(
-    source: &'s str,
-    emacs: &'s Token<'s>,
-    rust: &'s RadioTarget<'s>,
-) -> Result<DiffResult, Box<dyn std::error::Error>> {
-    let mut this_status = DiffStatus::Good;
-    let emacs_name = "radio-target";
-    if assert_name(emacs, emacs_name).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    if assert_bounds(source, emacs, rust).is_err() {
-        this_status = DiffStatus::Bad;
-    }
-
-    Ok(DiffResult {
-        status: this_status,
-        name: emacs_name.to_owned(),
-        message: None,
-        children: Vec::new(),
-    })
-}

src/parser/mod.rs

@@ -26,7 +26,6 @@ mod plain_list;
 mod plain_text;
 mod planning;
 mod property_drawer;
-mod radio_link;
 mod regular_link;
 pub mod sexp;
 mod source;
@@ -68,8 +67,6 @@ pub use object::Code;
 pub use object::Italic;
 pub use object::Object;
 pub use object::PlainText;
-pub use object::RadioLink;
-pub use object::RadioTarget;
 pub use object::RegularLink;
 pub use object::StrikeThrough;
 pub use object::Underline;

src/parser/object.rs

@@ -1,10 +1,8 @@
 use super::source::Source;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub enum Object<'s> {
     RegularLink(RegularLink<'s>),
-    RadioLink(RadioLink<'s>),
-    RadioTarget(RadioTarget<'s>),
     Bold(Bold<'s>),
     Italic(Italic<'s>),
     Underline(Underline<'s>),
@@ -14,64 +12,52 @@ pub enum Object<'s> {
     PlainText(PlainText<'s>),
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct Bold<'s> {
     pub source: &'s str,
     pub children: Vec<Object<'s>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct Italic<'s> {
     pub source: &'s str,
     pub children: Vec<Object<'s>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct Underline<'s> {
     pub source: &'s str,
     pub children: Vec<Object<'s>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct StrikeThrough<'s> {
     pub source: &'s str,
     pub children: Vec<Object<'s>>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct Code<'s> {
     pub source: &'s str,
     pub contents: &'s str,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct Verbatim<'s> {
     pub source: &'s str,
     pub contents: &'s str,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct PlainText<'s> {
     pub source: &'s str,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 pub struct RegularLink<'s> {
     pub source: &'s str,
 }
 
-#[derive(Debug, PartialEq)]
-pub struct RadioTarget<'s> {
-    pub source: &'s str,
-    pub children: Vec<Object<'s>>,
-}
-
-#[derive(Debug, PartialEq)]
-pub struct RadioLink<'s> {
-    pub source: &'s str,
-    pub children: Vec<Object<'s>>,
-}
-
 impl<'s> Source<'s> for Object<'s> {
     fn get_source(&'s self) -> &'s str {
         match self {
@@ -83,8 +69,6 @@ impl<'s> Source<'s> for Object<'s> {
             Object::Verbatim(obj) => obj.source,
             Object::PlainText(obj) => obj.source,
             Object::RegularLink(obj) => obj.source,
-            Object::RadioLink(obj) => obj.source,
-            Object::RadioTarget(obj) => obj.source,
         }
     }
 }
@@ -130,15 +114,3 @@ impl<'s> Source<'s> for RegularLink<'s> {
         self.source
     }
 }
-
-impl<'s> Source<'s> for RadioLink<'s> {
-    fn get_source(&'s self) -> &'s str {
-        self.source
-    }
-}
-
-impl<'s> Source<'s> for RadioTarget<'s> {
-    fn get_source(&'s self) -> &'s str {
-        self.source
-    }
-}

src/parser/object_parser.rs

@@ -8,8 +8,6 @@ use super::regular_link::regular_link;
 use super::Context;
 use crate::error::Res;
 use crate::parser::object::Object;
-use crate::parser::radio_link::radio_link;
-use crate::parser::radio_link::radio_target;
 use crate::parser::text_markup::text_markup;
 
 #[tracing::instrument(ret, level = "debug")]
@@ -21,11 +19,6 @@ pub fn standard_set_object<'r, 's>(
     not(|i| context.check_exit_matcher(i))(input)?;
 
     alt((
-        map(parser_with_context!(radio_link)(context), Object::RadioLink),
-        map(
-            parser_with_context!(radio_target)(context),
-            Object::RadioTarget,
-        ),
         parser_with_context!(text_markup)(context),
         map(
             parser_with_context!(regular_link)(context),
@@ -40,7 +33,7 @@ pub fn minimal_set_object<'r, 's>(
     context: Context<'r, 's>,
     input: &'s str,
 ) -> Res<&'s str, Object<'s>> {
-    // TODO: add  entities, LaTeX fragments, superscripts and subscripts
+    // TODO: add  text markup, entities, LaTeX fragments, superscripts and subscripts
     not(|i| context.check_exit_matcher(i))(input)?;
 
     alt((
@@ -56,11 +49,6 @@ pub fn any_object_except_plain_text<'r, 's>(
 ) -> Res<&'s str, Object<'s>> {
     // Used for exit matchers so this does not check exit matcher condition.
     alt((
-        map(parser_with_context!(radio_link)(context), Object::RadioLink),
-        map(
-            parser_with_context!(radio_target)(context),
-            Object::RadioTarget,
-        ),
         parser_with_context!(text_markup)(context),
         map(
             parser_with_context!(regular_link)(context),

src/parser/plain_text.rs

@@ -1,31 +1,55 @@
-use nom::branch::alt;
-use nom::character::complete::anychar;
-use nom::combinator::peek;
+use nom::combinator::not;
 use nom::combinator::recognize;
-use nom::combinator::verify;
-use nom::multi::many_till;
 
 use super::object::PlainText;
 use super::Context;
+use crate::error::CustomError;
+use crate::error::MyError;
 use crate::error::Res;
+use crate::parser::exiting::ExitClass;
 use crate::parser::object_parser::any_object_except_plain_text;
+use crate::parser::parser_context::ContextElement;
+use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
-use crate::parser::util::exit_matcher_parser;
 
 #[tracing::instrument(ret, level = "debug")]
 pub fn plain_text<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, PlainText<'s>> {
-    let (remaining, source) = recognize(verify(
-        many_till(
-            anychar,
-            peek(alt((
-                parser_with_context!(exit_matcher_parser)(context),
-                parser_with_context!(plain_text_end)(context),
-            ))),
-        ),
-        |(children, _exit_contents)| !children.is_empty(),
-    ))(input)?;
-
-    Ok((remaining, PlainText { source }))
+    if input.len() == 0 {
+        return Err(nom::Err::Error(CustomError::MyError(MyError(
+            "Zero input length to plain_text.",
+        ))));
+    }
+    let parser_context =
+        context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+            class: ExitClass::Beta,
+            exit_matcher: &plain_text_end,
+        }));
+    let mut current_input = input.char_indices();
+    loop {
+        match current_input.next() {
+            Some((offset, _char)) => {
+                let remaining = &input[offset..];
+                let exit_matcher_status = not(|i| parser_context.check_exit_matcher(i))(remaining);
+                if exit_matcher_status.is_err() {
+                    if offset == 0 {
+                        // If we're at the start of the input, then nothing is plain text, so fire an error for zero-length match.
+                        exit_matcher_status?;
+                    } else {
+                        return Ok((
+                            &input[offset..],
+                            PlainText {
+                                source: &input[..offset],
+                            },
+                        ));
+                    }
+                }
+            }
+            None => {
+                // We hit the end of the file, so all input must be plain text
+                return Ok((&input[input.len()..], PlainText { source: input }));
+            }
+        };
+    }
 }
 
 #[tracing::instrument(ret, level = "debug")]

src/parser/radio_link.rs

@@ -1,56 +0,0 @@
-use nom::branch::alt;
-use nom::bytes::complete::tag;
-use nom::character::complete::line_ending;
-use nom::character::complete::space0;
-use nom::combinator::verify;
-use nom::multi::many_till;
-
-use super::Context;
-use crate::error::Res;
-use crate::parser::exiting::ExitClass;
-use crate::parser::object_parser::minimal_set_object;
-use crate::parser::parser_context::ContextElement;
-use crate::parser::parser_context::ExitMatcherNode;
-use crate::parser::parser_with_context::parser_with_context;
-use crate::parser::util::exit_matcher_parser;
-use crate::parser::util::get_consumed;
-use crate::parser::util::not_yet_implemented;
-use crate::parser::RadioLink;
-use crate::parser::RadioTarget;
-
-#[tracing::instrument(ret, level = "debug")]
-pub fn radio_link<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, RadioLink<'s>> {
-    not_yet_implemented()?;
-    todo!();
-}
-
-#[tracing::instrument(ret, level = "debug")]
-pub fn radio_target<'r, 's>(
-    context: Context<'r, 's>,
-    input: &'s str,
-) -> Res<&'s str, RadioTarget<'s>> {
-    let (remaining, _opening) = tag("<<<")(input)?;
-    let parser_context =
-        context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
-            exit_matcher: &radio_target_end,
-        }));
-
-    let (remaining, (children, _exit_contents)) = verify(
-        many_till(
-            parser_with_context!(minimal_set_object)(&parser_context),
-            parser_with_context!(exit_matcher_parser)(&parser_context),
-        ),
-        |(children, _exit_contents)| !children.is_empty(),
-    )(remaining)?;
-
-    let (remaining, _closing) = tag(">>>")(remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
-    let source = get_consumed(input, remaining);
-    Ok((remaining, RadioTarget { source, children }))
-}
-
-#[tracing::instrument(ret, level = "debug")]
-fn radio_target_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
-    alt((tag("<"), tag(">"), line_ending))(input)
-}

src/parser/text_markup.rs

@@ -12,7 +12,6 @@ use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many_till;
 use nom::sequence::terminated;
-use tracing::span;
 
 use super::Context;
 use crate::error::CustomError;
@@ -132,16 +131,6 @@ fn _text_markup_object<'r, 's, 'x>(
         |(children, _exit_contents)| !children.is_empty(),
     )(remaining)?;
 
-    {
-        let span = span!(tracing::Level::DEBUG, "Checking parent exit.");
-        let _enter = span.enter();
-        if exit_matcher_parser(context, remaining).is_ok() {
-            return Err(nom::Err::Error(CustomError::MyError(MyError(
-                "Parent exit matcher is triggering.",
-            ))));
-        }
-    }
-
     // TODO: Sometimes its plain text, not objects
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
     let (remaining, _trailing_whitespace) = space0(remaining)?;
@@ -179,16 +168,6 @@ fn _text_markup_string<'r, 's, 'x>(
         |(children, _exit_contents)| !children.is_empty(),
     ))(remaining)?;
 
-    {
-        let span = span!(tracing::Level::DEBUG, "Checking parent exit.");
-        let _enter = span.enter();
-        if exit_matcher_parser(context, remaining).is_ok() {
-            return Err(nom::Err::Error(CustomError::MyError(MyError(
-                "Parent exit matcher is triggering.",
-            ))));
-        }
-    }
-
     // TODO: Sometimes its plain text, not objects
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
     let (remaining, _trailing_whitespace) = space0(remaining)?;

src/parser/util.rs

@@ -197,6 +197,32 @@ pub fn always_fail<'r, 's>(_context: Context<'r, 's>, input: &'s str) -> Res<&'s
     ))))
 }
 
+/// Walk backwards unconsuming blank lines and line endings.
+///
+/// List items are a special case where the trailing blank lines do not belong to it, unlike all other elements. Rather than write that special logic into each child parser, this just walks backwards through the consumed input to unconsume trailing blank lines and line breaks.
+#[tracing::instrument(ret, level = "debug")]
+pub fn regurgitate<'s>(input: &'s str, remaining: &'s str) -> &'s str {
+    assert!(is_slice_of(input, remaining));
+    let mut offset = remaining.as_ptr() as usize - input.as_ptr() as usize;
+    let source = &input[..offset];
+    let mut char_indices = source.char_indices().rev();
+    loop {
+        match char_indices.next() {
+            Some((off, chr)) => {
+                if chr == '\n' {
+                    offset = off;
+                } else if chr != ' ' && chr != '\t' {
+                    return &input[offset..];
+                }
+            }
+            None => {
+                // It was all whitespace, so return the full input string
+                return input;
+            }
+        };
+    }
+}
+
 #[tracing::instrument(ret, level = "debug")]
 pub fn whitespace_eof(input: &str) -> Res<&str, &str> {
     recognize(tuple((multispace0, eof)))(input)
@@ -210,13 +236,6 @@ pub fn text_until_exit<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<
     ))(input)
 }
 
-#[allow(dead_code)]
-pub fn not_yet_implemented() -> Res<&'static str, ()> {
-    return Err(nom::Err::Error(CustomError::MyError(MyError(
-        "Not implemented yet.",
-    ))));
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -230,4 +249,14 @@ mod tests {
         assert!(is_slice_of(input, yellow_heart));
         assert_eq!(yellow_heart, "💛");
     }
+
+    #[test]
+    fn regurgitate_unicode() {
+        let input = "🧡💛\n\t   \t  \n\n💚💙💜";
+        let (green_heart_index, _) = input.char_indices().skip(12).next().unwrap();
+        let starting_with_green_heart = &input[green_heart_index..];
+        let after_yellow = regurgitate(input, starting_with_green_heart);
+        assert!(is_slice_of(input, after_yellow));
+        assert_eq!(after_yellow, "\n\t   \t  \n\n💚💙💜");
+    }
 }

toy_language.txt

@@ -1,3 +1,13 @@
-foo *bar /baz *lorem* ipsum/ dolar* alpha
+prologue *goes here* I guess *bold
+text*
 
-foo *bar /baz _lorem_ ipsum/ dolar* alpha
+bold*wont* start *or stop*when there is text outside it
+
+I guess *regular
+
+text*
+
+[[foo][foo *bar]] baz* car
+
+
+*nesting *bold entrances* and* exits