Merge branch 'additional_detects'

This ensures that both code paths are capable of being parsed by rust-analyzer simultaneously and I think it will be less confusing to newcomers.

Cargo.toml

@@ -21,9 +21,15 @@ name = "organic"
 path = "src/lib.rs"
 
 [[bin]]
-# This bin exists for development purposes only. The real target of this crate is the library.
-name = "parse"
-path = "src/main.rs"
+  # This bin exists for development purposes only. The real target of this crate is the library.
+  name = "parse"
+  path = "src/main.rs"
+
+[[bin]]
+  # This bin exists for development purposes only. The real target of this crate is the library.
+  name = "compare"
+  path = "src/bin_compare.rs"
+  required-features = ["compare"]
 
 [dependencies]
 nom = "7.1.1"

build.rs

@@ -86,7 +86,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
     match name {
         "autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
-        "autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
         _ => None,
     }
 }

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -23,7 +23,7 @@ function main {
     if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
         CARGO_TARGET_DIR=$(realpath target/)
     fi
-    PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
+    PARSE="${CARGO_TARGET_DIR}/release-lto/compare"
 
     local all_status=0
     set +e

org_mode_samples/greater_element/plain_list/description_list_with_double_colon_in_tag.org

@@ -1 +1,2 @@
 - =foo :: bar= :: baz
+- lorem :: ipsum :: dolar

org_mode_samples/lesser_element/diary_sexp/not_real_sexp.org

@@ -0,0 +1,6 @@
+%%(foo
+)
+
+%%(bar    ;  baz
+
+lorem

scripts/run_docker_compare.bash

@@ -38,7 +38,7 @@ function launch_container {
 
     if [ "$SHELL" != "YES" ]; then
         local features_joined=$(IFS=","; echo "${features[*]}")
-        additional_args+=(cargo run --no-default-features --features "$features_joined")
+        additional_args+=(cargo run --bin compare --no-default-features --features "$features_joined")
         additional_flags+=(--read-only)
     else
         additional_args+=(/bin/sh)

src/bin_compare.rs

@@ -0,0 +1,122 @@
+#![feature(round_char_boundary)]
+#![feature(exact_size_is_empty)]
+use std::io::Read;
+use std::path::Path;
+
+use ::organic::parser::parse;
+use organic::compare_document;
+use organic::emacs_parse_anonymous_org_document;
+use organic::emacs_parse_file_org_document;
+use organic::get_emacs_version;
+use organic::get_org_mode_version;
+use organic::parser::parse_with_settings;
+use organic::parser::sexp::sexp_with_padding;
+use organic::GlobalSettings;
+use organic::LocalFileAccessInterface;
+
+#[cfg(feature = "tracing")]
+use crate::init_tracing::init_telemetry;
+#[cfg(feature = "tracing")]
+use crate::init_tracing::shutdown_telemetry;
+#[cfg(feature = "tracing")]
+mod init_tracing;
+
+#[cfg(not(feature = "tracing"))]
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    main_body()
+}
+
+#[cfg(feature = "tracing")]
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let rt = tokio::runtime::Runtime::new()?;
+    let result = rt.block_on(async {
+        init_telemetry()?;
+        let main_body_result = main_body();
+        shutdown_telemetry()?;
+        main_body_result
+    });
+    result
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn main_body() -> Result<(), Box<dyn std::error::Error>> {
+    let args = std::env::args().skip(1);
+    if args.is_empty() {
+        let org_contents = read_stdin_to_string()?;
+        run_anonymous_parse(org_contents)
+    } else {
+        for arg in args {
+            run_parse_on_file(arg)?
+        }
+        Ok(())
+    }
+}
+
+fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
+    let mut stdin_contents = String::new();
+    std::io::stdin()
+        .lock()
+        .read_to_string(&mut stdin_contents)?;
+    Ok(stdin_contents)
+}
+
+fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
+    let org_contents = org_contents.as_ref();
+    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
+    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
+    let rust_parsed = parse(org_contents)?;
+    let org_sexp = emacs_parse_anonymous_org_document(org_contents)?;
+    let (_remaining, parsed_sexp) =
+        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
+
+    println!("{}\n\n\n", org_contents);
+    println!("{}", org_sexp);
+    println!("{:#?}", rust_parsed);
+
+    // We do the diffing after printing out both parsed forms in case the diffing panics
+    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
+    diff_result.print(org_contents)?;
+
+    if diff_result.is_bad() {
+        Err("Diff results do not match.")?;
+    }
+
+    Ok(())
+}
+
+fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
+    let org_path = org_path.as_ref();
+    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
+    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
+    let parent_directory = org_path
+        .parent()
+        .ok_or("Should be contained inside a directory.")?;
+    let org_contents = std::fs::read_to_string(org_path)?;
+    let org_contents = org_contents.as_str();
+    let file_access_interface = LocalFileAccessInterface {
+        working_directory: Some(parent_directory.to_path_buf()),
+    };
+    let global_settings = {
+        let mut global_settings = GlobalSettings::default();
+        global_settings.file_access = &file_access_interface;
+        global_settings
+    };
+    let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
+    let org_sexp = emacs_parse_file_org_document(org_path)?;
+    let (_remaining, parsed_sexp) =
+        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
+
+    println!("{}\n\n\n", org_contents);
+    println!("{}", org_sexp);
+    println!("{:#?}", rust_parsed);
+
+    // We do the diffing after printing out both parsed forms in case the diffing panics
+    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
+    diff_result.print(org_contents)?;
+
+    if diff_result.is_bad() {
+        Err("Diff results do not match.")?;
+    }
+
+    Ok(())
+}

src/context/context.rs

@@ -108,7 +108,7 @@ impl<'g, 'r, 's> Context<'g, 'r, 's> {
         &'r self,
         i: OrgSource<'s>,
     ) -> IResult<OrgSource<'s>, OrgSource<'s>, CustomError<OrgSource<'s>>> {
-        let mut current_class_filter = ExitClass::Delta;
+        let mut current_class_filter = ExitClass::Gamma;
         for current_node in self.iter_context() {
             let context_element = current_node.get_data();
             match context_element {

src/context/exiting.rs

@@ -4,7 +4,6 @@ pub enum ExitClass {
     Alpha = 2,
     Beta = 3,
     Gamma = 4,
-    Delta = 5,
 }
 
 impl std::fmt::Display for ExitClass {

src/main.rs

@@ -4,19 +4,7 @@ use std::io::Read;
 use std::path::Path;
 
 use ::organic::parser::parse;
-#[cfg(feature = "compare")]
-use organic::compare_document;
-#[cfg(feature = "compare")]
-use organic::emacs_parse_anonymous_org_document;
-#[cfg(feature = "compare")]
-use organic::emacs_parse_file_org_document;
-#[cfg(feature = "compare")]
-use organic::get_emacs_version;
-#[cfg(feature = "compare")]
-use organic::get_org_mode_version;
 use organic::parser::parse_with_settings;
-#[cfg(feature = "compare")]
-use organic::parser::sexp::sexp_with_padding;
 use organic::GlobalSettings;
 use organic::LocalFileAccessInterface;
 
@@ -66,85 +54,14 @@ fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
     Ok(stdin_contents)
 }
 
-#[cfg(feature = "compare")]
 fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
-    let org_contents = org_contents.as_ref();
-    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
-    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
-    let rust_parsed = parse(org_contents)?;
-    let org_sexp = emacs_parse_anonymous_org_document(org_contents)?;
-    let (_remaining, parsed_sexp) =
-        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
-
-    println!("{}\n\n\n", org_contents);
-    println!("{}", org_sexp);
-    println!("{:#?}", rust_parsed);
-
-    // We do the diffing after printing out both parsed forms in case the diffing panics
-    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
-    diff_result.print(org_contents)?;
-
-    if diff_result.is_bad() {
-        Err("Diff results do not match.")?;
-    }
-
-    Ok(())
-}
-
-#[cfg(not(feature = "compare"))]
-fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
-    eprintln!(
-        "This program was built with compare disabled. Only parsing with organic, not comparing."
-    );
     let rust_parsed = parse(org_contents.as_ref())?;
     println!("{:#?}", rust_parsed);
     Ok(())
 }
 
-#[cfg(feature = "compare")]
 fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
     let org_path = org_path.as_ref();
-    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
-    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
-    let parent_directory = org_path
-        .parent()
-        .ok_or("Should be contained inside a directory.")?;
-    let org_contents = std::fs::read_to_string(org_path)?;
-    let org_contents = org_contents.as_str();
-    let file_access_interface = LocalFileAccessInterface {
-        working_directory: Some(parent_directory.to_path_buf()),
-    };
-    let global_settings = {
-        let mut global_settings = GlobalSettings::default();
-        global_settings.file_access = &file_access_interface;
-        global_settings
-    };
-    let rust_parsed = parse_with_settings(org_contents, &global_settings)?;
-    let org_sexp = emacs_parse_file_org_document(org_path)?;
-    let (_remaining, parsed_sexp) =
-        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
-
-    println!("{}\n\n\n", org_contents);
-    println!("{}", org_sexp);
-    println!("{:#?}", rust_parsed);
-
-    // We do the diffing after printing out both parsed forms in case the diffing panics
-    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
-    diff_result.print(org_contents)?;
-
-    if diff_result.is_bad() {
-        Err("Diff results do not match.")?;
-    }
-
-    Ok(())
-}
-
-#[cfg(not(feature = "compare"))]
-fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
-    let org_path = org_path.as_ref();
-    eprintln!(
-        "This program was built with compare disabled. Only parsing with organic, not comparing."
-    );
     let parent_directory = org_path
         .parent()
         .ok_or("Should be contained inside a directory.")?;

src/parser/comment.rs

@@ -67,6 +67,17 @@ fn comment_line<'b, 'g, 'r, 's>(
     Ok((remaining, source))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn detect_comment<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((
+        start_of_line,
+        space0,
+        tag("#"),
+        alt((tag(" "), line_ending, eof)),
+    ))(input)?;
+    Ok((input, ()))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/parser/diary_sexp.rs

@@ -1,16 +1,11 @@
 use nom::branch::alt;
+use nom::bytes::complete::is_not;
 use nom::bytes::complete::tag;
-use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
-use nom::character::complete::space0;
 use nom::combinator::eof;
-use nom::combinator::opt;
-use nom::combinator::recognize;
-use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
-use super::sexp::sexp;
 use crate::context::RefContext;
 use crate::error::Res;
 use crate::parser::util::get_consumed;
@@ -23,17 +18,9 @@ pub fn diary_sexp<'b, 'g, 'r, 's>(
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, DiarySexp<'s>> {
     start_of_line(input)?;
-    let (remaining, _leading_whitespace) = space0(input)?;
-    let (remaining, _clock) = tag("%%")(remaining)?;
-    let (remaining, _gap_whitespace) = space0(remaining)?;
-    let (remaining, _sexp) = recognize(sexp)(remaining)?;
-    let (remaining, _trailing_comment) = opt(tuple((
-        space0,
-        tag(";"),
-        many_till(anychar, alt((line_ending, eof))),
-    )))(remaining)?;
-    let (remaining, _trailing_whitespace) =
-        recognize(tuple((space0, alt((line_ending, eof)))))(remaining)?;
+    let (remaining, _clock) = tag("%%(")(input)?;
+    let (remaining, _contents) = is_not("\r\n")(remaining)?;
+    let (remaining, _eol) = alt((line_ending, eof))(remaining)?;
 
     let source = get_consumed(input, remaining);
     Ok((
@@ -43,3 +30,9 @@ pub fn diary_sexp<'b, 'g, 'r, 's>(
         },
     ))
 }
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn detect_diary_sexp<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((start_of_line, tag("%%(")))(input)?;
+    Ok((input, ()))
+}

src/parser/element_parser.rs

@@ -4,10 +4,14 @@ use nom::multi::many0;
 
 use super::clock::clock;
 use super::comment::comment;
+use super::comment::detect_comment;
+use super::diary_sexp::detect_diary_sexp;
 use super::diary_sexp::diary_sexp;
 use super::drawer::drawer;
 use super::dynamic_block::dynamic_block;
+use super::fixed_width_area::detect_fixed_width_area;
 use super::fixed_width_area::fixed_width_area;
+use super::footnote_definition::detect_footnote_definition;
 use super::footnote_definition::footnote_definition;
 use super::greater_block::greater_block;
 use super::horizontal_rule::horizontal_rule;
@@ -24,6 +28,7 @@ use super::org_source::OrgSource;
 use super::paragraph::paragraph;
 use super::plain_list::detect_plain_list;
 use super::plain_list::plain_list;
+use super::table::detect_table;
 use super::util::get_consumed;
 use super::util::maybe_consume_trailing_whitespace_if_not_exiting;
 use crate::context::parser_with_context;
@@ -135,7 +140,16 @@ fn _detect_element<'b, 'g, 'r, 's>(
     input: OrgSource<'s>,
     can_be_paragraph: bool,
 ) -> Res<OrgSource<'s>, ()> {
-    if detect_plain_list(input).is_ok() {
+    if alt((
+        detect_plain_list,
+        detect_footnote_definition,
+        detect_diary_sexp,
+        detect_comment,
+        detect_fixed_width_area,
+        detect_table,
+    ))(input)
+    .is_ok()
+    {
         return Ok((input, ()));
     }
     if _element(context, input, can_be_paragraph).is_ok() {

src/parser/fixed_width_area.rs

@@ -47,7 +47,7 @@ fn fixed_width_area_line<'b, 'g, 'r, 's>(
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
     start_of_line(input)?;
     let (remaining, _indent) = space0(input)?;
-    let (remaining, (_hash, _leading_whitespace_and_content, _line_ending)) = tuple((
+    let (remaining, (_colon, _leading_whitespace_and_content, _line_ending)) = tuple((
         tag(":"),
         opt(tuple((space1, is_not("\r\n")))),
         alt((line_ending, eof)),
@@ -55,3 +55,14 @@ fn fixed_width_area_line<'b, 'g, 'r, 's>(
     let source = get_consumed(input, remaining);
     Ok((remaining, source))
 }
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn detect_fixed_width_area<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((
+        start_of_line,
+        space0,
+        tag(":"),
+        alt((tag(" "), line_ending, eof)),
+    ))(input)?;
+    Ok((input, ()))
+}

src/parser/footnote_definition.rs

@@ -122,7 +122,7 @@ fn footnote_definition_end<'b, 'g, 'r, 's>(
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn detect_footnote_definition<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+pub fn detect_footnote_definition<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
     tuple((start_of_line, tag_no_case("[fn:"), label, tag("]")))(input)?;
     Ok((input, ()))
 }

src/parser/plain_list.rs

@@ -1,5 +1,6 @@
 use nom::branch::alt;
 use nom::bytes::complete::tag;
+use nom::character::complete::anychar;
 use nom::character::complete::digit1;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
@@ -301,18 +302,11 @@ fn item_tag<'b, 'g, 'r, 's>(
     context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
-    let contexts = [
-        ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Gamma,
-            exit_matcher: &item_tag_line_ending_end,
-        }),
-        ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Delta,
-            exit_matcher: &item_tag_end,
-        }),
-    ];
-    let parser_context = context.with_additional_node(&contexts[0]);
-    let parser_context = parser_context.with_additional_node(&contexts[1]);
+    let parser_context = ContextElement::ExitMatcherNode(ExitMatcherNode {
+        class: ExitClass::Gamma,
+        exit_matcher: &item_tag_end,
+    });
+    let parser_context = context.with_additional_node(&parser_context);
     let (remaining, (children, _exit_contents)) = verify(
         many_till(
             // TODO: Should this be using a different set like the minimal set?
@@ -321,7 +315,7 @@ fn item_tag<'b, 'g, 'r, 's>(
         ),
         |(children, _exit_contents)| !children.is_empty(),
     )(input)?;
-    let (remaining, _) = tuple((one_of(" \t"), tag("::")))(remaining)?;
+    let (remaining, _) = item_tag_divider(remaining)?;
     Ok((remaining, children))
 }
 
@@ -330,19 +324,22 @@ fn item_tag_end<'b, 'g, 'r, 's>(
     _context: RefContext<'b, 'g, 'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    recognize(tuple((
-        one_of(" \t"),
-        tag("::"),
-        alt((recognize(one_of(" \t")), line_ending, eof)),
-    )))(input)
+    alt((
+        recognize(tuple((
+            item_tag_divider,
+            opt(tuple((
+                peek(one_of(" \t")),
+                many_till(anychar, peek(alt((item_tag_divider, line_ending, eof)))),
+            ))),
+            alt((line_ending, eof)),
+        ))),
+        line_ending,
+    ))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-fn item_tag_line_ending_end<'b, 'g, 'r, 's>(
-    _context: RefContext<'b, 'g, 'r, 's>,
-    input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    line_ending(input)
+fn item_tag_divider<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
+    recognize(tuple((one_of(" \t"), tag("::"))))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]

src/parser/table.rs

@@ -61,7 +61,6 @@ pub fn org_mode_table<'b, 'g, 'r, 's>(
     let (remaining, formulas) =
         many0(parser_with_context!(table_formula_keyword)(context))(remaining)?;
 
-    // TODO: Consume trailing formulas
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -74,6 +73,12 @@ pub fn org_mode_table<'b, 'g, 'r, 's>(
     ))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn detect_table<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((start_of_line, space0, tag("|")))(input)?;
+    Ok((input, ()))
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn table_end<'b, 'g, 'r, 's>(
     _context: RefContext<'b, 'g, 'r, 's>,