Merge branch 'source_based_tests'

Makefile

@@ -35,12 +35,12 @@ clean:
 
 .PHONY: test
 test:
-> cargo test --lib --test test_loader -- --test-threads $(TESTJOBS)
+> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: dockertest
 dockertest:
 > $(MAKE) -C docker/organic_test
-> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test"
+> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --no-fail-fast --lib --test test_loader"
 
 .PHONY: integrationtest
 integrationtest:

build.rs

@@ -74,8 +74,7 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
         "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."),
-        "trailing_whitespace_blank_lines_after_paragraphs" => Some("Waiting on mailing list response about ownership of the trailing blank lines."),
+        "export_snippet_paragraph_break_precedence" => Some("The latest code for org-mode is matching the export snippet without the closing @@."), // https://list.orgmode.org/orgmode/fb61ea28-f004-4c25-adf7-69fc55683ed4@app.fastmail.com/T/#u
         _ => None,
     }
 }

docker/organic_test/Dockerfile

@@ -1,4 +1,30 @@
-FROM rustlang/rust:nightly-alpine3.17
+FROM alpine:3.17 AS build
 
-RUN apk add --no-cache musl-dev emacs
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
+
+
+FROM build AS build-emacs
+
+RUN git clone --depth 1 --branch emacs-29.1 https://git.savannah.gnu.org/git/emacs.git /root/emacs
+WORKDIR /root/emacs
+RUN mkdir /root/dist
+RUN ./autogen.sh
+RUN ./configure --prefix /usr --without-x --without-sound
+RUN make
+RUN make DESTDIR="/root/dist" install
+
+
+FROM build AS build-org-mode
+COPY --from=build-emacs /root/dist/ /
+RUN mkdir /root/dist
+RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin b89bc55867d7cb809c379d371d12d409db785154 && git -C /root/org-mode checkout FETCH_HEAD
+WORKDIR /root/org-mode
+RUN make compile
+RUN make DESTDIR="/root/dist" install
+
+
+FROM rustlang/rust:nightly-alpine3.17
+RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+COPY --from=build-emacs /root/dist/ /
+COPY --from=build-org-mode /root/dist/ /

scripts/run_docker_integration_test.bash

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+REALPATH=$(command -v uu-realpath || command -v realpath)
+
+samples_dir=$(readlink -f "$DIR/../org_mode_samples")
+
+function get_test_names {
+    for test_file in "$@"
+    do
+        if [ -e "$test_file" ]; then
+            test_file_full_path=$(readlink -f "$test_file")
+            relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
+            without_extension="${relative_to_samples%.org}"
+            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+        else
+            echo "$test_file" | tr '[:upper:]' '[:lower:]'
+        fi
+    done
+}
+
+get_test_names "$@" | while read test; do
+    (
+        cd "$DIR/../"
+        make -C docker/organic_test
+        docker run --rm -v "$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --no-fail-fast --lib --test test_loader \"$test\" -- --show-output"
+    )
+done

src/compare/util.rs

@@ -48,14 +48,31 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         .nth(1)
         .ok_or("Should have an attributes child.")?;
     let attributes_map = attributes_child.as_map()?;
-    let begin = attributes_map
-        .get(":begin")
-        .ok_or("Missing :begin attribute.")?
-        .as_atom()?;
-    let end = attributes_map
-        .get(":end")
-        .ok_or("Missing :end attribute.")?
-        .as_atom()?;
+    let standard_properties = attributes_map.get(":standard-properties");
+    let (begin, end) = if standard_properties.is_some() {
+        let std_props = standard_properties
+            .expect("if statement proves its Some")
+            .as_vector()?;
+        let begin = std_props
+            .get(0)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        let end = std_props
+            .get(1)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        (begin, end)
+    } else {
+        let begin = attributes_map
+            .get(":begin")
+            .ok_or("Missing :begin attribute.")?
+            .as_atom()?;
+        let end = attributes_map
+            .get(":end")
+            .ok_or("Missing :end attribute.")?
+            .as_atom()?;
+        (begin, end)
+    };
     let (rust_begin, rust_end) = get_offsets(source, rust);
     if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end {
         Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;

src/parser/document.rs

@@ -160,7 +160,7 @@ fn zeroth_section<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
         opt(parser_with_context!(comment)(
             &without_consuming_whitespace_context,
         )),
-        parser_with_context!(property_drawer)(&without_consuming_whitespace_context),
+        parser_with_context!(property_drawer)(context),
         many0(blank_line),
     )))(input)?;
 

src/parser/export_snippet.rs

@@ -1,8 +1,6 @@
-use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::combinator::opt;
-use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many1;
@@ -11,6 +9,9 @@ use nom::sequence::tuple;
 
 use super::Context;
 use crate::error::Res;
+use crate::parser::exiting::ExitClass;
+use crate::parser::parser_context::ContextElement;
+use crate::parser::parser_context::ExitMatcherNode;
 use crate::parser::parser_with_context::parser_with_context;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
@@ -23,8 +24,15 @@ pub fn export_snippet<'r, 's>(
 ) -> Res<&'s str, ExportSnippet<'s>> {
     let (remaining, _) = tag("@@")(input)?;
     let (remaining, backend_name) = backend(context, remaining)?;
-    let (remaining, backend_contents) =
-        opt(tuple((tag(":"), parser_with_context!(contents)(context))))(remaining)?;
+    let parser_context =
+        context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
+            class: ExitClass::Beta,
+            exit_matcher: &export_snippet_end,
+        }));
+    let (remaining, backend_contents) = opt(tuple((
+        tag(":"),
+        parser_with_context!(contents)(&parser_context),
+    )))(remaining)?;
     let (remaining, _) = tag("@@")(remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
@@ -48,14 +56,13 @@ fn backend<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn contents<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
     let (remaining, source) = recognize(verify(
-        many_till(
-            anychar,
-            peek(alt((
-                parser_with_context!(exit_matcher_parser)(context),
-                tag("@@"),
-            ))),
-        ),
+        many_till(anychar, parser_with_context!(exit_matcher_parser)(context)),
         |(children, _exit_contents)| !children.is_empty(),
     ))(input)?;
     Ok((remaining, source))
 }
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn export_snippet_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+    tag("@@")(input)
+}

src/parser/sexp.rs

@@ -23,6 +23,7 @@ pub enum Token<'s> {
     Atom(&'s str),
     List(Vec<Token<'s>>),
     TextWithProperties(TextWithProperties<'s>),
+    Vector(Vec<Token<'s>>),
 }
 
 #[derive(Debug)]
@@ -73,6 +74,13 @@ enum ParseState {
 }
 
 impl<'s> Token<'s> {
+    pub fn as_vector<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
+        Ok(match self {
+            Token::Vector(children) => Ok(children),
+            _ => Err(format!("wrong token type {:?}", self)),
+        }?)
+    }
+
     pub fn as_list<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::List(children) => Ok(children),
@@ -136,7 +144,7 @@ pub fn sexp<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn token<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    alt((list, atom))(input)
+    alt((list, vector, atom))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
@@ -151,16 +159,33 @@ fn list<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::List(children)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn vector<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("[")(input)?;
+    let (remaining, children) = delimited(
+        multispace0,
+        separated_list1(multispace1, token),
+        multispace0,
+    )(remaining)?;
+    let (remaining, _) = tag("]")(remaining)?;
+    Ok((remaining, Token::Vector(children)))
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    not(peek(tag(")")))(input)?;
-    alt((text_with_properties, quoted_atom, unquoted_atom))(input)
+    not(peek(one_of(")]")))(input)?;
+    alt((
+        text_with_properties,
+        hash_notation,
+        quoted_atom,
+        unquoted_atom,
+    ))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn unquoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, body) = take_till1(|c| match c {
-        ' ' | '\t' | '\r' | '\n' | ')' => true,
+        ' ' | '\t' | '\r' | '\n' | ')' | ']' => true,
         _ => false,
     })(input)?;
     Ok((remaining, Token::Atom(body)))
@@ -182,6 +207,18 @@ fn quoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::Atom(source)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn hash_notation<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("#<")(input)?;
+    let (remaining, _body) = take_till1(|c| match c {
+        '>' => true,
+        _ => false,
+    })(remaining)?;
+    let (remaining, _) = tag(">")(remaining)?;
+    let source = get_consumed(input, remaining);
+    Ok((remaining, Token::Atom(source)))
+}
+
 fn text_with_properties<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, _) = tag("#(")(input)?;
     let (remaining, (text, props)) = delimited(
@@ -237,6 +274,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
     }
 
@@ -249,6 +287,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,
@@ -308,6 +347,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,