Enable tests that were disabled before.

This is to integrate fixes that have been committed to org-mode but have not made it into emacs, while also getting the latest emacs on alpine.

.lighthouse/pipeline-rust-test.yaml

@@ -4,6 +4,10 @@ metadata:
   name: rust-test
 spec:
   pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
     params:
       - name: image-name
         description: The name for the built image
@@ -201,7 +205,6 @@ spec:
       secret:
         secretName: harbor-plain
   serviceAccountName: build-bot
-  timeout: 240h0m0s
   params:
     - name: image-name
       value: "harbor.fizz.buzz/private/organic-test"

Cargo.toml

@@ -10,6 +10,12 @@ readme = "README.md"
 keywords = ["emacs", "org-mode"]
 categories = ["parsing"]
 resolver = "2"
+include = [
+  "LICENSE",
+  "**/*.rs",
+  "Cargo.toml",
+  "tests/*"
+]
 
 [lib]
 name = "organic"

Makefile

@@ -37,6 +37,11 @@ clean:
 test:
 > cargo test --lib --test test_loader -- --test-threads $(TESTJOBS)
 
+.PHONY: dockertest
+dockertest:
+> $(MAKE) -C docker/organic_test
+> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --lib --test test_loader"
+
 .PHONY: integrationtest
 integrationtest:
 > cargo test --no-fail-fast --test test_loader -- --test-threads $(TESTJOBS)

build.rs

@@ -74,7 +74,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
         "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."),
         _ => None,
     }
 }

docker/organic_test/Dockerfile

@@ -1,4 +1,30 @@
-FROM rustlang/rust:nightly-alpine3.17
+FROM alpine:3.17 AS build
 
-RUN apk add --no-cache musl-dev emacs
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
+
+
+FROM build AS build-emacs
+
+RUN git clone --depth 1 --branch emacs-29.1 https://git.savannah.gnu.org/git/emacs.git /root/emacs
+WORKDIR /root/emacs
+RUN mkdir /root/dist
+RUN ./autogen.sh
+RUN ./configure --prefix /usr --without-x --without-sound
+RUN make
+RUN make DESTDIR="/root/dist" install
+
+
+FROM build AS build-org-mode
+COPY --from=build-emacs /root/dist/ /
+RUN mkdir /root/dist
+RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin 70a082c9fbf6fd7d05fb56b26c6f2039b8edd478 && git -C /root/org-mode checkout FETCH_HEAD
+WORKDIR /root/org-mode
+RUN make compile
+RUN make DESTDIR="/root/dist" install
+
+
+FROM rustlang/rust:nightly-alpine3.17
+RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+COPY --from=build-emacs /root/dist/ /
+COPY --from=build-org-mode /root/dist/ /

org_mode_samples/trailing_whitespace/blank_lines_after_paragraphs.org

@@ -0,0 +1,18 @@
+foo bar.
+
+
+
+
+* Lorem
+baz
+
+
+
+
+* Ipsum
+alpha
+
+
+
+
+beta

src/compare/util.rs

@@ -48,6 +48,21 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         .nth(1)
         .ok_or("Should have an attributes child.")?;
     let attributes_map = attributes_child.as_map()?;
+    let standard_properties = attributes_map.get(":standard-properties");
+    let (begin, end) = if standard_properties.is_some() {
+        let std_props = standard_properties
+            .expect("if statement proves its Some")
+            .as_vector()?;
+        let begin = std_props
+            .get(0)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        let end = std_props
+            .get(1)
+            .ok_or("Missing first element in standard properties")?
+            .as_atom()?;
+        (begin, end)
+    } else {
         let begin = attributes_map
             .get(":begin")
             .ok_or("Missing :begin attribute.")?
@@ -56,6 +71,8 @@ pub fn assert_bounds<'s, S: Source<'s>>(
             .get(":end")
             .ok_or("Missing :end attribute.")?
             .as_atom()?;
+        (begin, end)
+    };
     let (rust_begin, rust_end) = get_offsets(source, rust);
     if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end {
         Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;

src/parser/sexp.rs

@@ -23,6 +23,7 @@ pub enum Token<'s> {
     Atom(&'s str),
     List(Vec<Token<'s>>),
     TextWithProperties(TextWithProperties<'s>),
+    Vector(Vec<Token<'s>>),
 }
 
 #[derive(Debug)]
@@ -73,6 +74,13 @@ enum ParseState {
 }
 
 impl<'s> Token<'s> {
+    pub fn as_vector<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
+        Ok(match self {
+            Token::Vector(children) => Ok(children),
+            _ => Err(format!("wrong token type {:?}", self)),
+        }?)
+    }
+
     pub fn as_list<'p>(&'p self) -> Result<&'p Vec<Token<'s>>, Box<dyn std::error::Error>> {
         Ok(match self {
             Token::List(children) => Ok(children),
@@ -136,7 +144,7 @@ pub fn sexp<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn token<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    alt((list, atom))(input)
+    alt((list, vector, atom))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
@@ -151,16 +159,33 @@ fn list<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::List(children)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn vector<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("[")(input)?;
+    let (remaining, children) = delimited(
+        multispace0,
+        separated_list1(multispace1, token),
+        multispace0,
+    )(remaining)?;
+    let (remaining, _) = tag("]")(remaining)?;
+    Ok((remaining, Token::Vector(children)))
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
-    not(peek(tag(")")))(input)?;
-    alt((text_with_properties, quoted_atom, unquoted_atom))(input)
+    not(peek(one_of(")]")))(input)?;
+    alt((
+        text_with_properties,
+        hash_notation,
+        quoted_atom,
+        unquoted_atom,
+    ))(input)
 }
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn unquoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, body) = take_till1(|c| match c {
-        ' ' | '\t' | '\r' | '\n' | ')' => true,
+        ' ' | '\t' | '\r' | '\n' | ')' | ']' => true,
         _ => false,
     })(input)?;
     Ok((remaining, Token::Atom(body)))
@@ -182,6 +207,18 @@ fn quoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     Ok((remaining, Token::Atom(source)))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn hash_notation<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
+    let (remaining, _) = tag("#<")(input)?;
+    let (remaining, _body) = take_till1(|c| match c {
+        '>' => true,
+        _ => false,
+    })(remaining)?;
+    let (remaining, _) = tag(">")(remaining)?;
+    let source = get_consumed(input, remaining);
+    Ok((remaining, Token::Atom(source)))
+}
+
 fn text_with_properties<'s>(input: &'s str) -> Res<&'s str, Token<'s>> {
     let (remaining, _) = tag("#(")(input)?;
     let (remaining, (text, props)) = delimited(
@@ -237,6 +274,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
     }
 
@@ -249,6 +287,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,
@@ -308,6 +347,7 @@ mod tests {
             Token::Atom(_) => false,
             Token::List(_) => true,
             Token::TextWithProperties(_) => false,
+            Token::Vector(_) => false,
         });
         let children = match parsed {
             Token::List(children) => children,