From d1e0ee831c1c466e06f9cd3c6a60852c3748c233 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 12 Aug 2023 23:14:28 -0400 Subject: [PATCH 01/12] Switch to installing emacs and org-mode from source in test container. This is to integrate fixes that have been committed to org-mode but have not made it into emacs, while also getting the latest emacs on alpine. --- docker/organic_test/Dockerfile | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docker/organic_test/Dockerfile b/docker/organic_test/Dockerfile index 392a005..f6964ad 100644 --- a/docker/organic_test/Dockerfile +++ b/docker/organic_test/Dockerfile @@ -1,4 +1,30 @@ -FROM rustlang/rust:nightly-alpine3.17 +FROM alpine:3.17 AS build -RUN apk add --no-cache musl-dev emacs +RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk + + +FROM build AS build-emacs + +RUN git clone --depth 1 --branch emacs-29.1 https://git.savannah.gnu.org/git/emacs.git /root/emacs +WORKDIR /root/emacs +RUN mkdir /root/dist +RUN ./autogen.sh +RUN ./configure --prefix /usr --without-x --without-sound +RUN make +RUN make DESTDIR="/root/dist" install + + +FROM build AS build-org-mode +COPY --from=build-emacs /root/dist/ / +RUN mkdir /root/dist +RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin 70a082c9fbf6fd7d05fb56b26c6f2039b8edd478 && git -C /root/org-mode checkout FETCH_HEAD +WORKDIR /root/org-mode +RUN make compile +RUN make DESTDIR="/root/dist" install + + +FROM rustlang/rust:nightly-alpine3.17 +RUN apk add --no-cache musl-dev ncurses gnutls RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache +COPY --from=build-emacs /root/dist/ / +COPY --from=build-org-mode /root/dist/ / From cf0991fdff52d839eeaf4b4c24450c6f9f944a99 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 13 Aug 2023 00:41:55 -0400 Subject: [PATCH 02/12] Add support for parsing vectors in the elisp parser. --- src/parser/sexp.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/parser/sexp.rs b/src/parser/sexp.rs index 2b06892..5b89153 100644 --- a/src/parser/sexp.rs +++ b/src/parser/sexp.rs @@ -23,6 +23,7 @@ pub enum Token<'s> { Atom(&'s str), List(Vec>), TextWithProperties(TextWithProperties<'s>), + Vector(Vec>), } #[derive(Debug)] @@ -136,7 +137,7 @@ pub fn sexp<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] fn token<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { - alt((list, atom))(input) + alt((list, vector, atom))(input) } #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] @@ -151,16 +152,29 @@ fn list<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { Ok((remaining, Token::List(children))) } +#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] +fn vector<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { + let (remaining, _) = tag("[")(input)?; + let (remaining, children) = delimited( + multispace0, + separated_list1(multispace1, token), + multispace0, + )(remaining)?; + let (remaining, _) = tag("]")(remaining)?; + Ok((remaining, Token::Vector(children))) +} + #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] fn atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { - not(peek(tag(")")))(input)?; + not(peek(one_of(")]")))(input)?; + // TODO: Add calls to hash notation alt((text_with_properties, quoted_atom, unquoted_atom))(input) } #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] fn unquoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { let (remaining, body) = take_till1(|c| match c { - ' ' | '\t' | '\r' | '\n' | ')' => true, + ' ' | '\t' | '\r' | '\n' | ')' | ']' => true, _ => false, })(input)?; Ok((remaining, Token::Atom(body))) @@ -237,6 +251,7 @@ mod tests { Token::Atom(_) => false, Token::List(_) => true, Token::TextWithProperties(_) => false, + Token::Vector(_) => false, }); } @@ -249,6 +264,7 @@ mod tests { Token::Atom(_) => false, Token::List(_) => true, Token::TextWithProperties(_) => false, + Token::Vector(_) => false, }); let children = match parsed { Token::List(children) => children, @@ -308,6 +324,7 @@ mod tests { Token::Atom(_) => false, Token::List(_) => true, Token::TextWithProperties(_) => false, + Token::Vector(_) => false, }); let children = match parsed { Token::List(children) => children, From f7afcec824b81d3d46af6dcd78b8050fb95dd73c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 13 Aug 2023 00:46:15 -0400 Subject: [PATCH 03/12] Add support for hash notation in the elisp parser. --- src/parser/sexp.rs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/parser/sexp.rs b/src/parser/sexp.rs index 5b89153..cb44d5f 100644 --- a/src/parser/sexp.rs +++ b/src/parser/sexp.rs @@ -167,8 +167,12 @@ fn vector<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] fn atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { not(peek(one_of(")]")))(input)?; - // TODO: Add calls to hash notation - alt((text_with_properties, quoted_atom, unquoted_atom))(input) + alt(( + text_with_properties, + hash_notation, + quoted_atom, + unquoted_atom, + ))(input) } #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] @@ -196,6 +200,18 @@ fn quoted_atom<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { Ok((remaining, Token::Atom(source))) } +#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] +fn hash_notation<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { + let (remaining, _) = tag("#<")(input)?; + let (remaining, _body) = take_till1(|c| match c { + '>' => true, + _ => false, + })(remaining)?; + let (remaining, _) = tag(">")(remaining)?; + let source = get_consumed(input, remaining); + Ok((remaining, Token::Atom(source))) +} + fn text_with_properties<'s>(input: &'s str) -> Res<&'s str, Token<'s>> { let (remaining, _) = tag("#(")(input)?; let (remaining, (text, props)) = delimited( From e33ec4a02c4e3742ecf074474c87de0a9a55e139 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 13 Aug 2023 01:06:55 -0400 Subject: [PATCH 04/12] Add support for reading begin/end bounds in the new standard-properties format. --- Makefile | 2 +- src/compare/util.rs | 33 +++++++++++++++++++++++++-------- src/parser/sexp.rs | 7 +++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index bb1cbdb..235a305 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test: .PHONY: dockertest dockertest: > $(MAKE) -C docker/organic_test -> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test" +> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --lib --test test_loader" .PHONY: integrationtest integrationtest: diff --git a/src/compare/util.rs b/src/compare/util.rs index 2386943..a3441d4 100644 --- a/src/compare/util.rs +++ b/src/compare/util.rs @@ -48,14 +48,31 @@ pub fn assert_bounds<'s, S: Source<'s>>( .nth(1) .ok_or("Should have an attributes child.")?; let attributes_map = attributes_child.as_map()?; - let begin = attributes_map - .get(":begin") - .ok_or("Missing :begin attribute.")? - .as_atom()?; - let end = attributes_map - .get(":end") - .ok_or("Missing :end attribute.")? - .as_atom()?; + let standard_properties = attributes_map.get(":standard-properties"); + let (begin, end) = if standard_properties.is_some() { + let std_props = standard_properties + .expect("if statement proves its Some") + .as_vector()?; + let begin = std_props + .get(0) + .ok_or("Missing first element in standard properties")? + .as_atom()?; + let end = std_props + .get(1) + .ok_or("Missing first element in standard properties")? + .as_atom()?; + (begin, end) + } else { + let begin = attributes_map + .get(":begin") + .ok_or("Missing :begin attribute.")? + .as_atom()?; + let end = attributes_map + .get(":end") + .ok_or("Missing :end attribute.")? + .as_atom()?; + (begin, end) + }; let (rust_begin, rust_end) = get_offsets(source, rust); if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end { Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?; diff --git a/src/parser/sexp.rs b/src/parser/sexp.rs index cb44d5f..c018879 100644 --- a/src/parser/sexp.rs +++ b/src/parser/sexp.rs @@ -74,6 +74,13 @@ enum ParseState { } impl<'s> Token<'s> { + pub fn as_vector<'p>(&'p self) -> Result<&'p Vec>, Box> { + Ok(match self { + Token::Vector(children) => Ok(children), + _ => Err(format!("wrong token type {:?}", self)), + }?) + } + pub fn as_list<'p>(&'p self) -> Result<&'p Vec>, Box> { Ok(match self { Token::List(children) => Ok(children), From b75eed6b1e00da35864ed9aca31047bbadc8fc74 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 13 Aug 2023 01:11:37 -0400 Subject: [PATCH 05/12] Enable tests that were disabled before. --- build.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.rs b/build.rs index 4c6613c..163a9b7 100644 --- a/build.rs +++ b/build.rs @@ -74,8 +74,6 @@ fn is_expect_fail(name: &str) -> Option<&str> { "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."), "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."), "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."), - "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."), - "trailing_whitespace_blank_lines_after_paragraphs" => Some("Waiting on mailing list response about ownership of the trailing blank lines."), _ => None, } } From 00354ccc2047f77665bf76e988107043a514dfca Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 10:57:48 -0400 Subject: [PATCH 06/12] Add a volume for cargo cache. This is to be a good citizen by not downloading all the rust dependencies every time I run the tests locally. Unfortunately, it will still compile all the dependencies each time, but that is a local operation. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 235a305..cc8d53f 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ test: .PHONY: dockertest dockertest: > $(MAKE) -C docker/organic_test -> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --lib --test test_loader" +> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --lib --test test_loader" .PHONY: integrationtest integrationtest: From cc83431d62f0652d85744961f2464e02de875244 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 11:57:12 -0400 Subject: [PATCH 07/12] Consume trailing whitespace for property drawers. This is a change between the org-mode in emacs 29.1 and the org-mode currently in main. --- src/parser/document.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/document.rs b/src/parser/document.rs index 0807f3b..d4a129f 100644 --- a/src/parser/document.rs +++ b/src/parser/document.rs @@ -160,7 +160,7 @@ fn zeroth_section<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s opt(parser_with_context!(comment)( &without_consuming_whitespace_context, )), - parser_with_context!(property_drawer)(&without_consuming_whitespace_context), + parser_with_context!(property_drawer)(context), many0(blank_line), )))(input)?; From 422804d8467ea1b5816162cb69493297b2994e50 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 12:18:55 -0400 Subject: [PATCH 08/12] Add script for running specific tests inside docker. --- Makefile | 4 +-- scripts/run_docker_integration_test.bash | 31 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100755 scripts/run_docker_integration_test.bash diff --git a/Makefile b/Makefile index cc8d53f..2eb2466 100644 --- a/Makefile +++ b/Makefile @@ -35,12 +35,12 @@ clean: .PHONY: test test: -> cargo test --lib --test test_loader -- --test-threads $(TESTJOBS) +> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS) .PHONY: dockertest dockertest: > $(MAKE) -C docker/organic_test -> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --lib --test test_loader" +> docker run --rm -i -t -v "$$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --no-fail-fast --lib --test test_loader" .PHONY: integrationtest integrationtest: diff --git a/scripts/run_docker_integration_test.bash b/scripts/run_docker_integration_test.bash new file mode 100755 index 0000000..0d587dc --- /dev/null +++ b/scripts/run_docker_integration_test.bash @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +set -euo pipefail +IFS=$'\n\t' +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +REALPATH=$(command -v uu-realpath || command -v realpath) + +samples_dir=$(readlink -f "$DIR/../org_mode_samples") + +function get_test_names { + for test_file in "$@" + do + if [ -e "$test_file" ]; then + test_file_full_path=$(readlink -f "$test_file") + relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path") + without_extension="${relative_to_samples%.org}" + echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]' + else + echo "$test_file" | tr '[:upper:]' '[:lower:]' + fi + done +} + +get_test_names "$@" | while read test; do + ( + cd "$DIR/../" + make -C docker/organic_test + docker run --rm -v "$(readlink -f ./):/.source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry -w / organic-test sh -c "cp -r /.source /source && cd /source && cargo test --no-fail-fast --lib --test test_loader \"$test\" -- --show-output" + ) +done From eb379af78d6ff6e1e1fba5abde5584a52af81978 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 13:13:32 -0400 Subject: [PATCH 09/12] Switch export snippet to use exit matchers. --- src/parser/export_snippet.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/parser/export_snippet.rs b/src/parser/export_snippet.rs index 851ba5e..cfc0c53 100644 --- a/src/parser/export_snippet.rs +++ b/src/parser/export_snippet.rs @@ -1,8 +1,6 @@ -use nom::branch::alt; use nom::bytes::complete::tag; use nom::character::complete::anychar; use nom::combinator::opt; -use nom::combinator::peek; use nom::combinator::recognize; use nom::combinator::verify; use nom::multi::many1; @@ -11,6 +9,9 @@ use nom::sequence::tuple; use super::Context; use crate::error::Res; +use crate::parser::exiting::ExitClass; +use crate::parser::parser_context::ContextElement; +use crate::parser::parser_context::ExitMatcherNode; use crate::parser::parser_with_context::parser_with_context; use crate::parser::util::exit_matcher_parser; use crate::parser::util::get_consumed; @@ -23,8 +24,15 @@ pub fn export_snippet<'r, 's>( ) -> Res<&'s str, ExportSnippet<'s>> { let (remaining, _) = tag("@@")(input)?; let (remaining, backend_name) = backend(context, remaining)?; - let (remaining, backend_contents) = - opt(tuple((tag(":"), parser_with_context!(contents)(context))))(remaining)?; + let parser_context = + context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode { + class: ExitClass::Beta, + exit_matcher: &export_snippet_end, + })); + let (remaining, backend_contents) = opt(tuple(( + tag(":"), + parser_with_context!(contents)(&parser_context), + )))(remaining)?; let (remaining, _) = tag("@@")(remaining)?; let source = get_consumed(input, remaining); Ok(( @@ -48,14 +56,13 @@ fn backend<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] fn contents<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { let (remaining, source) = recognize(verify( - many_till( - anychar, - peek(alt(( - parser_with_context!(exit_matcher_parser)(context), - tag("@@"), - ))), - ), + many_till(anychar, parser_with_context!(exit_matcher_parser)(context)), |(children, _exit_contents)| !children.is_empty(), ))(input)?; Ok((remaining, source)) } + +#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))] +fn export_snippet_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> { + tag("@@")(input) +} From 899073e54fcd31c1b2315397946b81bf0f3d9bdf Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 13:33:05 -0400 Subject: [PATCH 10/12] Update to the latest org-mode. --- docker/organic_test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/organic_test/Dockerfile b/docker/organic_test/Dockerfile index f6964ad..31797cb 100644 --- a/docker/organic_test/Dockerfile +++ b/docker/organic_test/Dockerfile @@ -17,7 +17,7 @@ RUN make DESTDIR="/root/dist" install FROM build AS build-org-mode COPY --from=build-emacs /root/dist/ / RUN mkdir /root/dist -RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin 70a082c9fbf6fd7d05fb56b26c6f2039b8edd478 && git -C /root/org-mode checkout FETCH_HEAD +RUN mkdir /root/org-mode && git -C /root/org-mode init && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin b89bc55867d7cb809c379d371d12d409db785154 && git -C /root/org-mode checkout FETCH_HEAD WORKDIR /root/org-mode RUN make compile RUN make DESTDIR="/root/dist" install From 92abac37e241bd49a66cd1da9249dc477bbf742a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 13:57:01 -0400 Subject: [PATCH 11/12] s/precedent/precedence/ I used the wrong word. This is referring to the priority between paragraphs ending vs export snippets ending, not a reference to something occurring in the past. --- ...ragraph_break_precedent.org => paragraph_break_precedence.org} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename org_mode_samples/export_snippet/{paragraph_break_precedent.org => paragraph_break_precedence.org} (100%) diff --git a/org_mode_samples/export_snippet/paragraph_break_precedent.org b/org_mode_samples/export_snippet/paragraph_break_precedence.org similarity index 100% rename from org_mode_samples/export_snippet/paragraph_break_precedent.org rename to org_mode_samples/export_snippet/paragraph_break_precedence.org From fbabf60559007babee9a3cc276a3f8d39ce87879 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 14 Aug 2023 14:01:00 -0400 Subject: [PATCH 12/12] Add ignore to test export_snippet_paragraph_break_precedence. --- build.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/build.rs b/build.rs index 163a9b7..5478ee2 100644 --- a/build.rs +++ b/build.rs @@ -74,6 +74,7 @@ fn is_expect_fail(name: &str) -> Option<&str> { "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."), "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."), "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."), + "export_snippet_paragraph_break_precedence" => Some("The latest code for org-mode is matching the export snippet without the closing @@."), // https://list.orgmode.org/orgmode/fb61ea28-f004-4c25-adf7-69fc55683ed4@app.fastmail.com/T/#u _ => None, } }