fixup

This shows that greater blocks of different types can be directly nested even if they are both special blocks (as long as they are different special blocks).

.lighthouse/pipeline-rust-test.yaml

@@ -99,18 +99,22 @@ spec:
         runAfter:
           - fetch-repository
       - name: run-image
-        taskRef:
-          name: run-docker-image
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            name: ""
+            workingDir: "$(workspaces.source.path)"
           workspaces:
             - name: source
-            workspace: git-source
+              mountPath: /source
             - name: cargo-cache
-            workspace: cargo-cache
-        runAfter:
-          - build-image
-        params:
-          - name: args
-            value:
+              mountPath: /usr/local/cargo/registry
+              optional: true
+          steps:
+            - name: run
+              image: "$(params.IMAGE)"
+              command: []
+              args:
                 [
                   --no-default-features,
                   --features,
@@ -120,8 +124,16 @@ spec:
                   --test,
                   test_loader,
                 ]
-          - name: docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: IMAGE
             value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+        runAfter:
+          - build-image
     finally:
       - name: report-success
         when:

Makefile

@@ -40,7 +40,7 @@ test:
 .PHONY: dockertest
 dockertest:
 > $(MAKE) -C docker/organic_test
-> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+> docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: dockerclean
 dockerclean:
@@ -60,7 +60,7 @@ jaeger:
 # 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
 #
 # These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
-> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
+> docker run -d --rm --name organicdocker --read-only -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
 
 .PHONY: jaegerweb
 jaegerweb:

build.rs

@@ -87,7 +87,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
         "autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "autogen_unicode_hearts" => Some("Unicode is coming out of emacs strange."),
         _ => None,
     }
 }

docker/cargo_fmt/Makefile

@@ -29,8 +29,8 @@ endif
 # NOTE: This target will write to folders underneath the git-root
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)

docker/organic_build/Makefile

@@ -30,8 +30,8 @@ endif
 # NOTE: This target will write to folders underneath the git-root
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)

docker/organic_test/Dockerfile

@@ -26,6 +26,7 @@ RUN make DESTDIR="/root/dist" install
 
 
 FROM rustlang/rust:nightly-alpine3.17
+ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /

docker/organic_test/Makefile

@@ -29,8 +29,8 @@ endif
 
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)

org_mode_samples/greater_element/greater_block/deeply_nested.org

@@ -0,0 +1,18 @@
+#+begin_defun
+foo
+#+begin_lorem
+,#+begin_center
+bar
+,#+end_center
+ipsum
+#+end_lorem
+baz
+#+end_defun
+
+#+begin_center
+#+begin_quote
+#+begin_center
+lorem
+#+end_center
+#+end_quote
+#+end_center

org_mode_samples/greater_element/greater_block/nested_special_block.org

@@ -0,0 +1,12 @@
+#+begin_defun
+foo
+#+begin_lorem
+ipsum
+#+end_lorem
+bar
+#+begin_center
+#+begin_quote
+baz
+#+end_quote
+#+end_center
+#+end_defun

org_mode_samples/greater_element/greater_block/special_block.org

@@ -0,0 +1,5 @@
+#+begin_defun
+foo
+
+{{{bar(baz)}}}
+#+end_defun

scripts/run_docker_compare.bash

@@ -41,6 +41,7 @@ function launch_container {
     if [ "$SHELL" != "YES" ]; then
         local features_joined=$(IFS=","; echo "${features[*]}")
         additional_args+=(cargo run --no-default-features --features "$features_joined")
+        additional_flags+=(--read-only)
     else
         additional_args+=(/bin/sh)
         additional_flags+=(-t)
@@ -50,7 +51,7 @@ function launch_container {
         additional_flags+=(--env RUST_BACKTRACE=full)
     fi
 
-    docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
+    docker run "${additional_flags[@]}" --init --rm -i --mount type=tmpfs,destination=/tmp -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
 }
 
 main "${@}"

scripts/run_docker_integration_test.bash

@@ -33,7 +33,7 @@ function get_test_names {
             local test_file_full_path=$($REALPATH "$test_file")
             local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
             local without_extension="${relative_to_samples%.org}"
-            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+            echo "autogen_${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
         fi
@@ -56,7 +56,7 @@ cargo test --no-default-features --features compare --no-fail-fast --lib --test
 EOF
           )
 
-    docker run "${additional_flags[@]}" --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
+    docker run "${additional_flags[@]}" --init --rm --read-only --mount type=tmpfs,destination=/tmp -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
 }
 
 

scripts/run_integration_test.bash

@@ -25,7 +25,7 @@ function get_test_names {
             local test_file_full_path=$($REALPATH "$test_file")
             local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
             local without_extension="${relative_to_samples%.org}"
-            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+            echo "${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
         fi

src/compare/diff.rs

@@ -737,7 +737,7 @@ fn compare_greater_block<'s>(
     let emacs_name = match rust.name.to_lowercase().as_str() {
         "center" => "center-block",
         "quote" => "quote-block",
-        _ => todo!(),
+        _ => "special-block",
     };
     if assert_name(emacs, emacs_name).is_err() {
         this_status = DiffStatus::Bad;
@@ -1457,7 +1457,7 @@ fn compare_plain_text<'s>(
         .as_atom()?
         .parse()?;
     let emacs_text_length = end_ind - start_ind;
-    if rust_source.len() != emacs_text_length {
+    if rust_source.chars().count() != emacs_text_length {
         this_status = DiffStatus::Bad;
         message = Some(format!(
             "(emacs len != rust len) {:?} != {:?}",

src/compare/util.rs

@@ -13,7 +13,7 @@ fn is_slice_of(parent: &str, child: &str) -> bool {
 /// Get the offset into source that the rust object exists at.
 ///
 /// These offsets are zero-based unlike the elisp ones.
-pub fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
+fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
     let rust_object_source = rust_object.get_source();
     assert!(is_slice_of(source, rust_object_source));
     let offset = rust_object_source.as_ptr() as usize - source.as_ptr() as usize;
@@ -50,8 +50,11 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         standard_properties.end.ok_or("Token should have an end.")?,
     );
     let (rust_begin, rust_end) = get_offsets(source, rust);
-    if (rust_begin + 1) != begin || (rust_end + 1) != end {
-        Err(format!("Rust bounds (in bytes) ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;
+    let rust_begin_char_offset = (&source[..rust_begin]).chars().count();
+    let rust_end_char_offset =
+        rust_begin_char_offset + (&source[rust_begin..rust_end]).chars().count();
+    if (rust_begin_char_offset + 1) != begin || (rust_end_char_offset + 1) != end {
+        Err(format!("Rust bounds (in chars) ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin_char_offset + 1, rust_end = rust_end_char_offset + 1, emacs_begin=begin, emacs_end=end))?;
     }
 
     Ok(())

src/parser/citation.rs

@@ -2,7 +2,6 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
-
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;

src/parser/element_parser.rs

@@ -128,7 +128,7 @@ fn _detect_element<'r, 's>(
     input: OrgSource<'s>,
     can_be_paragraph: bool,
 ) -> Res<OrgSource<'s>, ()> {
-    if detect_plain_list(context, input).is_ok() {
+    if detect_plain_list(input).is_ok() {
         return Ok((input, ()));
     }
     if _element(context, input, can_be_paragraph).is_ok() {

src/parser/entity.rs

@@ -2,7 +2,6 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::satisfy;
-
 use nom::combinator::eof;
 use nom::combinator::peek;
 use nom::combinator::recognize;

src/parser/footnote_definition.rs

@@ -80,21 +80,10 @@ fn footnote_definition_end<'r, 's>(
     context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    let allow_nesting_context =
-        context.with_additional_node(ContextElement::Context("allow nesting footnotes"));
-    let footnote_definition_matcher = parser_with_context!(footnote_definition)(
-        if immediate_in_section(context, "footnote definition") {
-            &allow_nesting_context
-        } else {
-            context
-        },
-    );
-    let maybe_consume_trailing_whitespace_matcher =
-        parser_with_context!(maybe_consume_trailing_whitespace)(context);
     let (remaining, source) = alt((
         recognize(tuple((
-            maybe_consume_trailing_whitespace_matcher,
-            footnote_definition_matcher,
+            parser_with_context!(maybe_consume_trailing_whitespace)(context),
+            detect_footnote_definition,
         ))),
         recognize(tuple((
             start_of_line,
@@ -107,6 +96,12 @@ fn footnote_definition_end<'r, 's>(
     Ok((remaining, source))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn detect_footnote_definition<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((start_of_line, tag_no_case("[fn:"), label, tag("]")))(input)?;
+    Ok((input, ()))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/parser/greater_block.rs

@@ -12,6 +12,7 @@ use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::in_section;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -26,7 +27,6 @@ use crate::parser::source::SetSource;
 use crate::parser::util::blank_line;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
-use crate::parser::util::immediate_in_section;
 use crate::parser::util::start_of_line;
 use crate::parser::Element;
 use crate::parser::Paragraph;
@@ -49,11 +49,11 @@ pub fn greater_block<'r, 's>(
         }),
     ))(remaining)?;
     let context_name = match Into::<&str>::into(name).to_lowercase().as_str() {
-        "center" => "center block",
-        "quote" => "quote block",
-        _ => "greater block",
+        "center" => "center block".to_owned(),
+        "quote" => "quote block".to_owned(),
+        name @ _ => format!("special block {}", name),
     };
-    if immediate_in_section(context, context_name) {
+    if in_section(context, context_name.as_str()) {
         return Err(nom::Err::Error(CustomError::MyError(MyError(
             "Cannot nest objects of the same element".into(),
         ))));
@@ -63,7 +63,7 @@ pub fn greater_block<'r, 's>(
     let (remaining, _nl) = line_ending(remaining)?;
     let parser_context = context
         .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
-        .with_additional_node(ContextElement::Context(context_name))
+        .with_additional_node(ContextElement::Context(context_name.as_str()))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
             class: ExitClass::Alpha,
             exit_matcher: &exit_with_name,

src/parser/inline_babel_call.rs

@@ -4,7 +4,6 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
-
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;

src/parser/inline_source_block.rs

@@ -4,7 +4,6 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
-
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;

src/parser/plain_list.rs

@@ -38,10 +38,7 @@ use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting;
 use crate::parser::util::start_of_line;
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-pub fn detect_plain_list<'r, 's>(
-    _context: Context<'r, 's>,
-    input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, ()> {
+pub fn detect_plain_list<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
     if verify(
         tuple((
             start_of_line,
@@ -522,24 +519,21 @@ dolar"#,
             r#"+
 "#,
         );
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 
     #[test]
     fn detect_eof() {
         let input = OrgSource::new(r#"+"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 
     #[test]
     fn detect_no_gap() {
         let input = OrgSource::new(r#"+foo"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         // Since there is no whitespace after the '+' this is a paragraph, not a plain list.
         assert!(result.is_err());
     }
@@ -547,8 +541,7 @@ dolar"#,
     #[test]
     fn detect_with_gap() {
         let input = OrgSource::new(r#"+ foo"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 }

src/parser/regular_link.rs

@@ -3,7 +3,6 @@ use nom::bytes::complete::escaped;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::take_till1;
 use nom::character::complete::one_of;
-
 use nom::combinator::verify;
 use nom::multi::many_till;
 

src/parser/subscript_and_superscript.rs

@@ -2,7 +2,6 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::character::complete::one_of;
-
 use nom::combinator::map;
 use nom::combinator::not;
 use nom::combinator::opt;

src/parser/target.rs

@@ -1,7 +1,6 @@
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::character::complete::one_of;
-
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;