Merge branch 'file_based_diff'

We were running into issues where the documents grew too large for being passed as a string to emacs, and we need to handle #+setupfile so we need to start handling org-mode documents as files and not just as anonymous streams of text.

The anonymous stream of text handling will remain because the automated tests use it.

Cargo.toml

@@ -22,7 +22,7 @@ path = "src/lib.rs"
 
 [[bin]]
 # This bin exists for development purposes only. The real target of this crate is the library.
-name = "compare"
+name = "parse"
 path = "src/main.rs"
 
 [dependencies]

Makefile

@@ -40,7 +40,7 @@ test:
 .PHONY: dockertest
 dockertest:
 > $(MAKE) -C docker/organic_test
-> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+> docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: dockerclean
 dockerclean:
@@ -60,7 +60,7 @@ jaeger:
 # 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
 #
 # These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
-> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
+> docker run -d --rm --name organicdocker --read-only -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
 
 .PHONY: jaegerweb
 jaegerweb:

build.rs

@@ -73,7 +73,7 @@ fn write_header(test_file: &mut File) {
 #[feature(exit_status_error)]
 use organic::compare_document;
 use organic::parser::document;
-use organic::emacs_parse_org_document;
+use organic::emacs_parse_anonymous_org_document;
 use organic::parser::sexp::sexp_with_padding;
 
 "#
@@ -87,7 +87,6 @@ fn is_expect_fail(name: &str) -> Option<&str> {
         "autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
         "autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
         "autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "autogen_unicode_hearts" => Some("Unicode is coming out of emacs strange."),
         _ => None,
     }
 }

docker/cargo_fmt/Makefile

@@ -29,8 +29,8 @@ endif
 # NOTE: This target will write to folders underneath the git-root
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)

docker/organic_build/Makefile

@@ -30,8 +30,8 @@ endif
 # NOTE: This target will write to folders underneath the git-root
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)

docker/organic_test/Dockerfile

@@ -26,6 +26,7 @@ RUN make DESTDIR="/root/dist" install
 
 
 FROM rustlang/rust:nightly-alpine3.17
+ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /

docker/organic_test/Makefile

@@ -29,8 +29,8 @@ endif
 
 .PHONY: run
 run: build
-	docker run --rm --init -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
 
 .PHONY: shell
 shell: build
-	docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)

org_mode_samples/greater_element/greater_block/deeply_nested.org

@@ -0,0 +1,18 @@
+#+begin_defun
+foo
+#+begin_lorem
+,#+begin_center
+bar
+,#+end_center
+ipsum
+#+end_lorem
+baz
+#+end_defun
+
+#+begin_center
+#+begin_quote
+#+begin_center
+lorem
+#+end_center
+#+end_quote
+#+end_center

org_mode_samples/greater_element/greater_block/nested_special_block.org

@@ -0,0 +1,12 @@
+#+begin_defun
+foo
+#+begin_lorem
+ipsum
+#+end_lorem
+bar
+#+begin_center
+#+begin_quote
+baz
+#+end_quote
+#+end_center
+#+end_defun

org_mode_samples/greater_element/greater_block/special_block.org

@@ -0,0 +1,5 @@
+#+begin_defun
+foo
+
+{{{bar(baz)}}}
+#+end_defun

org_mode_samples/greater_element/plain_list/description_list_object_key.org

@@ -0,0 +1 @@
+- {{{foo(bar)}}} :: baz

org_mode_samples/greater_element/plain_list/trailing_whitespace_eof.org

@@ -0,0 +1,2 @@
+- foo
+

org_mode_samples/lesser_element/diary_sexp/with_comment_at_end_of_line.org

@@ -0,0 +1 @@
+%%(foo bar)    ;    baz

org_mode_samples/object/text_markup/verbatim_with_equals_inside.org

@@ -0,0 +1 @@
+foo ==>bar=.

scripts/callgrind.bash

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$DIR/../"
 
 RUSTFLAGS="-C opt-level=0" cargo build --no-default-features
-valgrind --tool=callgrind --callgrind-out-file=callgrind.out target/debug/compare
+valgrind --tool=callgrind --callgrind-out-file=callgrind.out target/debug/parse
 
 echo "You probably want to run:"
 echo "callgrind_annotate --auto=yes callgrind.out"

scripts/perf.bash

@@ -16,7 +16,7 @@ function main {
         additional_flags+=(--profile "$PROFILE")
     fi
     cargo build --no-default-features "${additional_flags[@]}"
-    perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/compare
+    perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/parse
 
     # Convert to a format firefox will read
     # flags to consider --show-info

scripts/run_docker_compare.bash

@@ -9,14 +9,13 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 : ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
 : ${NO_COLOR:=""} # Set to anything to disable color output
 
-
 cd "$DIR/../"
 REALPATH=$(command -v uu-realpath || command -v realpath)
 MAKE=$(command -v gmake || command -v make)
 
 function main {
     build_container
-    launch_container
+    launch_container "${@}"
 }
 
 function build_container {
@@ -41,6 +40,7 @@ function launch_container {
     if [ "$SHELL" != "YES" ]; then
         local features_joined=$(IFS=","; echo "${features[*]}")
         additional_args+=(cargo run --no-default-features --features "$features_joined")
+        additional_flags+=(--read-only)
     else
         additional_args+=(/bin/sh)
         additional_flags+=(-t)
@@ -50,7 +50,17 @@ function launch_container {
         additional_flags+=(--env RUST_BACKTRACE=full)
     fi
 
-    docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
+    if [ $# -gt 0 ]; then
+        # If we passed in args, we need to forward them along
+        for path in "${@}"; do
+            local full_path=$($REALPATH "$path")
+            local containing_folder=$(dirname "$full_path")
+            local file_name=$(basename "$full_path")
+            docker run "${additional_flags[@]}" --init --rm -i --mount type=tmpfs,destination=/tmp -v "${containing_folder}:/input:ro"  -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}" -- "/input/$file_name"
+        done
+    else
+        docker run "${additional_flags[@]}" --init --rm -i --mount type=tmpfs,destination=/tmp -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}"
+    fi
 }
 
 main "${@}"

scripts/run_docker_integration_test.bash

@@ -33,7 +33,7 @@ function get_test_names {
             local test_file_full_path=$($REALPATH "$test_file")
             local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
             local without_extension="${relative_to_samples%.org}"
-            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+            echo "autogen_${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
         fi
@@ -56,7 +56,7 @@ cargo test --no-default-features --features compare --no-fail-fast --lib --test
 EOF
           )
 
-    docker run "${additional_flags[@]}" --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
+    docker run "${additional_flags[@]}" --init --rm --read-only --mount type=tmpfs,destination=/tmp -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script"
 }
 
 

scripts/run_integration_test.bash

@@ -25,7 +25,7 @@ function get_test_names {
             local test_file_full_path=$($REALPATH "$test_file")
             local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
             local without_extension="${relative_to_samples%.org}"
-            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+            echo "${without_extension//\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
         fi

scripts/time_parse.bash

@@ -17,7 +17,7 @@ function main {
         additional_flags+=(--profile "$PROFILE")
     fi
     cargo build --no-default-features "${additional_flags[@]}"
-    time ./target/${PROFILE}/compare
+    time ./target/${PROFILE}/parse "${@}"
 }
 
 main "${@}"

src/compare/diff.rs

@@ -737,7 +737,7 @@ fn compare_greater_block<'s>(
     let emacs_name = match rust.name.to_lowercase().as_str() {
         "center" => "center-block",
         "quote" => "quote-block",
-        _ => todo!(),
+        _ => "special-block",
     };
     if assert_name(emacs, emacs_name).is_err() {
         this_status = DiffStatus::Bad;
@@ -1457,7 +1457,7 @@ fn compare_plain_text<'s>(
         .as_atom()?
         .parse()?;
     let emacs_text_length = end_ind - start_ind;
-    if rust_source.len() != emacs_text_length {
+    if rust_source.chars().count() != emacs_text_length {
         this_status = DiffStatus::Bad;
         message = Some(format!(
             "(emacs len != rust len) {:?} != {:?}",

src/compare/mod.rs

@@ -2,6 +2,7 @@ mod diff;
 mod parse;
 mod util;
 pub use diff::compare_document;
-pub use parse::emacs_parse_org_document;
+pub use parse::emacs_parse_anonymous_org_document;
+pub use parse::emacs_parse_file_org_document;
 pub use parse::get_emacs_version;
 pub use parse::get_org_mode_version;

src/compare/parse.rs

@@ -1,6 +1,9 @@
+use std::path::Path;
 use std::process::Command;
 
-pub fn emacs_parse_org_document<C>(file_contents: C) -> Result<String, Box<dyn std::error::Error>>
+pub fn emacs_parse_anonymous_org_document<C>(
+    file_contents: C,
+) -> Result<String, Box<dyn std::error::Error>>
 where
     C: AsRef<str>,
 {
@@ -15,14 +18,46 @@ where
         escaped_file_contents = escaped_file_contents
     );
     let mut cmd = Command::new("emacs");
-    let proc = cmd
+    let cmd = cmd
         .arg("-q")
         .arg("--no-site-file")
         .arg("--no-splash")
         .arg("--batch")
         .arg("--eval")
         .arg(elisp_script);
-    let out = proc.output()?;
+    let out = cmd.output()?;
+    out.status.exit_ok()?;
+    let org_sexp = out.stderr;
+    Ok(String::from_utf8(org_sexp)?)
+}
+
+pub fn emacs_parse_file_org_document<P>(file_path: P) -> Result<String, Box<dyn std::error::Error>>
+where
+    P: AsRef<Path>,
+{
+    let file_path = file_path.as_ref().canonicalize()?;
+    let containing_directory = file_path.parent().ok_or(format!(
+        "Failed to get containing directory for path {}",
+        file_path.display()
+    ))?;
+    let elisp_script = format!(
+        r#"(progn
+     (org-mode)
+     (message "%s" (pp-to-string (org-element-parse-buffer)))
+)"#
+    );
+    let mut cmd = Command::new("emacs");
+    let cmd = cmd
+        .current_dir(containing_directory)
+        .arg("-q")
+        .arg("--no-site-file")
+        .arg("--no-splash")
+        .arg("--batch")
+        .arg("--insert")
+        .arg(file_path.as_os_str())
+        .arg("--eval")
+        .arg(elisp_script);
+    let out = cmd.output()?;
     out.status.exit_ok()?;
     let org_sexp = out.stderr;
     Ok(String::from_utf8(org_sexp)?)
@@ -55,7 +90,7 @@ pub fn get_emacs_version() -> Result<String, Box<dyn std::error::Error>> {
      (message "%s" (version))
 )"#;
     let mut cmd = Command::new("emacs");
-    let proc = cmd
+    let cmd = cmd
         .arg("-q")
         .arg("--no-site-file")
         .arg("--no-splash")
@@ -63,7 +98,7 @@ pub fn get_emacs_version() -> Result<String, Box<dyn std::error::Error>> {
         .arg("--eval")
         .arg(elisp_script);
 
-    let out = proc.output()?;
+    let out = cmd.output()?;
     out.status.exit_ok()?;
     Ok(String::from_utf8(out.stderr)?)
 }
@@ -74,7 +109,7 @@ pub fn get_org_mode_version() -> Result<String, Box<dyn std::error::Error>> {
      (message "%s" (org-version nil t nil))
 )"#;
     let mut cmd = Command::new("emacs");
-    let proc = cmd
+    let cmd = cmd
         .arg("-q")
         .arg("--no-site-file")
         .arg("--no-splash")
@@ -82,7 +117,7 @@ pub fn get_org_mode_version() -> Result<String, Box<dyn std::error::Error>> {
         .arg("--eval")
         .arg(elisp_script);
 
-    let out = proc.output()?;
+    let out = cmd.output()?;
     out.status.exit_ok()?;
     Ok(String::from_utf8(out.stderr)?)
 }

src/compare/util.rs

@@ -13,7 +13,7 @@ fn is_slice_of(parent: &str, child: &str) -> bool {
 /// Get the offset into source that the rust object exists at.
 ///
 /// These offsets are zero-based unlike the elisp ones.
-pub fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
+fn get_offsets<'s, S: Source<'s>>(source: &'s str, rust_object: &'s S) -> (usize, usize) {
     let rust_object_source = rust_object.get_source();
     assert!(is_slice_of(source, rust_object_source));
     let offset = rust_object_source.as_ptr() as usize - source.as_ptr() as usize;
@@ -50,8 +50,11 @@ pub fn assert_bounds<'s, S: Source<'s>>(
         standard_properties.end.ok_or("Token should have an end.")?,
     );
     let (rust_begin, rust_end) = get_offsets(source, rust);
-    if (rust_begin + 1) != begin || (rust_end + 1) != end {
-        Err(format!("Rust bounds (in bytes) ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;
+    let rust_begin_char_offset = (&source[..rust_begin]).chars().count();
+    let rust_end_char_offset =
+        rust_begin_char_offset + (&source[rust_begin..rust_end]).chars().count();
+    if (rust_begin_char_offset + 1) != begin || (rust_end_char_offset + 1) != end {
+        Err(format!("Rust bounds (in chars) ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin_char_offset + 1, rust_end = rust_end_char_offset + 1, emacs_begin=begin, emacs_end=end))?;
     }
 
     Ok(())

src/lib.rs

@@ -6,7 +6,9 @@ mod compare;
 #[cfg(feature = "compare")]
 pub use compare::compare_document;
 #[cfg(feature = "compare")]
-pub use compare::emacs_parse_org_document;
+pub use compare::emacs_parse_anonymous_org_document;
+#[cfg(feature = "compare")]
+pub use compare::emacs_parse_file_org_document;
 #[cfg(feature = "compare")]
 pub use compare::get_emacs_version;
 #[cfg(feature = "compare")]

src/main.rs

@@ -1,11 +1,15 @@
 #![feature(round_char_boundary)]
+#![feature(exact_size_is_empty)]
 use std::io::Read;
+use std::path::Path;
 
 use ::organic::parser::document;
 #[cfg(feature = "compare")]
 use organic::compare_document;
 #[cfg(feature = "compare")]
-use organic::emacs_parse_org_document;
+use organic::emacs_parse_anonymous_org_document;
+#[cfg(feature = "compare")]
+use organic::emacs_parse_file_org_document;
 #[cfg(feature = "compare")]
 use organic::get_emacs_version;
 #[cfg(feature = "compare")]
@@ -39,8 +43,16 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn main_body() -> Result<(), Box<dyn std::error::Error>> {
-    let org_contents = read_stdin_to_string()?;
-    run_compare(org_contents)
+    let args = std::env::args();
+    if args.is_empty() {
+        let org_contents = read_stdin_to_string()?;
+        run_anonymous_parse(org_contents)
+    } else {
+        for arg in args.skip(1) {
+            run_parse_on_file(arg)?
+        }
+        Ok(())
+    }
 }
 
 fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
@@ -52,14 +64,12 @@ fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
 }
 
 #[cfg(feature = "compare")]
-fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
-    let emacs_version = get_emacs_version()?;
-    let org_mode_version = get_org_mode_version()?;
+fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
     let org_contents = org_contents.as_ref();
-    eprintln!("Using emacs version: {}", emacs_version.trim());
-    eprintln!("Using org-mode version: {}", org_mode_version.trim());
+    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
+    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
     let (remaining, rust_parsed) = document(org_contents).map_err(|e| e.to_string())?;
-    let org_sexp = emacs_parse_org_document(org_contents)?;
+    let org_sexp = emacs_parse_anonymous_org_document(org_contents)?;
     let (_remaining, parsed_sexp) =
         sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
 
@@ -82,7 +92,7 @@ fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error:
 }
 
 #[cfg(not(feature = "compare"))]
-fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
+fn run_anonymous_parse<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
     eprintln!(
         "This program was built with compare disabled. Only parsing with organic, not comparing."
     );
@@ -90,3 +100,48 @@ fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error:
     println!("{:#?}", rust_parsed);
     Ok(())
 }
+
+#[cfg(feature = "compare")]
+fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
+    let org_path = org_path.as_ref();
+    eprintln!("Using emacs version: {}", get_emacs_version()?.trim());
+    eprintln!("Using org-mode version: {}", get_org_mode_version()?.trim());
+    // TODO: This should take into account the original file path when parsing in Organic, not just in emacs.
+    let org_contents = std::fs::read_to_string(org_path)?;
+    let org_contents = org_contents.as_str();
+    let (remaining, rust_parsed) = document(org_contents).map_err(|e| e.to_string())?;
+    let org_sexp = emacs_parse_file_org_document(org_path)?;
+    let (_remaining, parsed_sexp) =
+        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
+
+    println!("{}\n\n\n", org_contents);
+    println!("{}", org_sexp);
+    println!("{:#?}", rust_parsed);
+
+    // We do the diffing after printing out both parsed forms in case the diffing panics
+    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
+    diff_result.print(org_contents)?;
+
+    if diff_result.is_bad() {
+        Err("Diff results do not match.")?;
+    }
+    if remaining != "" {
+        Err(format!("There was unparsed text remaining: {}", remaining))?;
+    }
+
+    Ok(())
+}
+
+#[cfg(not(feature = "compare"))]
+fn run_parse_on_file<P: AsRef<Path>>(org_path: P) -> Result<(), Box<dyn std::error::Error>> {
+    let org_path = org_path.as_ref();
+    eprintln!(
+        "This program was built with compare disabled. Only parsing with organic, not comparing."
+    );
+    // TODO: This should take into account the original file path when parsing
+    let org_contents = std::fs::read_to_string(org_path)?;
+    let org_contents = org_contents.as_str();
+    let (_remaining, rust_parsed) = document(org_contents).map_err(|e| e.to_string())?;
+    println!("{:#?}", rust_parsed);
+    Ok(())
+}

src/parser/angle_link.rs

@@ -5,6 +5,7 @@ use nom::combinator::recognize;
 use nom::multi::many_till;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
@@ -26,6 +27,8 @@ pub fn angle_link<'r, 's>(
     let (remaining, _separator) = tag(":")(remaining)?;
     let (remaining, path) = path_angle(context, remaining)?;
     let (remaining, _) = tag(">")(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/citation.rs

@@ -2,7 +2,6 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
-use nom::character::complete::space0;
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;
@@ -14,6 +13,7 @@ use nom::sequence::tuple;
 use super::citation_reference::must_balance_bracket;
 use super::org_source::BracketDepth;
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::Res;
@@ -48,7 +48,8 @@ pub fn citation<'r, 's>(
         parser_with_context!(global_suffix)(context),
     ))))(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/diary_sexp.rs

@@ -1,9 +1,12 @@
 use nom::branch::alt;
 use nom::bytes::complete::tag;
+use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::space0;
 use nom::combinator::eof;
+use nom::combinator::opt;
 use nom::combinator::recognize;
+use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
@@ -24,6 +27,11 @@ pub fn diary_sexp<'r, 's>(
     let (remaining, _clock) = tag("%%")(remaining)?;
     let (remaining, _gap_whitespace) = space0(remaining)?;
     let (remaining, _sexp) = recognize(sexp)(remaining)?;
+    let (remaining, _trailing_comment) = opt(tuple((
+        space0,
+        tag(";"),
+        many_till(anychar, alt((line_ending, eof))),
+    )))(remaining)?;
     let (remaining, _trailing_whitespace) =
         recognize(tuple((space0, alt((line_ending, eof)))))(remaining)?;
 

src/parser/element_parser.rs

@@ -128,7 +128,7 @@ fn _detect_element<'r, 's>(
     input: OrgSource<'s>,
     can_be_paragraph: bool,
 ) -> Res<OrgSource<'s>, ()> {
-    if detect_plain_list(context, input).is_ok() {
+    if detect_plain_list(input).is_ok() {
         return Ok((input, ()));
     }
     if _element(context, input, can_be_paragraph).is_ok() {

src/parser/entity.rs

@@ -2,12 +2,12 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
 use nom::character::complete::satisfy;
-use nom::character::complete::space0;
 use nom::combinator::eof;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -440,7 +440,8 @@ pub fn entity<'r, 's>(
     let (remaining, _) = tag("\\")(input)?;
     let (remaining, entity_name) = name(context, remaining)?;
     let (remaining, _) = alt((tag("{}"), peek(recognize(entity_end))))(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
 
     let source = get_consumed(input, remaining);
     Ok((

src/parser/export_snippet.rs

@@ -8,6 +8,7 @@ use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
@@ -35,6 +36,8 @@ pub fn export_snippet<'r, 's>(
         parser_with_context!(contents)(&parser_context),
     )))(remaining)?;
     let (remaining, _) = tag("@@")(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/footnote_definition.rs

@@ -80,21 +80,10 @@ fn footnote_definition_end<'r, 's>(
     context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, OrgSource<'s>> {
-    let allow_nesting_context =
-        context.with_additional_node(ContextElement::Context("allow nesting footnotes"));
-    let footnote_definition_matcher = parser_with_context!(footnote_definition)(
-        if immediate_in_section(context, "footnote definition") {
-            &allow_nesting_context
-        } else {
-            context
-        },
-    );
-    let maybe_consume_trailing_whitespace_matcher =
-        parser_with_context!(maybe_consume_trailing_whitespace)(context);
     let (remaining, source) = alt((
         recognize(tuple((
-            maybe_consume_trailing_whitespace_matcher,
-            footnote_definition_matcher,
+            parser_with_context!(maybe_consume_trailing_whitespace)(context),
+            detect_footnote_definition,
         ))),
         recognize(tuple((
             start_of_line,
@@ -107,6 +96,12 @@ fn footnote_definition_end<'r, 's>(
     Ok((remaining, source))
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn detect_footnote_definition<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
+    tuple((start_of_line, tag_no_case("[fn:"), label, tag("]")))(input)?;
+    Ok((input, ()))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/parser/footnote_reference.rs

@@ -1,13 +1,13 @@
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::tag_no_case;
-use nom::character::complete::space0;
 use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::org_source::BracketDepth;
 use super::org_source::OrgSource;
 use super::parser_context::ContextElement;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -54,7 +54,8 @@ fn anonymous_footnote<'r, 's>(
     )(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,
@@ -89,7 +90,8 @@ fn inline_footnote<'r, 's>(
     )(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,
@@ -103,13 +105,14 @@ fn inline_footnote<'r, 's>(
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn footnote_reference_only<'r, 's>(
-    _context: Context<'r, 's>,
+    context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, FootnoteReference<'s>> {
     let (remaining, _) = tag_no_case("[fn:")(input)?;
     let (remaining, label_contents) = label(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/greater_block.rs

@@ -12,6 +12,7 @@ use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::in_section;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -26,7 +27,6 @@ use crate::parser::source::SetSource;
 use crate::parser::util::blank_line;
 use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
-use crate::parser::util::immediate_in_section;
 use crate::parser::util::start_of_line;
 use crate::parser::Element;
 use crate::parser::Paragraph;
@@ -49,11 +49,11 @@ pub fn greater_block<'r, 's>(
         }),
     ))(remaining)?;
     let context_name = match Into::<&str>::into(name).to_lowercase().as_str() {
-        "center" => "center block",
-        "quote" => "quote block",
-        _ => "greater block",
+        "center" => "center block".to_owned(),
+        "quote" => "quote block".to_owned(),
+        name @ _ => format!("special block {}", name),
     };
-    if immediate_in_section(context, context_name) {
+    if in_section(context, context_name.as_str()) {
         return Err(nom::Err::Error(CustomError::MyError(MyError(
             "Cannot nest objects of the same element".into(),
         ))));
@@ -63,7 +63,7 @@ pub fn greater_block<'r, 's>(
     let (remaining, _nl) = line_ending(remaining)?;
     let parser_context = context
         .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
-        .with_additional_node(ContextElement::Context(context_name))
+        .with_additional_node(ContextElement::Context(context_name.as_str()))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
             class: ExitClass::Alpha,
             exit_matcher: &exit_with_name,

src/parser/inline_babel_call.rs

@@ -4,7 +4,6 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;
@@ -12,6 +11,7 @@ use nom::multi::many_till;
 
 use super::org_source::BracketDepth;
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -34,7 +34,8 @@ pub fn inline_babel_call<'r, 's>(
     let (remaining, _header1) = opt(parser_with_context!(header)(context))(remaining)?;
     let (remaining, _argument) = argument(context, remaining)?;
     let (remaining, _header2) = opt(parser_with_context!(header)(context))(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/inline_source_block.rs

@@ -4,7 +4,6 @@ use nom::bytes::complete::tag_no_case;
 use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::opt;
 use nom::combinator::recognize;
 use nom::combinator::verify;
@@ -14,6 +13,7 @@ use tracing::span;
 
 use super::org_source::BracketDepth;
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -35,7 +35,8 @@ pub fn inline_source_block<'r, 's>(
     let (remaining, _) = lang(context, remaining)?;
     let (remaining, _header1) = opt(parser_with_context!(header)(context))(remaining)?;
     let (remaining, _body) = body(context, remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/latex_fragment.rs

@@ -5,7 +5,6 @@ use nom::character::complete::anychar;
 use nom::character::complete::line_ending;
 use nom::character::complete::none_of;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
@@ -14,6 +13,7 @@ use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -36,7 +36,8 @@ pub fn latex_fragment<'r, 's>(
         parser_with_context!(dollar_char_fragment)(context),
         parser_with_context!(bordered_dollar_fragment)(context),
     ))(input)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/org_macro.rs

@@ -1,6 +1,5 @@
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
-use nom::character::complete::space0;
 use nom::combinator::not;
 use nom::combinator::opt;
 use nom::combinator::peek;
@@ -9,7 +8,9 @@ use nom::multi::many0;
 use nom::multi::separated_list0;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
+use crate::error::CustomError;
 use crate::error::Res;
 use crate::parser::object::OrgMacro;
 use crate::parser::parser_with_context::parser_with_context;
@@ -25,7 +26,8 @@ pub fn org_macro<'r, 's>(
     let (remaining, macro_name) = org_macro_name(context, remaining)?;
     let (remaining, macro_args) = opt(parser_with_context!(org_macro_args)(context))(remaining)?;
     let (remaining, _) = tag("}}}")(remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
 
     let source = get_consumed(input, remaining);
     Ok((
@@ -86,6 +88,11 @@ fn org_macro_arg<'r, 's>(
         }
         if next_char == '\\' {
             escaping = true;
+            if peek(tag::<_, _, CustomError<_>>(")"))(new_remaining).is_ok() {
+                // Special case for backslash at the end of a macro
+                remaining = new_remaining;
+                break;
+            }
         }
         if next_char == ',' || next_char == ')' {
             break;

src/parser/plain_list.rs

@@ -38,10 +38,7 @@ use crate::parser::util::maybe_consume_trailing_whitespace_if_not_exiting;
 use crate::parser::util::start_of_line;
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
-pub fn detect_plain_list<'r, 's>(
-    _context: Context<'r, 's>,
-    input: OrgSource<'s>,
-) -> Res<OrgSource<'s>, ()> {
+pub fn detect_plain_list<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
     if verify(
         tuple((
             start_of_line,
@@ -69,6 +66,7 @@ pub fn plain_list<'r, 's>(
 ) -> Res<OrgSource<'s>, PlainList<'s>> {
     let parser_context = context
         .with_additional_node(ContextElement::Context("plain list"))
+        .with_additional_node(ContextElement::ConsumeTrailingWhitespace(true))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
             class: ExitClass::Beta,
             exit_matcher: &plain_list_end,
@@ -143,29 +141,32 @@ pub fn plain_list_item<'r, 's>(
         Into::<&str>::into(bull) != "*" || indent_level > 0
     })(remaining)?;
 
-    let maybe_contentless_item: Res<OrgSource<'_>, OrgSource<'_>> = eof(remaining);
+    let (remaining, maybe_tag) = opt(tuple((
+        space1,
+        parser_with_context!(item_tag)(context),
+        tag(" ::"),
+    )))(remaining)?;
+    let maybe_contentless_item: Res<OrgSource<'_>, OrgSource<'_>> =
+        peek(recognize(tuple((many0(blank_line), eof))))(remaining);
     match maybe_contentless_item {
-        Ok((rem, _ws)) => {
-            let source = get_consumed(input, rem);
+        Ok((_rem, _ws)) => {
+            let (remaining, _trailing_ws) = opt(blank_line)(remaining)?;
+            let source = get_consumed(input, remaining);
             return Ok((
-                rem,
+                remaining,
                 PlainListItem {
                     source: source.into(),
                     indentation: indent_level,
                     bullet: bull.into(),
-                    tag: Vec::new(),
+                    tag: maybe_tag
+                        .map(|(_ws, item_tag, _divider)| item_tag)
+                        .unwrap_or(Vec::new()),
                     children: Vec::new(),
                 },
             ));
         }
         Err(_) => {}
     };
-
-    let (remaining, maybe_tag) = opt(tuple((
-        space1,
-        parser_with_context!(item_tag)(context),
-        tag(" ::"),
-    )))(remaining)?;
     let (remaining, _ws) = item_tag_post_gap(context, remaining)?;
     let exit_matcher = plain_list_item_end(indent_level);
     let parser_context = context
@@ -175,14 +176,24 @@ pub fn plain_list_item<'r, 's>(
             exit_matcher: &exit_matcher,
         }));
 
-    let (remaining, (children, _exit_contents)) = many_till(
-        parser_with_context!(element(true))(&parser_context),
-        alt((
-            peek(recognize(tuple((start_of_line, many0(blank_line), eof)))),
-            parser_with_context!(exit_matcher_parser)(&parser_context),
-        )),
+    let (mut remaining, (mut children, _exit_contents)) = many_till(
+        include_input(parser_with_context!(element(true))(&parser_context)),
+        parser_with_context!(exit_matcher_parser)(&parser_context),
     )(remaining)?;
 
+    if !children.is_empty() && !context.should_consume_trailing_whitespace() {
+        let final_item_context =
+            parser_context.with_additional_node(ContextElement::ConsumeTrailingWhitespace(false));
+        let (final_child_start, _original_final_child) = children
+            .pop()
+            .expect("if-statement already checked that children was non-empty.");
+        let (remain, reparsed_final_element) = include_input(parser_with_context!(element(true))(
+            &final_item_context,
+        ))(final_child_start)?;
+        remaining = remain;
+        children.push(reparsed_final_element);
+    }
+
     let (remaining, _trailing_ws) =
         maybe_consume_trailing_whitespace_if_not_exiting(context, remaining)?;
 
@@ -196,11 +207,23 @@ pub fn plain_list_item<'r, 's>(
             tag: maybe_tag
                 .map(|(_ws, item_tag, _divider)| item_tag)
                 .unwrap_or(Vec::new()),
-            children,
+            children: children.into_iter().map(|(_start, item)| item).collect(),
         },
     ));
 }
 
+fn include_input<'s, F, O>(
+    mut inner: F,
+) -> impl FnMut(OrgSource<'s>) -> Res<OrgSource<'s>, (OrgSource<'s>, O)>
+where
+    F: FnMut(OrgSource<'s>) -> Res<OrgSource<'s>, O>,
+{
+    move |input: OrgSource<'_>| {
+        let (remaining, output) = inner(input)?;
+        Ok((remaining, (input, output)))
+    }
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn bullet<'s>(i: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     alt((
@@ -496,24 +519,21 @@ dolar"#,
             r#"+
 "#,
         );
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 
     #[test]
     fn detect_eof() {
         let input = OrgSource::new(r#"+"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 
     #[test]
     fn detect_no_gap() {
         let input = OrgSource::new(r#"+foo"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         // Since there is no whitespace after the '+' this is a paragraph, not a plain list.
         assert!(result.is_err());
     }
@@ -521,8 +541,7 @@ dolar"#,
     #[test]
     fn detect_with_gap() {
         let input = OrgSource::new(r#"+ foo"#);
-        let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let result = detect_plain_list(&initial_context, input);
+        let result = detect_plain_list(input);
         assert!(result.is_ok());
     }
 }

src/parser/radio_link.rs

@@ -6,6 +6,7 @@ use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use super::Object;
 use crate::error::CustomError;
@@ -104,7 +105,8 @@ pub fn radio_target<'r, 's>(
     )(remaining)?;
 
     let (remaining, _closing) = tag(">>>")(remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/regular_link.rs

@@ -3,13 +3,13 @@ use nom::bytes::complete::escaped;
 use nom::bytes::complete::tag;
 use nom::bytes::complete::take_till1;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::org_source::OrgSource;
 use super::parser_with_context::parser_with_context;
 use super::util::get_consumed;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use super::Object;
 use super::RegularLink;
@@ -39,7 +39,8 @@ pub fn regular_link_without_description<'r, 's>(
     let (remaining, _opening_bracket) = tag("[[")(input)?;
     let (remaining, _path) = pathreg(context, remaining)?;
     let (remaining, _closing_bracket) = tag("]]")(remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,
@@ -59,7 +60,8 @@ pub fn regular_link_with_description<'r, 's>(
     let (remaining, _closing_bracket) = tag("][")(remaining)?;
     let (remaining, _description) = description(context, remaining)?;
     let (remaining, _closing_bracket) = tag("]]")(remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/statistics_cookie.rs

@@ -1,10 +1,10 @@
 use nom::branch::alt;
 use nom::bytes::complete::tag;
-use nom::character::complete::space0;
 use nom::combinator::recognize;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::Res;
 use crate::parser::parser_with_context::parser_with_context;
@@ -23,12 +23,13 @@ pub fn statistics_cookie<'r, 's>(
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn percent_statistics_cookie<'r, 's>(
-    _context: Context<'r, 's>,
+    context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, StatisticsCookie<'s>> {
     let (remaining, source) =
         recognize(tuple((tag("["), nom::character::complete::u64, tag("%]"))))(input)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     Ok((
         remaining,
         StatisticsCookie {
@@ -39,7 +40,7 @@ pub fn percent_statistics_cookie<'r, 's>(
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn fraction_statistics_cookie<'r, 's>(
-    _context: Context<'r, 's>,
+    context: Context<'r, 's>,
     input: OrgSource<'s>,
 ) -> Res<OrgSource<'s>, StatisticsCookie<'s>> {
     let (remaining, source) = recognize(tuple((
@@ -49,7 +50,8 @@ pub fn fraction_statistics_cookie<'r, 's>(
         nom::character::complete::u64,
         tag("]"),
     )))(input)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     Ok((
         remaining,
         StatisticsCookie {

src/parser/subscript_and_superscript.rs

@@ -2,7 +2,6 @@ use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::map;
 use nom::combinator::not;
 use nom::combinator::opt;
@@ -13,6 +12,7 @@ use nom::multi::many_till;
 
 use super::org_source::BracketDepth;
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use super::Object;
 use crate::error::CustomError;
@@ -37,7 +37,8 @@ pub fn subscript<'r, 's>(
     let (remaining, _) = tag("_")(input)?;
     pre(context, input)?;
     let (remaining, _body) = script_body(context, remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,
@@ -56,7 +57,8 @@ pub fn superscript<'r, 's>(
     let (remaining, _) = tag("^")(input)?;
     pre(context, input)?;
     let (remaining, _body) = script_body(context, remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
     Ok((
         remaining,

src/parser/target.rs

@@ -1,13 +1,13 @@
 use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::combinator::verify;
 use nom::multi::many_till;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -49,7 +49,8 @@ pub fn target<'r, 's>(
         ))));
     }
     let (remaining, _) = tag(">>")(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((

src/parser/text_markup.rs

@@ -17,6 +17,7 @@ use tracing::span;
 
 use super::org_source::OrgSource;
 use super::radio_link::RematchObject;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::MyError;
@@ -204,7 +205,8 @@ fn _text_markup_object<'r, 's, 'x>(
     }
 
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     Ok((remaining, children))
 }
 
@@ -254,7 +256,8 @@ fn _text_markup_string<'r, 's, 'x>(
     }
 
     let (remaining, _close) = text_markup_end_specialized(context, remaining)?;
-    let (remaining, _trailing_whitespace) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     Ok((remaining, contents))
 }
 
@@ -277,7 +280,7 @@ pub fn pre<'r, 's>(_context: Context<'r, 's>, input: OrgSource<'s>) -> Res<OrgSo
 
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn post<'r, 's>(_context: Context<'r, 's>, input: OrgSource<'s>) -> Res<OrgSource<'s>, ()> {
-    let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\">")), line_ending))(input)?;
+    let (remaining, _) = alt((recognize(one_of(" \r\n\t-.,;:!?')}[\"")), line_ending))(input)?;
     Ok((remaining, ()))
 }
 

src/parser/timestamp.rs

@@ -3,7 +3,6 @@ use nom::bytes::complete::tag;
 use nom::character::complete::anychar;
 use nom::character::complete::digit1;
 use nom::character::complete::one_of;
-use nom::character::complete::space0;
 use nom::character::complete::space1;
 use nom::combinator::opt;
 use nom::combinator::recognize;
@@ -12,6 +11,7 @@ use nom::multi::many_till;
 use nom::sequence::tuple;
 
 use super::org_source::OrgSource;
+use super::util::maybe_consume_object_trailing_whitespace_if_not_exiting;
 use super::Context;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
@@ -49,7 +49,8 @@ fn diary_timestamp<'r, 's>(
     let (remaining, _) = tag("<%%(")(input)?;
     let (remaining, _body) = sexp(context, remaining)?;
     let (remaining, _) = tag(")>")(remaining)?;
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -112,7 +113,8 @@ fn active_timestamp<'r, 's>(
     )))(remaining)?;
     let (remaining, _) = tag(">")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -145,7 +147,8 @@ fn inactive_timestamp<'r, 's>(
     )))(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -166,7 +169,8 @@ fn active_date_range_timestamp<'r, 's>(
     let (remaining, _separator) = tag("--")(remaining)?;
     let (remaining, _second_timestamp) = active_timestamp(context, remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -206,7 +210,8 @@ fn active_time_range_timestamp<'r, 's>(
     )))(remaining)?;
     let (remaining, _) = tag(">")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -227,7 +232,8 @@ fn inactive_date_range_timestamp<'r, 's>(
     let (remaining, _separator) = tag("--")(remaining)?;
     let (remaining, _second_timestamp) = inactive_timestamp(context, remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((
@@ -267,7 +273,8 @@ fn inactive_time_range_timestamp<'r, 's>(
     )))(remaining)?;
     let (remaining, _) = tag("]")(remaining)?;
 
-    let (remaining, _) = space0(remaining)?;
+    let (remaining, _trailing_whitespace) =
+        maybe_consume_object_trailing_whitespace_if_not_exiting(context, remaining)?;
     let source = get_consumed(input, remaining);
 
     Ok((

src/parser/util.rs

@@ -68,6 +68,18 @@ pub fn element_trailing_whitespace<'s>(input: OrgSource<'s>) -> Res<OrgSource<'s
     alt((eof, recognize(many0(blank_line))))(input)
 }
 
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn maybe_consume_object_trailing_whitespace_if_not_exiting<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Option<OrgSource<'s>>> {
+    if exit_matcher_parser(context, input).is_err() {
+        opt(space0)(input)
+    } else {
+        Ok((input, None))
+    }
+}
+
 #[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn maybe_consume_trailing_whitespace_if_not_exiting<'r, 's>(
     context: Context<'r, 's>,

tests/test_template

@@ -3,7 +3,7 @@ fn {name}() {{
     let todo_org_path = "{path}";
     let org_contents = std::fs::read_to_string(todo_org_path).expect("Read org file.");
     println!("{{}}", org_contents);
-    let org_sexp = emacs_parse_org_document(org_contents.as_str()).expect("Use emacs to parse org file.");
+    let org_sexp = emacs_parse_anonymous_org_document(org_contents.as_str()).expect("Use emacs to parse org file.");
     println!("{{}}", org_sexp);
     let (_remaining, parsed_sexp) = sexp_with_padding(org_sexp.as_str()).expect("Sexp Parse failure");
     let (remaining, rust_parsed) = document(org_contents.as_str()).expect("Org Parse failure");