Use tekton matrix for specifying all combinations of features for the build CI job.

This is an optimization. When you have something like plain text which ends when it hits the next element, we only need to parse enough to detect that an element is about to occur. For elements like plain lists, this is as simple as parsing a line starting with optional whitespace and then a bullet, which avoids parsing the entire plain list tree. The benefit is most noticeable in deeply nested plain lists.

.dockerignore

@@ -1,3 +1,4 @@
 **/.git
 target
 Cargo.lock
+notes/

.lighthouse/pipeline-rust-build.yaml

@@ -0,0 +1,208 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: rust-build
+spec:
+  pipelineSpec:
+    params:
+      - name: image-name
+        description: The name for the built image
+        type: string
+      - name: path-to-image-context
+        description: The path to the build context
+        type: string
+      - name: path-to-dockerfile
+        description: The path to the Dockerfile
+        type: string
+    tasks:
+      - name: report-pending
+        taskRef:
+          name: gitea-set-status
+        runAfter:
+          - fetch-repository
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          name: git-clone
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+      - name: build-image
+        taskRef:
+          name: kaniko
+        params:
+          - name: IMAGE
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+          - name: CONTEXT
+            value: $(params.path-to-image-context)
+          - name: DOCKERFILE
+            value: $(params.path-to-dockerfile)
+          - name: BUILDER_IMAGE
+            value: "gcr.io/kaniko-project/executor:v1.12.1"
+          - name: EXTRA_ARGS
+            value:
+              - --cache=true
+              - --cache-copy-layers
+              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - --use-new-run # Should result in a speed-up
+              - --reproducible # To remove timestamps so layer caching works.
+              - --snapshot-mode=redo
+              - --skip-unused-stages=true
+              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: dockerconfig
+            workspace: docker-credentials
+        runAfter:
+          - fetch-repository
+      - name: build-organic
+        taskRef:
+          name: run-docker-image
+        matrix:
+          params:
+            - name: feature-compare
+              value:
+                - "true"
+                - "false"
+            - name: feature-tracing
+              value:
+                - "true"
+                - "false"
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        runAfter:
+          - build-image
+        params:
+          - name: command
+            value: ["/bin/sh", "-c"]
+          - name: args
+            value:
+              - |
+                set -euo pipefail
+                IFS=$$'\n\t'
+                features=()
+                if [ $(params.feature-compare) = "true" ]; then features+=(compare); fi
+                if [ $(params.feature-tracing) = "true" ]; then features+=(tracing); fi
+                if [ $${#features[@]} -eq 0 ]; then
+                    exec cargo build --no-default-features
+                else
+                    featurelist=$$(IFS="," ; echo "$${features[*]}")
+                    exec cargo build --no-default-features --features "$$featurelist"
+                fi
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: cargo-cache-autoclean
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: command
+            value: [cargo, cache, --autoclean]
+          - name: args
+            value: []
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    workspaces:
+      - name: git-source
+      - name: docker-credentials
+      - name: cargo-cache
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: rust-source
+    - name: cargo-cache
+      persistentVolumeClaim:
+        claimName: organic-cargo-cache-build
+    - name: docker-credentials
+      secret:
+        secretName: harbor-plain
+  serviceAccountName: build-bot
+  timeout: 240h0m0s
+  params:
+    - name: image-name
+      value: "harbor.fizz.buzz/private/organic-build"
+    - name: path-to-image-context
+      value: docker/organic_build/
+    - name: path-to-dockerfile
+      value: docker/organic_build/Dockerfile

.lighthouse/pipeline-rust-test.yaml

@@ -4,6 +4,10 @@ metadata:
   name: rust-test
 spec:
   pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
     params:
       - name: image-name
         description: The name for the built image
@@ -201,7 +205,6 @@ spec:
       secret:
         secretName: harbor-plain
   serviceAccountName: build-bot
-  timeout: 240h0m0s
   params:
     - name: image-name
       value: "harbor.fizz.buzz/private/organic-test"

.lighthouse/pipeline-rustfmt.yaml

@@ -81,14 +81,6 @@ spec:
             value: $(params.PULL_BASE_SHA)
           - name: deleteExisting
             value: "true"
-      - name: detect-tag
-        taskRef:
-          name: detect-tag
-        workspaces:
-          - name: repo
-            workspace: git-source
-        runAfter:
-          - fetch-repository
       - name: build-image
         taskRef:
           name: kaniko
@@ -118,7 +110,6 @@ spec:
             workspace: docker-credentials
         runAfter:
           - fetch-repository
-          - detect-tag
       - name: rustfmt
         taskRef:
           name: run-docker-image

.lighthouse/triggers.yaml

@@ -16,3 +16,10 @@ spec:
       skip_branches:
         # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
         - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+    - name: rust-build
+      source: "pipeline-rust-build.yaml"
+      # Override https-based url from lighthouse events.
+      clone_uri: "git@code.fizz.buzz:talexander/organic.git"
+      skip_branches:
+        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
+        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"

Cargo.toml

@@ -1,30 +1,40 @@
 [package]
 name = "organic"
-version = "0.1.0"
+version = "0.1.2"
 authors = ["Tom Alexander <tom@fizz.buzz>"]
 description = "An org-mode parser."
 edition = "2021"
 license = "0BSD"
 repository = "https://code.fizz.buzz/talexander/organic"
-readme = "README.org"
+readme = "README.md"
 keywords = ["emacs", "org-mode"]
-categories = ["template-engine"]
+categories = ["parsing"]
+resolver = "2"
+include = [
+  "LICENSE",
+  "**/*.rs",
+  "Cargo.toml",
+  "tests/*"
+]
 
 [lib]
 name = "organic"
 path = "src/lib.rs"
 
 [[bin]]
-name = "toy"
+# This bin exists for development purposes only. The real target of this crate is the library.
+name = "compare"
 path = "src/main.rs"
 
 [dependencies]
 nom = "7.1.1"
-opentelemetry = "0.17.0"
-opentelemetry-jaeger = "0.16.0"
-tracing = "0.1.37"
-tracing-opentelemetry = "0.17.2"
-tracing-subscriber = {version="0.3.16", features=["env-filter"]}
+opentelemetry = { version = "0.20.0", optional = true, default-features = false, features = ["trace", "rt-tokio"] }
+opentelemetry-otlp = { version = "0.13.0", optional = true }
+opentelemetry-semantic-conventions = { version = "0.12.0", optional = true }
+tokio = { version = "1.30.0", optional = true, default-features = false, features = ["rt", "rt-multi-thread"] }
+tracing = { version = "0.1.37", optional = true }
+tracing-opentelemetry = { version = "0.20.0", optional = true }
+tracing-subscriber = { version = "0.3.17", optional = true, features = ["env-filter"] }
 
 [build-dependencies]
 walkdir = "2.3.3"
@@ -32,7 +42,14 @@ walkdir = "2.3.3"
 [features]
 default = ["compare"]
 compare = []
+tracing = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:tokio", "dep:tracing", "dep:tracing-opentelemetry", "dep:tracing-subscriber"]
 
-[profile.release]
+[profile.release-lto]
+inherits = "release"
 lto = true
 strip = "symbols"
+
+[profile.perf]
+inherits = "release"
+lto = true
+debug = true

Makefile

@@ -35,7 +35,17 @@ clean:
 
 .PHONY: test
 test:
-> cargo test --lib --test test_loader -- --test-threads $(TESTJOBS)
+> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+
+.PHONY: dockertest
+dockertest:
+> $(MAKE) -C docker/organic_test
+> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+
+.PHONY: dockerclean
+dockerclean:
+# Delete volumes created for running the tests in docker. This does not touch anything related to the jaeger docker container.
+> docker volume rm cargo-cache rust-cache
 
 .PHONY: integrationtest
 integrationtest:
@@ -45,17 +55,12 @@ integrationtest:
 unittest:
 > cargo test --lib -- --test-threads $(TESTJOBS)
 
-.PHONY: run
-run:
-> cargo run
-
-.PHONY: debug
-debug:
-> RUST_LOG=debug cargo run
-
 .PHONY: jaeger
 jaeger:
-> docker run -d --rm --name toylanguagedocker -p 6831:6831/udp -p 6832:6832/udp -p 16686:16686 -p 14268:14268 jaegertracing/all-in-one:latest
+# 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
+#
+# These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
+> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
 
 .PHONY: jaegerweb
 jaegerweb:
@@ -63,4 +68,4 @@ jaegerweb:
 
 .PHONY: jaegerstop
 jaegerstop:
-> docker stop toylanguagedocker
+> docker stop organicdocker

README.md

@@ -0,0 +1,13 @@
+# Organic - Free Range Org-Mode
+
+Organic is an emacs-less implementation of an [org-mode](https://orgmode.org/) parser.
+
+
+## Project Status
+
+This project is a personal learning project to grow my experience in [rust](https://www.rust-lang.org/). It is under development and at this time I would not recommend anyone use this code. The goal is to turn this into a project others can use, at which point more information will appear in this README.
+
+
+## License
+
+This project is released under the public-domain-equivalent [0BSD license](https://www.tldrlegal.com/license/bsd-0-clause-license). This license puts no restrictions on the use of this code (you do not even have to include the copyright notice or license text when using it). HOWEVER, this project has a couple permissively licensed dependencies which do require their copyright notices and/or license texts to be included. I am not a lawyer and this is not legal advice but it is my layperson's understanding that if you distribute a binary with this library linked in, you will need to abide by their terms since their code will also be linked in your binary. I try to keep the dependencies to a minimum and the most restrictive dependency I will ever include is a permissively licensed one.

README.org

@@ -1,10 +0,0 @@
-#+options: toc:nil
-#+title: Organic - Free Range Org-Mode
-
-Organic is an emacs-less implementation of an [[https://orgmode.org/][org-mode]] parser.
-
-* Project Status
-This project is a personal learning project to grow my experience in [[https://www.rust-lang.org/][rust]]. It is under development and at this time I would not recommend anyone use this code. The goal is to turn this into a project others can use, at which point more information will appear in this README.
-
-* License
-This project is released under the public-domain-equivalent [[https://www.tldrlegal.com/license/bsd-0-clause-license][0BSD license]]. This license puts no restrictions on the use of this code (you do not even have to include the copyright notice or license text when using it). HOWEVER, this project has a couple permissively licensed dependencies which do require their copyright notices and/or license texts to be included. I am not a lawyer and this is not legal advice but it is my layperson's understanding that if you distribute a binary with this library linked in, you will need to abide by their terms since their code will also be linked in your binary. I try to keep the dependencies to a minimum and the most restrictive dependency I will ever include is a permissively licensed one.

build.rs

@@ -41,6 +41,7 @@ fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
         .strip_suffix(".org")
         .expect("Should have .org extension")
         .replace("/", "_");
+    let test_name = format!("autogen_{}", test_name);
 
     if let Some(_reason) = is_expect_fail(test_name.as_str()) {
         write!(test_file, "#[ignore]\n").unwrap();
@@ -71,10 +72,9 @@ use organic::parser::sexp::sexp_with_padding;
 
 fn is_expect_fail(name: &str) -> Option<&str> {
     match name {
-        "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
-        "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
-        "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "export_snippet_paragraph_break_precedent" => Some("Emacs 28 has broken behavior so the tests in the CI fail."),
+        "autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
+        "autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
+        "autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
         _ => None,
     }
 }

docker/cargo_fmt/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../
+	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
 
 .PHONY: push
 push:

docker/organic_build/Dockerfile

@@ -0,0 +1,4 @@
+FROM rustlang/rust:nightly-alpine3.17
+
+RUN apk add --no-cache musl-dev
+RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache

docker/organic_build/Makefile

@@ -0,0 +1,35 @@
+IMAGE_NAME:=organic-build
+# REMOTE_REPO:=harbor.fizz.buzz/private
+
+.PHONY: all
+all: build push
+
+.PHONY: build
+build:
+	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+
+.PHONY: push
+push:
+ifdef REMOTE_REPO
+	docker tag $(IMAGE_NAME) $(REMOTE_REPO)/$(IMAGE_NAME)
+	docker push $(REMOTE_REPO)/$(IMAGE_NAME)
+else
+	@echo "REMOTE_REPO not defined, not pushing to a remote repo."
+endif
+
+.PHONY: clean
+clean:
+	docker rmi $(IMAGE_NAME)
+ifdef REMOTE_REPO
+	docker rmi $(REMOTE_REPO)/$(IMAGE_NAME)
+else
+	@echo "REMOTE_REPO not defined, not removing from remote repo."
+endif
+
+.PHONY: run
+run:
+	docker run --rm -i -t $(IMAGE_NAME)
+
+.PHONY: shell
+shell:
+	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)

docker/organic_test/Dockerfile

@@ -1,4 +1,32 @@
-FROM rustlang/rust:nightly-alpine3.17
+FROM alpine:3.17 AS build
+RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
 
-RUN apk add --no-cache musl-dev emacs
+
+FROM build AS build-emacs
+ARG EMACS_VERSION=emacs-29.1
+RUN git clone --depth 1 --branch $EMACS_VERSION https://git.savannah.gnu.org/git/emacs.git /root/emacs
+WORKDIR /root/emacs
+RUN mkdir /root/dist
+RUN ./autogen.sh
+RUN ./configure --prefix /usr --without-x --without-sound
+RUN make
+RUN make DESTDIR="/root/dist" install
+
+
+FROM build AS build-org-mode
+ARG ORG_VERSION=7bdec435ff5d86220d13c431e799c5ed44a57da1
+COPY --from=build-emacs /root/dist/ /
+RUN mkdir /root/dist
+# Savannah does not allow fetching specific revisions, so we're going to have to put unnecessary load on their server by cloning main and then checking out the revision we want.
+RUN git clone https://git.savannah.gnu.org/git/emacs/org-mode.git /root/org-mode && git -C /root/org-mode checkout $ORG_VERSION
+# RUN mkdir /root/org-mode && git -C /root/org-mode init --initial-branch=main && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin $ORG_VERSION && git -C /root/org-mode checkout FETCH_HEAD
+WORKDIR /root/org-mode
+RUN make compile
+RUN make DESTDIR="/root/dist" install
+
+
+FROM rustlang/rust:nightly-alpine3.17
+RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+COPY --from=build-emacs /root/dist/ /
+COPY --from=build-org-mode /root/dist/ /

docker/organic_test/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../
+	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
 
 .PHONY: push
 push:

notes/exit_matcher_loop_notes.txt

@@ -1,35 +0,0 @@
-Headings add exit matcher for heading
-
-Paragraphs add exit matcher for elements (but it should be sans paragraph)
-
-
-
-
-* foo
-* bar
-* baz
-
-context tree -> ()
-
-match * foo
-
-context tree -> exit(heading matcher)
-
-check exit
-invoke heading matcher
-check exit
-invoke heading matcher
-check exit
-invoke heading matcher
-adds second heading matcher exit
-
-
-Ways around this:
-- Always parse SOMETHING before checking for exit
-  - Doesn't always seem possible
-- Disable exit matchers during exit check
-  - Seems like it would break syntax
-- Have separate parsers for the beginning of the exit condition (for example, checking for just the headline instead of the full heading parser)
-  - Won't be possible with paragraphs ending at any other element
-- Check exit matchers in parent parser
-  - Will this work? seems like it would just create larger loops

notes/optimization_ideas.org

@@ -0,0 +1,27 @@
+* Analysis
+** Parse start per character
+It might help analysis to record how often we start a specific type of parse for each character. For example, at the start of a plain list, if we had a count of how often each character was the start of a parse of a list we could use that to see how often that list is getting re-parsed.
+* Optimizations
+** Edit whitespace for list items
+Whether or not a list item owns the trailing whitespace depends on if it is the last list item in that list. Since we do not know ahead of time if an item is the last item in the list, we have to either re-parse the list item or modify it after parsing.
+
+*** For
+We already are modifying the source of some elements after-the-fact with src_rust{set_source()} so this would be more of the same.
+*** Against
+I'd like to phase out such modifications because they seem hacky and fragile.
+** Make detect element function
+Some exit matchers are based on when the next element is found. Some elements do not need to be fully parsed to identify that they are a valid element. For example, src_org{1. foo} can already be identified as the start of a plain list (in the right context) without needing to parse the entire element.
+*** For
+Avoiding parsing the entire element for an exit matcher would reduce redundant parses.
+*** Against
+This adds code complexity and introduces the potential for bugs.
+
+How many elements can be reasonably early-detected? For example, src_org{#+begin_src foo} is not enough to detect the start of a source block because without the src_org{#+end_src} it is just plain text.
+** Grab multiple characters in plaintext parser before checking exit matcher
+Currently we check the exit matcher after each character inside the plain text parser (and many others). Are there character sequences we can assume no exit matcher will trigger between? For example, a contiguous string of latin-alphabet letters?
+*** For
+This could significantly reduce our calls to exit matchers.
+*** Against
+I think targets would break this.
+
+The exit matchers are already implicitly building this behavior since they should all exit very early when the starting character is wrong. Putting this logic in a centralized place, far away from where those characters are actually going to be used, is unfortunate for readability.

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_1.org

@@ -0,0 +1,9 @@
+1. foo
+
+   1. bar
+
+   2. baz
+
+2. lorem
+
+ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_2.org

@@ -0,0 +1,11 @@
+1. foo
+
+   1. bar
+
+   2. baz
+
+   cat
+
+2. lorem
+
+ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_3.org

@@ -0,0 +1,10 @@
+1. cat
+   1. foo
+
+      1. bar
+
+      2. baz
+
+   2. lorem
+
+ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_4.org

@@ -0,0 +1,12 @@
+1. cat
+   1. foo
+
+      1. bar
+
+      2. baz
+
+   2. lorem
+
+2. dog
+
+ipsum

org_mode_samples/trailing_whitespace/blank_lines_after_paragraphs.org

@@ -0,0 +1,18 @@
+foo bar.
+
+
+
+
+* Lorem
+baz
+
+
+
+
+* Ipsum
+alpha
+
+
+
+
+beta

scripts/callgrind.bash

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$DIR/../"
+
+RUSTFLAGS="-C opt-level=0" cargo build --no-default-features
+valgrind --tool=callgrind --callgrind-out-file=callgrind.out target/debug/compare
+
+echo "You probably want to run:"
+echo "callgrind_annotate --auto=yes callgrind.out"

scripts/perf.bash

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: ${PROFILE:="perf"}
+
+cd "$DIR/../"
+
+cargo build --profile "$PROFILE" --no-default-features
+perf record --freq=2000 --call-graph dwarf --output=perf.data target/${PROFILE}/compare
+# Convert to a format firefox will read
+# flags to consider --show-info
+perf script -F +pid --input perf.data > perf.firefox
+
+echo "You probably want to go to https://profiler.firefox.com/"
+echo "Either that or run hotspot"

scripts/run_docker_compare.bash

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+: ${SHELL:="NO"} # or YES to launch a shell instead of running the test
+: ${TRACE:="NO"} # or YES to send traces to jaeger
+: ${BACKTRACE:="NO"} # or YES to print a rust backtrace when panicking
+
+cd "$DIR/../"
+REALPATH=$(command -v uu-realpath || command -v realpath)
+MAKE=$(command -v gmake || command -v make)
+
+function main {
+    build_container
+    launch_container
+}
+
+function build_container {
+    $MAKE -C "$DIR/../docker/organic_test"
+}
+
+function launch_container {
+    local additional_flags=()
+    local additional_args=()
+
+    if [ "$SHELL" != "YES" ]; then
+        additional_args+=(cargo run)
+    else
+        additional_flags+=(-t)
+    fi
+
+    if [ "$TRACE" = "YES" ]; then
+        # We use the host network so it can talk to jaeger hosted at 127.0.0.1
+        additional_flags+=(--network=host --env RUST_LOG=debug)
+    fi
+
+    if [ "$BACKTRACE" = "YES" ]; then
+        additional_flags+=(--env RUST_BACKTRACE=full)
+    fi
+
+    docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test "${additional_args[@]}"
+}
+
+main "${@}"

scripts/run_docker_integration_test.bash

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+cd "$DIR/../"
+REALPATH=$(command -v uu-realpath || command -v realpath)
+MAKE=$(command -v gmake || command -v make)
+
+function main {
+    local test_names=$(get_test_names "${@}")
+    build_container
+
+    local test
+    while read test; do
+        launch_container "$test"
+    done<<<"$test_names"
+}
+
+function build_container {
+    $MAKE -C "$DIR/../docker/organic_test"
+}
+
+function get_test_names {
+    local test_file
+    local samples_dir=$($REALPATH "$DIR/../org_mode_samples")
+    for test_file in "$@"
+    do
+        if [ -e "$test_file" ]; then
+            local test_file_full_path=$($REALPATH "$test_file")
+            local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
+            local without_extension="${relative_to_samples%.org}"
+            echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
+        else
+            echo "$test_file" | tr '[:upper:]' '[:lower:]'
+        fi
+    done
+}
+
+function launch_container {
+    local test="$1"
+    local additional_args=()
+
+    local init_script=$(cat <<EOF
+set -euo pipefail
+IFS=\$'\n\t'
+
+cargo test --no-fail-fast --lib --test test_loader "$test" -- --show-output
+EOF
+          )
+
+    docker run --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test sh -c "$init_script"
+}
+
+
+main "${@}"

scripts/run_integration_test.bash

@@ -4,17 +4,27 @@ set -euo pipefail
 IFS=$'\n\t'
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+cd "$DIR/../"
 REALPATH=$(command -v uu-realpath || command -v realpath)
 
-samples_dir=$(readlink -f "$DIR/../org_mode_samples")
+function main {
+    local test_names=$(get_test_names "${@}")
+
+    local test
+    while read test; do
+        cargo test --no-fail-fast --test test_loader "$test" -- --show-output
+    done<<<"$test_names"
+}
 
 function get_test_names {
+    local test_file
+    local samples_dir=$($REALPATH "$DIR/../org_mode_samples")
     for test_file in "$@"
     do
         if [ -e "$test_file" ]; then
-            test_file_full_path=$(readlink -f "$test_file")
-            relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
-            without_extension="${relative_to_samples%.org}"
+            local test_file_full_path=$($REALPATH "$test_file")
+            local relative_to_samples=$($REALPATH --relative-to "$samples_dir" "$test_file_full_path")
+            local without_extension="${relative_to_samples%.org}"
             echo "${without_extension/\//_}" | tr '[:upper:]' '[:lower:]'
         else
             echo "$test_file" | tr '[:upper:]' '[:lower:]'
@@ -22,6 +32,4 @@ function get_test_names {
     done
 }
 
-get_test_names "$@" | while read test; do
-    (cd "$DIR/../" && cargo test --no-fail-fast --test test_loader "$test" -- --show-output)
-done
+main "${@}"

src/compare/mod.rs

@@ -3,3 +3,5 @@ mod parse;
 mod util;
 pub use diff::compare_document;
 pub use parse::emacs_parse_org_document;
+pub use parse::get_emacs_version;
+pub use parse::get_org_mode_version;

src/compare/parse.rs

@@ -1,22 +1,25 @@
-use std::path::Path;
 use std::process::Command;
 
-pub fn emacs_parse_org_document<'a, C>(file_path: C) -> Result<String, Box<dyn std::error::Error>>
+pub fn emacs_parse_org_document<C>(file_contents: C) -> Result<String, Box<dyn std::error::Error>>
 where
-    C: AsRef<Path>,
+    C: AsRef<str>,
 {
-    let elisp_script = r#"(progn
+    let escaped_file_contents = escape_elisp_string(file_contents);
+    let elisp_script = format!(
+        r#"(progn
+     (erase-buffer)
+     (insert "{escaped_file_contents}")
      (org-mode)
      (message "%s" (pp-to-string (org-element-parse-buffer)))
-)"#;
+)"#,
+        escaped_file_contents = escaped_file_contents
+    );
     let mut cmd = Command::new("emacs");
     let proc = cmd
         .arg("-q")
         .arg("--no-site-file")
         .arg("--no-splash")
         .arg("--batch")
-        .arg("--insert")
-        .arg(file_path.as_ref().as_os_str())
         .arg("--eval")
         .arg(elisp_script);
     let out = proc.output()?;
@@ -24,3 +27,62 @@ where
     let org_sexp = out.stderr;
     Ok(String::from_utf8(org_sexp)?)
 }
+
+fn escape_elisp_string<C>(file_contents: C) -> String
+where
+    C: AsRef<str>,
+{
+    let source = file_contents.as_ref();
+    let source_len = source.len();
+    // We allocate a string 10% larger than the source to account for escape characters. Without this, we would have more allocations during processing.
+    let mut output = String::with_capacity(source_len + (source_len / 10));
+    for c in source.chars() {
+        match c {
+            '"' | '\\' => {
+                output.push('\\');
+                output.push(c);
+            }
+            _ => {
+                output.push(c);
+            }
+        }
+    }
+    output
+}
+
+pub fn get_emacs_version() -> Result<String, Box<dyn std::error::Error>> {
+    let elisp_script = r#"(progn
+     (message "%s" (version))
+)"#;
+    let mut cmd = Command::new("emacs");
+    let proc = cmd
+        .arg("-q")
+        .arg("--no-site-file")
+        .arg("--no-splash")
+        .arg("--batch")
+        .arg("--eval")
+        .arg(elisp_script);
+
+    let out = proc.output()?;
+    out.status.exit_ok()?;
+    Ok(String::from_utf8(out.stderr)?)
+}
+
+pub fn get_org_mode_version() -> Result<String, Box<dyn std::error::Error>> {
+    let elisp_script = r#"(progn
+     (org-mode)
+     (message "%s" (org-version nil t nil))
+)"#;
+    let mut cmd = Command::new("emacs");
+    let proc = cmd
+        .arg("-q")
+        .arg("--no-site-file")
+        .arg("--no-splash")
+        .arg("--batch")
+        .arg("--eval")
+        .arg(elisp_script);
+
+    let out = proc.output()?;
+    out.status.exit_ok()?;
+    Ok(String::from_utf8(out.stderr)?)
+}

src/compare/util.rs

@@ -42,24 +42,100 @@ pub fn assert_bounds<'s, S: Source<'s>>(
     emacs: &'s Token<'s>,
     rust: &'s S,
 ) -> Result<(), Box<dyn std::error::Error>> {
+    let standard_properties = get_standard_properties(emacs)?;
+    let (begin, end) = (
+        standard_properties
+            .begin
+            .ok_or("Token should have a begin.")?,
+        standard_properties
+            .end
+            .ok_or("Token should have a begin.")?,
+    );
+    let (rust_begin, rust_end) = get_offsets(source, rust);
+    if (rust_begin + 1) != begin || (rust_end + 1) != end {
+        Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;
+    }
+
+    Ok(())
+}
+
+struct StandardProperties {
+    begin: Option<usize>,
+    #[allow(dead_code)]
+    post_affiliated: Option<usize>,
+    #[allow(dead_code)]
+    contents_begin: Option<usize>,
+    #[allow(dead_code)]
+    contents_end: Option<usize>,
+    end: Option<usize>,
+    #[allow(dead_code)]
+    post_blank: Option<usize>,
+}
+
+fn get_standard_properties<'s>(
+    emacs: &'s Token<'s>,
+) -> Result<StandardProperties, Box<dyn std::error::Error>> {
     let children = emacs.as_list()?;
     let attributes_child = children
         .iter()
         .nth(1)
         .ok_or("Should have an attributes child.")?;
     let attributes_map = attributes_child.as_map()?;
-    let begin = attributes_map
-        .get(":begin")
-        .ok_or("Missing :begin attribute.")?
-        .as_atom()?;
-    let end = attributes_map
-        .get(":end")
-        .ok_or("Missing :end attribute.")?
-        .as_atom()?;
-    let (rust_begin, rust_end) = get_offsets(source, rust);
-    if (rust_begin + 1).to_string() != begin || (rust_end + 1).to_string() != end {
-        Err(format!("Rust bounds ({rust_begin}, {rust_end}) do not match emacs bounds ({emacs_begin}, {emacs_end})", rust_begin = rust_begin + 1, rust_end = rust_end + 1, emacs_begin=begin, emacs_end=end))?;
+    let standard_properties = attributes_map.get(":standard-properties");
+    Ok(if standard_properties.is_some() {
+        let mut std_props = standard_properties
+            .expect("if statement proves its Some")
+            .as_vector()?
+            .into_iter();
+        let begin = maybe_token_to_usize(std_props.next())?;
+        let post_affiliated = maybe_token_to_usize(std_props.next())?;
+        let contents_begin = maybe_token_to_usize(std_props.next())?;
+        let contents_end = maybe_token_to_usize(std_props.next())?;
+        let end = maybe_token_to_usize(std_props.next())?;
+        let post_blank = maybe_token_to_usize(std_props.next())?;
+        StandardProperties {
+            begin,
+            post_affiliated,
+            contents_begin,
+            contents_end,
+            end,
+            post_blank,
+        }
+    } else {
+        let begin = maybe_token_to_usize(attributes_map.get(":begin").map(|token| *token))?;
+        let end = maybe_token_to_usize(attributes_map.get(":end").map(|token| *token))?;
+        let contents_begin =
+            maybe_token_to_usize(attributes_map.get(":contents-begin").map(|token| *token))?;
+        let contents_end =
+            maybe_token_to_usize(attributes_map.get(":contents-end").map(|token| *token))?;
+        let post_blank =
+            maybe_token_to_usize(attributes_map.get(":post-blank").map(|token| *token))?;
+        let post_affiliated =
+            maybe_token_to_usize(attributes_map.get(":post-affiliated").map(|token| *token))?;
+        StandardProperties {
+            begin,
+            post_affiliated,
+            contents_begin,
+            contents_end,
+            end,
+            post_blank,
+        }
+    })
 }
 
-    Ok(())
+fn maybe_token_to_usize(
+    token: Option<&Token<'_>>,
+) -> Result<Option<usize>, Box<dyn std::error::Error>> {
+    Ok(token
+        .map(|token| token.as_atom())
+        .map_or(Ok(None), |r| r.map(Some))?
+        .map(|val| {
+            if val == "nil" {
+                None
+            } else {
+                Some(val.parse::<usize>())
+            }
+        })
+        .flatten() // Outer option is whether or not the param exists, inner option is whether or not it is nil
+        .map_or(Ok(None), |r| r.map(Some))?)
 }

src/error/error.rs

@@ -4,6 +4,7 @@ use nom::IResult;
 
 pub type Res<T, U> = IResult<T, U, CustomError<T>>;
 
+// TODO: MyError probably shouldn't be based on the same type as the input type since it's used exclusively with static strings right now.
 #[derive(Debug, PartialEq)]
 pub enum CustomError<I> {
     MyError(MyError<I>),

src/init_tracing.rs

@@ -1,34 +1,71 @@
-use tracing_subscriber::layer::SubscriberExt;
+#[cfg(feature = "tracing")]
+use opentelemetry_otlp::WithExportConfig;
+#[cfg(feature = "tracing")]
+use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt;
+#[cfg(feature = "tracing")]
 use tracing_subscriber::util::SubscriberInitExt;
-// use tracing_subscriber::EnvFilter;
 
+const SERVICE_NAME: &'static str = "organic";
+
+// Despite the obvious verbosity that fully-qualifying everything causes, in these functions I am fully-qualifying everything relating to tracing. This is because the tracing feature involves multiple libraries working together and so I think it is beneficial to see which libraries contribute which bits.
+
+#[cfg(feature = "tracing")]
 pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
-    // let env_filter = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new("warn"));
+    // by default it will hit http://localhost:4317 with a gRPC payload
+    // TODO: I think the endpoint can be controlled by the OTEL_EXPORTER_OTLP_TRACES_ENDPOINT env variable instead of hard-coded into this code base. Regardless, I am the only developer right now so I am not too concerned.
+    let exporter = opentelemetry_otlp::new_exporter()
+        .tonic()
+        // Using "localhost" is broken inside the docker container when tracing
+        .with_endpoint("http://127.0.0.1:4317/v1/traces");
 
-    // let stdout = tracing_subscriber::fmt::Layer::new()
-    //     .pretty()
-    //     .with_file(true)
-    //     .with_line_number(true)
-    //     .with_thread_ids(false)
-    //     .with_target(false);
+    let tracer = opentelemetry_otlp::new_pipeline()
+        .tracing()
+        .with_exporter(exporter)
+        .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
+            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
+                opentelemetry_semantic_conventions::resource::SERVICE_NAME,
+                SERVICE_NAME.to_string(),
+            )]),
+        ))
+        // If I do install_batch then 1K+ spans will get orphaned off into their own trace and I get the error message "OpenTelemetry trace error occurred. cannot send message to batch processor as the channel is closed"
+        //
+        // If I do install_simple then it only creates 1 trace (which is good!) but my console gets spammed with this concerning log message that makes me think it might be dropping the extra spans on the floor: "OpenTelemetry trace error occurred. Exporter otlp encountered the following error(s): the grpc server returns error (Unknown error): , detailed error message: Service was not ready: transport error"
+        //
+        // I suspect it is related to this bug: https://github.com/open-telemetry/opentelemetry-rust/issues/888
+        //
+        // .install_simple()
+        .install_batch(opentelemetry::runtime::Tokio)
+        .expect("Error: Failed to initialize the tracer.");
 
-    opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
-    let tracer = opentelemetry_jaeger::new_pipeline()
-        .with_service_name("toy_language")
-        .install_simple()?;
+    let subscriber = tracing_subscriber::Registry::default();
+    let level_filter_layer = tracing_subscriber::EnvFilter::try_from_default_env()
+        .unwrap_or(tracing_subscriber::EnvFilter::new("WARN"));
+    let tracing_layer = tracing_opentelemetry::layer().with_tracer(tracer);
 
-    let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+    opentelemetry::global::set_text_map_propagator(
+        opentelemetry::sdk::propagation::TraceContextPropagator::new(),
+    );
 
-    tracing_subscriber::registry()
-        // .with(env_filter)
-        .with(opentelemetry)
-        // .with(stdout)
+    subscriber
+        .with(level_filter_layer)
+        .with(tracing_layer)
         .try_init()?;
 
     Ok(())
 }
 
+#[cfg(feature = "tracing")]
 pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
     opentelemetry::global::shutdown_tracer_provider();
     Ok(())
 }
+
+#[cfg(not(feature = "tracing"))]
+pub fn init_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(())
+}
+
+#[cfg(not(feature = "tracing"))]
+pub fn shutdown_telemetry() -> Result<(), Box<dyn std::error::Error>> {
+    Ok(())
+}

src/lib.rs

@@ -7,6 +7,10 @@ mod compare;
 pub use compare::compare_document;
 #[cfg(feature = "compare")]
 pub use compare::emacs_parse_org_document;
+#[cfg(feature = "compare")]
+pub use compare::get_emacs_version;
+#[cfg(feature = "compare")]
+pub use compare::get_org_mode_version;
 
 mod error;
 pub mod parser;

src/main.rs

@@ -1,17 +1,91 @@
 #![feature(round_char_boundary)]
-use ::organic::parser::document;
+use std::io::Read;
 
+use ::organic::parser::document;
+#[cfg(feature = "compare")]
+use organic::compare_document;
+#[cfg(feature = "compare")]
+use organic::emacs_parse_org_document;
+#[cfg(feature = "compare")]
+use organic::get_emacs_version;
+#[cfg(feature = "compare")]
+use organic::get_org_mode_version;
+#[cfg(feature = "compare")]
+use organic::parser::sexp::sexp_with_padding;
+
+#[cfg(feature = "tracing")]
 use crate::init_tracing::init_telemetry;
+#[cfg(feature = "tracing")]
 use crate::init_tracing::shutdown_telemetry;
+#[cfg(feature = "tracing")]
 mod init_tracing;
 
-const TEST_DOC: &'static str = include_str!("../toy_language.txt");
-
+#[cfg(not(feature = "tracing"))]
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    main_body()
+}
+
+#[cfg(feature = "tracing")]
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let rt = tokio::runtime::Runtime::new()?;
+    let result = rt.block_on(async {
         init_telemetry()?;
-    let parsed = document(TEST_DOC);
-    println!("{}\n\n\n", TEST_DOC);
-    println!("{:#?}", parsed);
+        let main_body_result = main_body();
         shutdown_telemetry()?;
+        main_body_result
+    });
+    result
+}
+
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn main_body() -> Result<(), Box<dyn std::error::Error>> {
+    let org_contents = read_stdin_to_string()?;
+    run_compare(org_contents)
+}
+
+fn read_stdin_to_string() -> Result<String, Box<dyn std::error::Error>> {
+    let mut stdin_contents = String::new();
+    std::io::stdin()
+        .lock()
+        .read_to_string(&mut stdin_contents)?;
+    Ok(stdin_contents)
+}
+
+#[cfg(feature = "compare")]
+fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
+    let emacs_version = get_emacs_version()?;
+    let org_mode_version = get_org_mode_version()?;
+    eprintln!("Using emacs version: {}", emacs_version.trim());
+    eprintln!("Using org-mode version: {}", org_mode_version.trim());
+    let (remaining, rust_parsed) = document(org_contents.as_ref()).map_err(|e| e.to_string())?;
+    let org_sexp = emacs_parse_org_document(org_contents.as_ref())?;
+    let (_remaining, parsed_sexp) =
+        sexp_with_padding(org_sexp.as_str()).map_err(|e| e.to_string())?;
+
+    println!("{}\n\n\n", org_contents.as_ref());
+    println!("{}", org_sexp);
+    println!("{:#?}", rust_parsed);
+
+    // We do the diffing after printing out both parsed forms in case the diffing panics
+    let diff_result = compare_document(&parsed_sexp, &rust_parsed)?;
+    diff_result.print()?;
+
+    if diff_result.is_bad() {
+        Err("Diff results do not match.")?;
+    }
+    if remaining != "" {
+        Err(format!("There was unparsed text remaining: {}", remaining))?;
+    }
+
+    Ok(())
+}
+
+#[cfg(not(feature = "compare"))]
+fn run_compare<P: AsRef<str>>(org_contents: P) -> Result<(), Box<dyn std::error::Error>> {
+    eprintln!(
+        "This program was built with compare disabled. Only parsing with organic, not comparing."
+    );
+    let (remaining, rust_parsed) = document(org_contents.as_ref()).map_err(|e| e.to_string())?;
+    println!("{:#?}", rust_parsed);
     Ok(())
 }

src/parser/angle_link.rs

@@ -4,6 +4,7 @@ use nom::combinator::peek;
 use nom::combinator::recognize;
 use nom::multi::many_till;
 
+use super::org_source::OrgSource;
 use super::Context;
 use crate::error::Res;
 use crate::parser::exiting::ExitClass;
@@ -15,8 +16,11 @@ use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
 use crate::parser::AngleLink;
 
-#[tracing::instrument(ret, level = "debug")]
-pub fn angle_link<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, AngleLink<'s>> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn angle_link<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, AngleLink<'s>> {
     let (remaining, _) = tag("<")(input)?;
     let (remaining, proto) = protocol(context, remaining)?;
     let (remaining, _separator) = tag(":")(remaining)?;
@@ -26,18 +30,21 @@ pub fn angle_link<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
     Ok((
         remaining,
         AngleLink {
-            source,
-            link_type: proto,
-            path,
+            source: source.into(),
+            link_type: proto.into(),
+            path: path.into(),
         },
     ))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn path_angle<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn path_angle<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let parser_context =
         context.with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Gamma,
             exit_matcher: &path_angle_end,
         }));
 
@@ -47,7 +54,10 @@ fn path_angle<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
     Ok((remaining, path))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn path_angle_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn path_angle_end<'r, 's>(
+    _context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     tag(">")(input)
 }

src/parser/citation.rs

@@ -11,6 +11,7 @@ use nom::multi::many_till;
 use nom::multi::separated_list1;
 use nom::sequence::tuple;
 
+use super::org_source::OrgSource;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::Res;
@@ -28,8 +29,11 @@ use crate::parser::util::exit_matcher_parser;
 use crate::parser::util::get_consumed;
 use crate::parser::Object;
 
-#[tracing::instrument(ret, level = "debug")]
-pub fn citation<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Citation<'s>> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+pub fn citation<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Citation<'s>> {
     // TODO: Despite being a standard object, citations cannot exist inside the global prefix/suffix for other citations because citations must contain something that matches @key which is forbidden inside the global prefix/suffix. This TODO is to evaluate if its worth putting in an explicit check for this (which can be easily accomplished by checking the output of `get_bracket_depth()`). I suspect its not worth it because I expect, outside of intentionally crafted inputs, this parser will exit immediately inside a citation since it is unlikely to find the "[cite" substring inside a citation global prefix/suffix.
     let (remaining, _) = tag_no_case("[cite")(input)?;
     let (remaining, _) = opt(citestyle)(remaining)?;
@@ -44,36 +48,41 @@ pub fn citation<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str
     let (remaining, _) = tag("]")(remaining)?;
     let (remaining, _) = space0(remaining)?;
     let source = get_consumed(input, remaining);
-    Ok((remaining, Citation { source }))
+    Ok((
+        remaining,
+        Citation {
+            source: source.into(),
+        },
+    ))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn citestyle<'r, 's>(input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn citestyle<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let (remaining, _) = tuple((tag("/"), style))(input)?;
     let (remaining, _) = opt(tuple((tag("/"), variant)))(remaining)?;
     let source = get_consumed(input, remaining);
     Ok((remaining, source))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn style<'r, 's>(input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn style<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     recognize(many1(verify(anychar, |c| {
         c.is_alphanumeric() || "_-".contains(*c)
     })))(input)
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn variant<'r, 's>(input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn variant<'r, 's>(input: OrgSource<'s>) -> Res<OrgSource<'s>, OrgSource<'s>> {
     recognize(many1(verify(anychar, |c| {
         c.is_alphanumeric() || "_-/".contains(*c)
     })))(input)
 }
 
-#[tracing::instrument(ret, level = "debug")]
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn global_prefix<'r, 's>(
     context: Context<'r, 's>,
-    input: &'s str,
-) -> Res<&'s str, Vec<Object<'s>>> {
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
     // TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
     let parser_context = context
         .with_additional_node(ContextElement::CitationBracket(CitationBracket {
@@ -81,7 +90,7 @@ fn global_prefix<'r, 's>(
             depth: 0,
         }))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Gamma,
             exit_matcher: &global_prefix_end,
         }));
     let (remaining, (children, _exit_contents)) = verify(
@@ -95,13 +104,16 @@ fn global_prefix<'r, 's>(
     Ok((remaining, children))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn global_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn global_prefix_end<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let context_depth = get_bracket_depth(context)
         .expect("This function should only be called from inside a citation.");
     let text_since_context_entry = get_consumed(context_depth.position, input);
     let mut current_depth = context_depth.depth;
-    for c in text_since_context_entry.chars() {
+    for c in Into::<&str>::into(text_since_context_entry).chars() {
         match c {
             '[' => {
                 current_depth += 1;
@@ -116,7 +128,7 @@ fn global_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'
         }
     }
     if current_depth == 0 {
-        let close_bracket = tag::<&str, &str, CustomError<&str>>("]")(input);
+        let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
         if close_bracket.is_ok() {
             return close_bracket;
         }
@@ -127,11 +139,11 @@ fn global_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'
     ))(input)
 }
 
-#[tracing::instrument(ret, level = "debug")]
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 fn global_suffix<'r, 's>(
     context: Context<'r, 's>,
-    input: &'s str,
-) -> Res<&'s str, Vec<Object<'s>>> {
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
     // TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
     let parser_context = context
         .with_additional_node(ContextElement::CitationBracket(CitationBracket {
@@ -139,7 +151,7 @@ fn global_suffix<'r, 's>(
             depth: 0,
         }))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Gamma,
             exit_matcher: &global_suffix_end,
         }));
     let (remaining, (children, _exit_contents)) = verify(
@@ -152,13 +164,16 @@ fn global_suffix<'r, 's>(
     Ok((remaining, children))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn global_suffix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn global_suffix_end<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let context_depth = get_bracket_depth(context)
         .expect("This function should only be called from inside a citation.");
     let text_since_context_entry = get_consumed(context_depth.position, input);
     let mut current_depth = context_depth.depth;
-    for c in text_since_context_entry.chars() {
+    for c in Into::<&str>::into(text_since_context_entry).chars() {
         match c {
             '[' => {
                 current_depth += 1;
@@ -173,7 +188,7 @@ fn global_suffix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'
         }
     }
     if current_depth == 0 {
-        let close_bracket = tag::<&str, &str, CustomError<&str>>("]")(input);
+        let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
         if close_bracket.is_ok() {
             return close_bracket;
         }
@@ -188,24 +203,21 @@ fn global_suffix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'
 mod tests {
     use super::*;
     use crate::parser::element_parser::element;
-    use crate::parser::parser_context::ContextElement;
     use crate::parser::parser_context::ContextTree;
     use crate::parser::parser_with_context::parser_with_context;
     use crate::parser::source::Source;
 
     #[test]
     fn citation_simple() {
-        let input = "[cite:@foo]";
+        let input = OrgSource::new("[cite:@foo]");
         let initial_context: ContextTree<'_, '_> = ContextTree::new();
-        let document_context =
-            initial_context.with_additional_node(ContextElement::DocumentRoot(input));
-        let paragraph_matcher = parser_with_context!(element(true))(&document_context);
+        let paragraph_matcher = parser_with_context!(element(true))(&initial_context);
         let (remaining, first_paragraph) = paragraph_matcher(input).expect("Parse first paragraph");
         let first_paragraph = match first_paragraph {
             crate::parser::Element::Paragraph(paragraph) => paragraph,
             _ => panic!("Should be a paragraph!"),
         };
-        assert_eq!(remaining, "");
+        assert_eq!(Into::<&str>::into(remaining), "");
         assert_eq!(first_paragraph.get_source(), "[cite:@foo]");
         assert_eq!(first_paragraph.children.len(), 1);
         assert_eq!(

src/parser/citation_reference.rs

@@ -10,6 +10,7 @@ use nom::multi::many_till;
 use nom::sequence::preceded;
 use nom::sequence::tuple;
 
+use super::org_source::OrgSource;
 use super::Context;
 use crate::error::CustomError;
 use crate::error::Res;
@@ -25,24 +26,29 @@ use crate::parser::util::get_consumed;
 use crate::parser::util::WORD_CONSTITUENT_CHARACTERS;
 use crate::parser::Object;
 
-#[tracing::instrument(ret, level = "debug")]
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn citation_reference<'r, 's>(
     context: Context<'r, 's>,
-    input: &'s str,
-) -> Res<&'s str, CitationReference<'s>> {
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, CitationReference<'s>> {
     let (remaining, _prefix) = opt(parser_with_context!(key_prefix)(context))(input)?;
     let (remaining, _key) = parser_with_context!(citation_reference_key)(context)(remaining)?;
     let (remaining, _suffix) = opt(parser_with_context!(key_suffix)(context))(remaining)?;
     let source = get_consumed(input, remaining);
 
-    Ok((remaining, CitationReference { source }))
+    Ok((
+        remaining,
+        CitationReference {
+            source: source.into(),
+        },
+    ))
 }
 
-#[tracing::instrument(ret, level = "debug")]
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn citation_reference_key<'r, 's>(
     context: Context<'r, 's>,
-    input: &'s str,
-) -> Res<&'s str, &'s str> {
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let (remaining, source) = recognize(tuple((
         tag("@"),
         many1(verify(
@@ -58,8 +64,11 @@ pub fn citation_reference_key<'r, 's>(
     Ok((remaining, source))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn key_prefix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Vec<Object<'s>>> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn key_prefix<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
     // TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
     let parser_context = context
         .with_additional_node(ContextElement::CitationBracket(CitationBracket {
@@ -67,7 +76,7 @@ fn key_prefix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
             depth: 0,
         }))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Gamma,
             exit_matcher: &key_prefix_end,
         }));
     let (remaining, (children, _exit_contents)) = verify(
@@ -80,8 +89,11 @@ fn key_prefix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
     Ok((remaining, children))
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn key_suffix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, Vec<Object<'s>>> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn key_suffix<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, Vec<Object<'s>>> {
     // TODO: I could insert CitationBracket entries in the context after each matched object to reduce the scanning done for counting brackets which should be more efficient.
     let parser_context = context
         .with_additional_node(ContextElement::CitationBracket(CitationBracket {
@@ -89,7 +101,7 @@ fn key_suffix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
             depth: 0,
         }))
         .with_additional_node(ContextElement::ExitMatcherNode(ExitMatcherNode {
-            class: ExitClass::Beta,
+            class: ExitClass::Gamma,
             exit_matcher: &key_suffix_end,
         }));
     let (remaining, (children, _exit_contents)) = verify(
@@ -102,7 +114,7 @@ fn key_suffix<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str,
     Ok((remaining, children))
 }
 
-#[tracing::instrument(ret, level = "debug")]
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
 pub fn get_bracket_depth<'r, 's>(context: Context<'r, 's>) -> Option<&'r CitationBracket<'s>> {
     for node in context.iter() {
         match node.get_data() {
@@ -113,13 +125,16 @@ pub fn get_bracket_depth<'r, 's>(context: Context<'r, 's>) -> Option<&'r Citatio
     None
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn key_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn key_prefix_end<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let context_depth = get_bracket_depth(context)
         .expect("This function should only be called from inside a citation reference.");
     let text_since_context_entry = get_consumed(context_depth.position, input);
     let mut current_depth = context_depth.depth;
-    for c in text_since_context_entry.chars() {
+    for c in Into::<&str>::into(text_since_context_entry).chars() {
         match c {
             '[' => {
                 current_depth += 1;
@@ -134,7 +149,7 @@ fn key_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
         }
     }
     if current_depth == 0 {
-        let close_bracket = tag::<&str, &str, CustomError<&str>>("]")(input);
+        let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
         if close_bracket.is_ok() {
             return close_bracket;
         }
@@ -145,13 +160,16 @@ fn key_prefix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
     ))(input)
 }
 
-#[tracing::instrument(ret, level = "debug")]
-fn key_suffix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s str, &'s str> {
+#[cfg_attr(feature = "tracing", tracing::instrument(ret, level = "debug"))]
+fn key_suffix_end<'r, 's>(
+    context: Context<'r, 's>,
+    input: OrgSource<'s>,
+) -> Res<OrgSource<'s>, OrgSource<'s>> {
     let context_depth = get_bracket_depth(context)
         .expect("This function should only be called from inside a citation reference.");
     let text_since_context_entry = get_consumed(context_depth.position, input);
     let mut current_depth = context_depth.depth;
-    for c in text_since_context_entry.chars() {
+    for c in Into::<&str>::into(text_since_context_entry).chars() {
         match c {
             '[' => {
                 current_depth += 1;
@@ -166,7 +184,7 @@ fn key_suffix_end<'r, 's>(context: Context<'r, 's>, input: &'s str) -> Res<&'s s
         }
     }
     if current_depth == 0 {
-        let close_bracket = tag::<&str, &str, CustomError<&str>>("]")(input);
+        let close_bracket = tag::<&str, OrgSource<'_>, CustomError<OrgSource<'_>>>("]")(input);
         if close_bracket.is_ok() {
             return close_bracket;
         }