Publish version 0.1.5.

This test will grab documents from external sources and compare Organic's parser vs the official org-mode parser to ensure they are parsing the same. This is so we do not introduce large irrelevant documents in the git history and so we do not introduce documents with restrictive licenses into the repository.

.dockerignore

@@ -1,3 +1,4 @@
 **/.git
 target
 Cargo.lock
+notes/

.lighthouse/pipeline-foreign-document-test.yaml

@@ -0,0 +1,203 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: rust-foreign-document-test
+spec:
+  pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
+    params:
+      - name: image-name
+        description: The name for the built image
+        type: string
+      - name: path-to-image-context
+        description: The path to the build context
+        type: string
+      - name: path-to-dockerfile
+        description: The path to the Dockerfile
+        type: string
+    tasks:
+      - name: do-stuff
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.18
+            name: ""
+            resources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: /workspace/source
+          steps:
+            - image: alpine:3.18
+              name: do-stuff-step
+              script: |
+                #!/usr/bin/env sh
+                echo "hello world"
+      - name: report-pending
+        taskRef:
+          name: gitea-set-status
+        runAfter:
+          - fetch-repository
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          name: git-clone
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+      - name: build-image
+        taskRef:
+          name: kaniko
+        params:
+          - name: IMAGE
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+          - name: CONTEXT
+            value: $(params.path-to-image-context)
+          - name: DOCKERFILE
+            value: $(params.path-to-dockerfile)
+          - name: BUILDER_IMAGE
+            value: "gcr.io/kaniko-project/executor:v1.12.1"
+          - name: EXTRA_ARGS
+            value:
+              - --target=foreign-document-test
+              - --cache=true
+              - --cache-copy-layers
+              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - --use-new-run # Should result in a speed-up
+              - --reproducible # To remove timestamps so layer caching works.
+              - --snapshot-mode=redo
+              - --skip-unused-stages=true
+              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: dockerconfig
+            workspace: docker-credentials
+        runAfter:
+          - fetch-repository
+      - name: run-image
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        runAfter:
+          - build-image
+        params:
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: cargo-cache-autoclean
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: command
+            value: [cargo, cache, --autoclean]
+          - name: args
+            value: []
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    workspaces:
+      - name: git-source
+      - name: docker-credentials
+      - name: cargo-cache
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: rust-source
+    - name: cargo-cache
+      persistentVolumeClaim:
+        claimName: organic-cargo-cache-test-foreign-document
+    - name: docker-credentials
+      secret:
+        secretName: harbor-plain
+  serviceAccountName: build-bot
+  params:
+    - name: image-name
+      value: "harbor.fizz.buzz/private/organic-test-foreign-document"
+    - name: path-to-image-context
+      value: docker/organic_test/
+    - name: path-to-dockerfile
+      value: docker/organic_test/Dockerfile

.lighthouse/pipeline-rust-build.yaml

@@ -14,10 +14,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: command
-        type: array
-        description: Command to run.
-        default: []
     tasks:
       - name: report-pending
         taskRef:
@@ -92,8 +88,6 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features"]
           - name: docker-image
@@ -109,8 +103,6 @@ spec:
         runAfter:
           - run-image-none
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "tracing"]
           - name: docker-image
@@ -126,8 +118,6 @@ spec:
         runAfter:
           - run-image-tracing
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "compare"]
           - name: docker-image
@@ -143,8 +133,6 @@ spec:
         runAfter:
           - run-image-compare
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: []
           - name: docker-image
@@ -160,8 +148,6 @@ spec:
         runAfter:
           - run-image-default
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
             value: ["--no-default-features", "--features", "tracing,compare"]
           - name: docker-image
@@ -256,5 +242,3 @@ spec:
       value: docker/organic_build/
     - name: path-to-dockerfile
       value: docker/organic_build/Dockerfile
-    - name: command
-      value: [cargo, build]

.lighthouse/pipeline-rust-test.yaml

@@ -18,14 +18,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: command
-        type: array
-        description: Command to run.
-        default: []
-      - name: args
-        type: array
-        description: Arguments passed to command.
-        default: []
     tasks:
       - name: do-stuff
         taskSpec:
@@ -91,6 +83,7 @@ spec:
             value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
+              - --target=tester
               - --cache=true
               - --cache-copy-layers
               - --cache-repo=harbor.fizz.buzz/kanikocache/cache
@@ -117,10 +110,17 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.command[*])"]
           - name: args
-            value: ["$(params.args[*])"]
+            value:
+              [
+                --no-default-features,
+                --features,
+                compare,
+                --no-fail-fast,
+                --lib,
+                --test,
+                test_loader,
+              ]
           - name: docker-image
             value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
     finally:
@@ -212,7 +212,3 @@ spec:
       value: docker/organic_test/
     - name: path-to-dockerfile
       value: docker/organic_test/Dockerfile
-    - name: command
-      value: [cargo, test]
-    - name: args
-      value: [--lib, --test, test_loader]

.lighthouse/pipeline-rustfmt.yaml

@@ -14,14 +14,6 @@ spec:
       - name: path-to-dockerfile
         description: The path to the Dockerfile
         type: string
-      - name: rustfmt-command
-        type: array
-        description: Command to run rustfmt.
-        default: []
-      - name: rustfmt-args
-        type: array
-        description: Arguments passed to rustfmt.
-        default: []
       - name: GIT_USER_NAME
         description: The username for git
         type: string
@@ -119,10 +111,6 @@ spec:
         runAfter:
           - build-image
         params:
-          - name: command
-            value: ["$(params.rustfmt-command[*])"]
-          - name: args
-            value: ["$(params.rustfmt-args[*])"]
           - name: docker-image
             value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
       - name: cargo-fix
@@ -240,7 +228,3 @@ spec:
       value: docker/cargo_fmt/
     - name: path-to-dockerfile
       value: docker/cargo_fmt/Dockerfile
-    - name: command
-      value: [cargo, fmt]
-    - name: args
-      value: []

.lighthouse/triggers.yaml

@@ -16,6 +16,13 @@ spec:
       skip_branches:
         # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
         - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+    - name: rust-foreign-document-test
+      source: "pipeline-foreign-document-test.yaml"
+      # Override https-based url from lighthouse events.
+      clone_uri: "git@code.fizz.buzz:talexander/organic.git"
+      skip_branches:
+        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
+        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
     - name: rust-build
       source: "pipeline-rust-build.yaml"
       # Override https-based url from lighthouse events.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "organic"
-version = "0.1.2"
+version = "0.1.5"
 authors = ["Tom Alexander <tom@fizz.buzz>"]
 description = "An org-mode parser."
 edition = "2021"
@@ -13,8 +13,7 @@ resolver = "2"
 include = [
   "LICENSE",
   "**/*.rs",
-  "Cargo.toml",
+  "Cargo.toml"
-  "tests/*"
 ]
 
 [lib]
@@ -23,7 +22,7 @@ path = "src/lib.rs"
 
 [[bin]]
 # This bin exists for development purposes only. The real target of this crate is the library.
-name = "compare"
+name = "parse"
 path = "src/main.rs"
 
 [dependencies]
@@ -40,10 +39,18 @@ tracing-subscriber = { version = "0.3.17", optional = true, features = ["env-fil
 walkdir = "2.3.3"
 
 [features]
-default = ["compare", "tracing"]
+default = []
 compare = []
 tracing = ["dep:opentelemetry", "dep:opentelemetry-otlp", "dep:opentelemetry-semantic-conventions", "dep:tokio", "dep:tracing", "dep:tracing-opentelemetry", "dep:tracing-subscriber"]
 
-[profile.release]
+# Optimized build for any sort of release.
+[profile.release-lto]
+inherits = "release"
 lto = true
 strip = "symbols"
+
+# Profile for performance testing with the "perf" tool. Notably keeps debug enabled and does not strip symbols to make reading the perf output easier.
+[profile.perf]
+inherits = "release"
+lto = true
+debug = true

Makefile

@@ -35,12 +35,16 @@ clean:
 
 .PHONY: test
 test:
-> cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+> cargo test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: dockertest
 dockertest:
 > $(MAKE) -C docker/organic_test
-> docker run --init --rm -i -t -v "$$(readlink -f ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test cargo test --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+> docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
+
+.PHONY: foreign_document_test
+foreign_document_test:
+> $(MAKE) -C docker/organic_test run_foreign_document_test
 
 .PHONY: dockerclean
 dockerclean:
@@ -49,18 +53,18 @@ dockerclean:
 
 .PHONY: integrationtest
 integrationtest:
-> cargo test --no-fail-fast --test test_loader -- --test-threads $(TESTJOBS)
+> cargo test --no-default-features --features compare --no-fail-fast --test test_loader -- --test-threads $(TESTJOBS)
 
 .PHONY: unittest
 unittest:
-> cargo test --lib -- --test-threads $(TESTJOBS)
+> cargo test --no-default-features --lib -- --test-threads $(TESTJOBS)
 
 .PHONY: jaeger
 jaeger:
 # 4317 for OTLP gRPC, 4318 for OTLP HTTP. We currently use gRPC but I forward both ports regardless.
 #
 # These flags didn't help even though they seem like they would: --collector.queue-size=20000 --collector.num-workers=100
-> docker run -d --rm --name organicdocker -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
+> docker run -d --rm --name organicdocker --read-only -p 4317:4317 -p 4318:4318 -p 16686:16686 -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.47 --collector.grpc-server.max-message-size=20000000 --collector.otlp.grpc.max-message-size=20000000
 
 .PHONY: jaegerweb
 jaegerweb:

build.rs

@@ -1,10 +1,16 @@
+#[cfg(feature = "compare")]
 use std::env;
+#[cfg(feature = "compare")]
 use std::fs::File;
+#[cfg(feature = "compare")]
 use std::io::Write;
+#[cfg(feature = "compare")]
 use std::path::Path;
 
+#[cfg(feature = "compare")]
 use walkdir::WalkDir;
 
+#[cfg(feature = "compare")]
 fn main() {
     let out_dir = env::var("OUT_DIR").unwrap();
     let destination = Path::new(&out_dir).join("tests.rs");
@@ -31,6 +37,10 @@ fn main() {
     }
 }
 
+#[cfg(not(feature = "compare"))]
+fn main() {}
+
+#[cfg(feature = "compare")]
 fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
     let test_name = test
         .path()
@@ -41,6 +51,7 @@ fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
         .strip_suffix(".org")
         .expect("Should have .org extension")
         .replace("/", "_");
+    let test_name = format!("autogen_{}", test_name);
 
     if let Some(_reason) = is_expect_fail(test_name.as_str()) {
         write!(test_file, "#[ignore]\n").unwrap();
@@ -54,14 +65,15 @@ fn write_test(test_file: &mut File, test: &walkdir::DirEntry) {
     .unwrap();
 }
 
+#[cfg(feature = "compare")]
 fn write_header(test_file: &mut File) {
     write!(
         test_file,
         r#"
 #[feature(exit_status_error)]
 use organic::compare_document;
-use organic::parser::document;
+use organic::parser::parse;
-use organic::emacs_parse_org_document;
+use organic::emacs_parse_anonymous_org_document;
 use organic::parser::sexp::sexp_with_padding;
 
 "#
@@ -69,13 +81,12 @@ use organic::parser::sexp::sexp_with_padding;
     .unwrap();
 }
 
+#[cfg(feature = "compare")]
 fn is_expect_fail(name: &str) -> Option<&str> {
     match name {
-        "drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
+        "autogen_greater_element_drawer_drawer_with_headline_inside" => Some("Apparently lines with :end: become their own paragraph. This odd behavior needs to be investigated more."),
-        "element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
+        "autogen_element_container_priority_footnote_definition_dynamic_block" => Some("Apparently broken begin lines become their own paragraph."),
-        "paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
+        "autogen_lesser_element_paragraphs_paragraph_with_backslash_line_breaks" => Some("The text we're getting out of the parse tree is already processed to remove line breaks, so our comparison needs to take that into account."),
-        "export_snippet_paragraph_break_precedence" => Some("The latest code for org-mode is matching the export snippet without the closing @@."), // https://list.orgmode.org/orgmode/fb61ea28-f004-4c25-adf7-69fc55683ed4@app.fastmail.com/T/#u
-        "plain_lists_trailing_whitespace_ownership" => Some("Seeing odd behavior about whitespace ownership."),
         _ => None,
     }
 }

docker/cargo_fmt/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:
@@ -26,10 +26,11 @@ else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
 
+# NOTE: This target will write to folders underneath the git-root
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME)

docker/organic_build/Dockerfile

@@ -2,3 +2,5 @@ FROM rustlang/rust:nightly-alpine3.17
 
 RUN apk add --no-cache musl-dev
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
+
+ENTRYPOINT ["cargo", "build"]

docker/organic_build/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:
@@ -25,11 +25,13 @@ ifdef REMOTE_REPO
 else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
+	docker volume rm cargo-cache
 
+# NOTE: This target will write to folders underneath the git-root
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME)

docker/organic_test/Dockerfile

@@ -1,11 +1,10 @@
 FROM alpine:3.17 AS build
-
 RUN apk add --no-cache build-base musl-dev git autoconf make texinfo gnutls-dev ncurses-dev gawk
 
 
 FROM build AS build-emacs
-
+ARG EMACS_VERSION=emacs-29.1
-RUN git clone --depth 1 --branch emacs-29.1 https://git.savannah.gnu.org/git/emacs.git /root/emacs
+RUN git clone --depth 1 --branch $EMACS_VERSION https://git.savannah.gnu.org/git/emacs.git /root/emacs
 WORKDIR /root/emacs
 RUN mkdir /root/dist
 RUN ./autogen.sh
@@ -15,16 +14,35 @@ RUN make DESTDIR="/root/dist" install
 
 
 FROM build AS build-org-mode
+ARG ORG_VERSION=7bdec435ff5d86220d13c431e799c5ed44a57da1
 COPY --from=build-emacs /root/dist/ /
 RUN mkdir /root/dist
-RUN mkdir /root/org-mode && git -C /root/org-mode init --initial-branch=main && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin 3cbd9f423385bf725dc964a5cff573bba17db3ff && git -C /root/org-mode checkout FETCH_HEAD
+# Savannah does not allow fetching specific revisions, so we're going to have to put unnecessary load on their server by cloning main and then checking out the revision we want.
+RUN git clone https://git.savannah.gnu.org/git/emacs/org-mode.git /root/org-mode && git -C /root/org-mode checkout $ORG_VERSION
+# RUN mkdir /root/org-mode && git -C /root/org-mode init --initial-branch=main && git -C /root/org-mode remote add origin https://git.savannah.gnu.org/git/emacs/org-mode.git && git -C /root/org-mode fetch origin $ORG_VERSION && git -C /root/org-mode checkout FETCH_HEAD
 WORKDIR /root/org-mode
 RUN make compile
 RUN make DESTDIR="/root/dist" install
 
 
-FROM rustlang/rust:nightly-alpine3.17
+FROM rustlang/rust:nightly-alpine3.17 AS tester
+ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
 COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
+
+ENTRYPOINT ["cargo", "test"]
+
+
+FROM build as foreign-document-gather
+RUN mkdir /foreign_documents
+
+
+FROM tester as foreign-document-test
+RUN apk add --no-cache bash
+RUN mkdir /foreign_documents
+COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
+COPY foreign_document_test_entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

docker/organic_test/Makefile

@@ -6,7 +6,11 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../
+	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester .
+
+.PHONY: build_foreign_document_test
+build_foreign_document_test:
+	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test .
 
 .PHONY: push
 push:
@@ -25,11 +29,16 @@ ifdef REMOTE_REPO
 else
 	@echo "REMOTE_REPO not defined, not removing from remote repo."
 endif
+	docker volume rm rust-cache cargo-cache
 
 .PHONY: run
-run:
+run: build
-	docker run --rm -i -t $(IMAGE_NAME)
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-default-features --features compare --no-fail-fast --lib --test test_loader
 
 .PHONY: shell
-shell:
+shell: build
-	docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME)
+	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+
+.PHONY: run_foreign_document_test
+run_foreign_document_test: build_foreign_document_test
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)-foreign-document

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Run the Organic compare script against a series of documents sourced from exterior places.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function log {
+    (>&2 echo "${@}")
+}
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function main {
+    cargo build --no-default-features --features compare --profile release-lto
+    if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
+        CARGO_TARGET_DIR=$(realpath target/)
+    fi
+    PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
+
+    run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"
+    run_compare "org-mode/org-manual.org" "/foreign_documents/org-mode/org-manual.org"
+}
+
+function run_compare {
+    local name="$1"
+    local target_document="$2"
+    set +e
+    $PARSE "$target_document" &> /dev/null
+    local status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        echo "$(green_text "GOOD")  $name"
+    else
+        echo "$(red_text "FAIL")  $name"
+        return 1
+    fi
+}
+
+function green_text {
+    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
+}
+
+function red_text {
+    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
+}
+
+function yellow_text {
+    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
+}
+
+main "${@}"

elisp_snippets/README.md

@@ -0,0 +1 @@
+This folder is for snippets of elisp that are useful for development.

elisp_snippets/dump_org_element_affiliated_keywords.el

@@ -0,0 +1,3 @@
+(dolist (var org-element-affiliated-keywords)
+  (message "\"%s\"," (downcase var))
+  )

elisp_snippets/dump_org_entities.el

@@ -0,0 +1,5 @@
+(dolist (var org-entities)
+  (when (listp var)
+      (message "\"%s\"," (nth 0 var))
+    )
+  )

notes/plain_list_ownership_notes.org

@@ -1,130 +0,0 @@
-* Test 1
-** Source
-#+begin_src org
-  1. foo
-
-     1. bar
-
-     2. baz
-
-  2. lorem
-
-  ipsum
-#+end_src
-** Ownership
-This table is just showing ownership for the plain list items, not the containing plain list nor the elements inside each item.
-
-| Plain List *Item*      | Owns trailing blank lines |
-|------------------------+---------------------------|
-| foo (includes bar baz) | Yes                       |
-| bar                    | Yes                       |
-| baz                    | Yes                       |
-| lorem                  | No                        |
-** Analysis
-In this test case, we see that the only list item that doesn't own its trailing blank lines is "lorem", the final list item of the outer-most list.
-* Test 2
-We add "cat" as a paragraph at the end of foo which makes "baz" lose its trailing blank lines.
-** Source
-#+begin_src org
-  1. foo
-
-     1. bar
-
-     2. baz
-
-     cat
-
-  2. lorem
-
-  ipsum
-#+end_src
-** Ownership
-| Plain List *Item*             | Owns trailing blank lines |
-|-------------------------------+---------------------------|
-| foo -> cat (includes bar baz) | Yes                       |
-| bar                           | Yes                       |
-| baz                           | No                        |
-| lorem                         | No                        |
-** Analysis
-In isolation, this implies that the final plain list item does not own its trailing blank lines, which conflicts with "baz" from test 1.
-
-New theory: List items own their trailing blank lines unless they are both the final list item and not the final element of a list item.
-
-| Plain List *Item*             | Owns trailing blank lines | Why                                                       |
-|-------------------------------+---------------------------+-----------------------------------------------------------|
-| foo -> cat (includes bar baz) | Yes                       | Not the final list item                                   |
-| bar                           | Yes                       | Not the final list item                                   |
-| baz                           | No                        | Final item of bar->baz and not the final element of "foo" |
-| lorem                         | No                        | Final item of foo->lorem and not contained in a list item |
-* Test 3
-So if that theory is true, taking the entire (foo -> lorem) list from test 1 and nesting it inside a list should coerce "lorem" to own its trailing blank lines since it would then be a final list item (of foo -> lorem) and the final element of the new list.
-** Source
-#+begin_src org
-  1. cat
-     1. foo
-
-        1. bar
-
-        2. baz
-
-     2. lorem
-
-  ipsum
-#+end_src
-** Ownership
-| Plain List *Item*           | Owns trailing blank lines |
-|-----------------------------+---------------------------|
-| cat (includes foo -> lorem) | No                        |
-| foo (includes bar baz)      | Yes                       |
-| bar                         | Yes                       |
-| baz                         | Yes                       |
-| lorem                       | No                        |
-** Analysis
-Against expectations, we did not coerce lorem to consume its trailing blank lines. What is different between "baz" and "lorem"? Well, "baz" is contained within "foo" which has a "lorem" after it, whereas "lorem" is contained within "cat" which does not have any list items after it.
-
-New theory: List items own their trailing blank lines unless they are both the final list item and not the final element of a non-final list item.
-| Plain List *Item*           | Owns trailing blank lines | Why                                                  |
-|-----------------------------+---------------------------+------------------------------------------------------|
-| cat (includes foo -> lorem) | No                        | Final list item and not contained in a list item     |
-| foo (includes bar baz)      | Yes                       | Not the final list item                              |
-| bar                         | Yes                       | Not the final list item                              |
-| baz                         | Yes                       | Final element of non-final list item                 |
-| lorem                       | No                        | Final list item and final element of final list item |
-* Test 4
-So if that theory is true, then we should be able to coerce lorem to consume its trailing blank lines by adding a second item to the cat list.
-** Source
-#+begin_src org
-  1. cat
-     1. foo
-
-        1. bar
-
-        2. baz
-
-     2. lorem
-
-  2. dog
-
-  ipsum
-#+end_src
-** Ownership
-| Plain List *Item*           | Owns trailing blank lines |
-|-----------------------------+---------------------------|
-| cat (includes foo -> lorem) | Yes                       |
-| foo (includes bar baz)      | Yes                       |
-| bar                         | Yes                       |
-| baz                         | Yes                       |
-| lorem                       | Yes                       |
-| dog                         | No                        |
-** Analysis
-For the first time our expectations were met!
-
-Enduring theory: List items own their trailing blank lines unless they are both the final list item and not the final element of a non-final list item.
-| Plain List *Item*           | Owns trailing blank lines | Why                                              |
-|-----------------------------+---------------------------+--------------------------------------------------|
-| cat (includes foo -> lorem) | Yes                       | Not the final list item                          |
-| foo (includes bar baz)      | Yes                       | Not the final list item                          |
-| bar                         | Yes                       | Not the final list item                          |
-| baz                         | Yes                       | Final element of non-final list item             |
-| lorem                       | Yes                       | Final element of non-final list item             |
-| dog                         | No                        | Final list item and not contained in a list item |

org_mode_samples/README.org

@@ -0,0 +1 @@
+This folder contains org-mode documents that get automatically included as tests using build.rs.

org_mode_samples/element_container_priority/plain_list_greater_block.org

@@ -1,6 +1,7 @@
-1. foo
+1. plain-list
    #+begin_center
 
 
    #+end_center
-2. bar
+
+   Is this still in the plain list?

org_mode_samples/exit_matcher_investigation/README.txt

@@ -1 +0,0 @@
-This folder is an investigation into whether or not my exit matchers should operate from the top down or bottom up.

org_mode_samples/exit_matcher_investigation/bold_with_asterisk_inside.org

@@ -1 +0,0 @@
-foo *bar baz * lorem* ipsum

org_mode_samples/exit_matcher_investigation/list_paragraph_nesting.README.txt

@@ -1,3 +0,0 @@
-Looks like 2 blank lines always exits the top-level plain list.
-
-Plain lists do not seem to go inside paragraphs but rather exist beside them.

org_mode_samples/exit_matcher_investigation/list_paragraph_nesting.org

@@ -1,12 +0,0 @@
-1. foo
-
-   bar
-
-   1. baz
-
-      lorem
-
-   ipsum
-
-
-dolar

org_mode_samples/exit_matcher_investigation/table_list.README.txt

@@ -1 +0,0 @@
-Looks like table cells cannot contain lists but can contain bolds

org_mode_samples/exit_matcher_investigation/table_list.org

@@ -1,5 +0,0 @@
-ip *su* m
-
-| foo      | bar |
-|----------+-----|
-| 1. lo *re* m |     |

org_mode_samples/greater_element/dynamic_block/simple.org

@@ -0,0 +1,25 @@
+#+BEGIN: clocktable :scope file :maxlevel 2
+#+CAPTION: Clock summary at [2023-08-25 Fri 05:34]
+| Headline     | Time   |
+|--------------+--------|
+| *Total time* | *0:00* |
+#+END:
+
+#+BEGIN: columnview :hlines 1 :id global
+| ITEM  | TODO | PRIORITY | TAGS                         |
+|-------+------+----------+------------------------------|
+| Foo   |      | B        |                              |
+|-------+------+----------+------------------------------|
+| Bar   | TODO | B        |                              |
+|-------+------+----------+------------------------------|
+| Baz   |      | B        | :thisisatag:                 |
+| Lorem |      | B        | :thisshouldinheritfromabove: |
+| Ipsum |      | B        | :multiple:tags:              |
+#+END:
+* Foo
+* TODO Bar
+* Baz :thisisatag:
+** Lorem :thisshouldinheritfromabove:
+*** Ipsum                                                   :multiple:tags:
+* Dolar ::
+* cat :dog: bat

org_mode_samples/greater_element/greater_block/deeply_nested.org

@@ -0,0 +1,18 @@
+#+begin_defun
+foo
+#+begin_lorem
+,#+begin_center
+bar
+,#+end_center
+ipsum
+#+end_lorem
+baz
+#+end_defun
+
+#+begin_center
+#+begin_quote
+#+begin_center
+lorem
+#+end_center
+#+end_quote
+#+end_center

org_mode_samples/greater_element/greater_block/nested_special_block.org

@@ -0,0 +1,12 @@
+#+begin_defun
+foo
+#+begin_lorem
+ipsum
+#+end_lorem
+bar
+#+begin_center
+#+begin_quote
+baz
+#+end_quote
+#+end_center
+#+end_defun

org_mode_samples/greater_element/greater_block/special_block.org

@@ -0,0 +1,5 @@
+#+begin_defun
+foo
+
+{{{bar(baz)}}}
+#+end_defun

org_mode_samples/greater_element/plain_list/content_on_next_line.org

@@ -0,0 +1,7 @@
+1. foo
+2.
+     bar
+   1.
+       #+begin_center
+Still in the list
+       #+end_center

org_mode_samples/greater_element/plain_list/description_eof.org

@@ -0,0 +1,2 @@
+- foo ::
+

org_mode_samples/greater_element/plain_list/description_list.org

@@ -0,0 +1,11 @@
+- foo :: bar
+- cat    ::
+  dog
+- lorem
+  :: ipsum
+-
+  lorem :: ipsum
+- dolar *bold* foo :: ipsum
+- big gap ::
+
+  stuff

org_mode_samples/greater_element/plain_list/description_list_object_key.org

@@ -0,0 +1 @@
+- {{{foo(bar)}}} :: baz

org_mode_samples/greater_element/plain_list/trailing_whitespace_eof.org

@@ -0,0 +1,2 @@
+- foo
+

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_1.org

@@ -6,6 +6,4 @@
 
 2. lorem
 
-
-
 ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_2.org

@@ -0,0 +1,11 @@
+1. foo
+
+   1. bar
+
+   2. baz
+
+   cat
+
+2. lorem
+
+ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_3.org

@@ -0,0 +1,10 @@
+1. cat
+   1. foo
+
+      1. bar
+
+      2. baz
+
+   2. lorem
+
+ipsum

org_mode_samples/greater_element/plain_list/trailing_whitespace_ownership_test_case_4.org

@@ -0,0 +1,12 @@
+1. cat
+   1. foo
+
+      1. bar
+
+      2. baz
+
+   2. lorem
+
+2. dog
+
+ipsum

org_mode_samples/greater_element/property_drawer/properties_with_underscore.org

@@ -0,0 +1,7 @@
+** foo
+:PROPERTIES:
+:DESCRIPTION: lorem
+:ALT_TITLE: ipsum
+:END:
+
+bar

org_mode_samples/lesser_element/diary_sexp/with_comment_at_end_of_line.org

@@ -0,0 +1 @@
+%%(foo bar)    ;    baz

org_mode_samples/lesser_element/keyword/floating_affiliated_keyword.org

@@ -0,0 +1,15 @@
+#+name: foo
+#+caption: bar
+#+caption: baz
+
+[[file:lorem/ipsum.png]]
+
+#+name: cat
+#+foo: dog
+[[file:lorem/ipsum.png]]
+
+#+name: cat
+#+foo: dog
+
+
+foo

org_mode_samples/object/citation/balanced_brackets.org

@@ -0,0 +1,22 @@
+# Extra open
+[cite/a/b-_/foo:unbalancedglobal[prefix;keyprefix @foo keysuffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;unbalancedkey[prefix @foo keysuffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;keyprefix @foo unbalancedkey[suffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;keyprefix @foo keysuffix;unbalancedglobal[suffix]
+
+
+# Extra close
+[cite/a/b-_/foo:unbalancedglobal]prefix;keyprefix @foo keysuffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;unbalancedkey]prefix @foo keysuffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;keyprefix @foo unbalancedkey]suffix;globalsuffix]
+
+[cite/a/b-_/foo:globalprefix;keyprefix @foo keysuffix;unbalancedglobal]suffix]
+
+
+# balanced:
+[cite/a/b-_/foo:gl[obalpref]ix;ke[ypref]ix @foo ke[ysuff]ix;gl[obalsuff]ix]

org_mode_samples/object/footnote_reference/balanced_brackets.org

@@ -0,0 +1,2 @@
+[fn:2:This footnote [ has balanced ] brackets inside it]
+[fn::This footnote does not have balanced [ brackets inside it]

org_mode_samples/object/latex_fragment/three_lines.org

@@ -0,0 +1,6 @@
+$foo
+bar
+baz
+lorem
+ipsum
+dolar$

org_mode_samples/object/plain_link/empty_links.org

@@ -0,0 +1,52 @@
+non-link text
+eww://
+rmail://
+mhe://
+irc://
+info://
+gnus://
+docview://
+bibtex://
+bbdb://
+w3m://
+doi://
+file+sys://
+file+emacs://
+shell://
+news://
+mailto://
+https://
+http://
+ftp://
+help://
+file://
+elisp://
+randomfakeprotocl://
+non-link text
+
+
+non-link text
+eww:
+rmail:
+mhe:
+irc:
+info:
+gnus:
+docview:
+bibtex:
+bbdb:
+w3m:
+doi:
+file+sys:
+file+emacs:
+shell:
+news:
+mailto:
+https:
+http:
+ftp:
+help:
+file:
+elisp:
+randomfakeprotocl:
+non-link text

org_mode_samples/object/plain_link/end_with_punctuation.org

@@ -0,0 +1,3 @@
+mailto:foo@bar.baz.
+
+mailto:foo@bar.baz....

org_mode_samples/object/plain_link/with_trailing_whitespace.org

@@ -0,0 +1 @@
+mailto:foo@bar.baz      .