From 4e8b3eb422773f0d6e9c10a8ec756e875780d754 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 6 Sep 2023 15:54:25 -0400
Subject: [PATCH 1/5] Introduce a foreign document test docker container.

This test will grab documents from external sources and compare Organic's parser vs the official org-mode parser to ensure they are parsing the same. This is so we do not introduce large irrelevant documents in the git history and so we do not introduce documents with restrictive licenses into the repository.
---
 .lighthouse/pipeline-rust-test.yaml           |  1 +
 docker/organic_test/Dockerfile                | 15 +++++-
 docker/organic_test/Makefile                  | 10 +++-
 .../foreign_document_test_entrypoint.sh       | 54 +++++++++++++++++++
 4 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 docker/organic_test/foreign_document_test_entrypoint.sh

diff --git a/.lighthouse/pipeline-rust-test.yaml b/.lighthouse/pipeline-rust-test.yaml
index c1b2716..e5b05bf 100644
--- a/.lighthouse/pipeline-rust-test.yaml
+++ b/.lighthouse/pipeline-rust-test.yaml
@@ -83,6 +83,7 @@ spec:
             value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
+              - --target=tester
               - --cache=true
               - --cache-copy-layers
               - --cache-repo=harbor.fizz.buzz/kanikocache/cache
diff --git a/docker/organic_test/Dockerfile b/docker/organic_test/Dockerfile
index 05a3216..fab4a46 100644
--- a/docker/organic_test/Dockerfile
+++ b/docker/organic_test/Dockerfile
@@ -25,7 +25,7 @@ RUN make compile
 RUN make DESTDIR="/root/dist" install
 
 
-FROM rustlang/rust:nightly-alpine3.17
+FROM rustlang/rust:nightly-alpine3.17 AS tester
 ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
@@ -33,3 +33,16 @@ COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
 
 ENTRYPOINT ["cargo", "test"]
+
+
+FROM build as foreign-document-gather
+RUN mkdir /foreign_documents
+
+
+FROM tester as foreign-document-test
+RUN apk add --no-cache bash
+RUN mkdir /foreign_documents
+COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
+COPY docker/organic_test/foreign_document_test_entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile
index f8133ad..e0f82ab 100644
--- a/docker/organic_test/Makefile
+++ b/docker/organic_test/Makefile
@@ -6,7 +6,11 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester ../../
+
+.PHONY: build_foreign_document_test
+build_foreign_document_test:
+	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test ../../
 
 .PHONY: push
 push:
@@ -34,3 +38,7 @@ run: build
 .PHONY: shell
 shell: build
 	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+
+.PHONY: run_foreign_document_test
+run_foreign_document_test: build_foreign_document_test
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)-foreign-document
diff --git a/docker/organic_test/foreign_document_test_entrypoint.sh b/docker/organic_test/foreign_document_test_entrypoint.sh
new file mode 100644
index 0000000..29a7363
--- /dev/null
+++ b/docker/organic_test/foreign_document_test_entrypoint.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Run the Organic compare script against a series of documents sourced from exterior places.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function log {
+    (>&2 echo "${@}")
+}
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function main {
+    cargo build --no-default-features --features compare --profile release-lto
+    PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
+
+    run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"
+    run_compare "org-mode/org-manual.org" "/foreign_documents/org-mode/org-manual.org"
+}
+
+function run_compare {
+    local name="$1"
+    local target_document="$2"
+    set +e
+    $PARSE "$target_document" &> /dev/null
+    local status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        echo "$(green_text "GOOD")  $name"
+    else
+        echo "$(red_text "FAIL")  $name"
+        return 1
+    fi
+}
+
+function green_text {
+    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
+}
+
+function red_text {
+    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
+}
+
+function yellow_text {
+    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
+}
+
+main "${@}"

From a6561d37fb1e4653d130c3643e8dc2057292d0c5 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 6 Sep 2023 16:00:09 -0400
Subject: [PATCH 2/5] Add the foreign document test to the CI.

---
 .../pipeline-foreign-document-test.yaml       | 203 ++++++++++++++++++
 .lighthouse/triggers.yaml                     |   7 +
 2 files changed, 210 insertions(+)
 create mode 100644 .lighthouse/pipeline-foreign-document-test.yaml

diff --git a/.lighthouse/pipeline-foreign-document-test.yaml b/.lighthouse/pipeline-foreign-document-test.yaml
new file mode 100644
index 0000000..e71302b
--- /dev/null
+++ b/.lighthouse/pipeline-foreign-document-test.yaml
@@ -0,0 +1,203 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: rust-foreign-document-test
+spec:
+  pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
+    params:
+      - name: image-name
+        description: The name for the built image
+        type: string
+      - name: path-to-image-context
+        description: The path to the build context
+        type: string
+      - name: path-to-dockerfile
+        description: The path to the Dockerfile
+        type: string
+    tasks:
+      - name: do-stuff
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.18
+            name: ""
+            resources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: /workspace/source
+          steps:
+            - image: alpine:3.18
+              name: do-stuff-step
+              script: |
+                #!/usr/bin/env sh
+                echo "hello world"
+      - name: report-pending
+        taskRef:
+          name: gitea-set-status
+        runAfter:
+          - fetch-repository
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          name: git-clone
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+      - name: build-image
+        taskRef:
+          name: kaniko
+        params:
+          - name: IMAGE
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+          - name: CONTEXT
+            value: $(params.path-to-image-context)
+          - name: DOCKERFILE
+            value: $(params.path-to-dockerfile)
+          - name: BUILDER_IMAGE
+            value: "gcr.io/kaniko-project/executor:v1.12.1"
+          - name: EXTRA_ARGS
+            value:
+              - --target=foreign-document-test
+              - --cache=true
+              - --cache-copy-layers
+              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - --use-new-run # Should result in a speed-up
+              - --reproducible # To remove timestamps so layer caching works.
+              - --snapshot-mode=redo
+              - --skip-unused-stages=true
+              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: dockerconfig
+            workspace: docker-credentials
+        runAfter:
+          - fetch-repository
+      - name: run-image
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        runAfter:
+          - build-image
+        params:
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: cargo-cache-autoclean
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: command
+            value: [cargo, cache, --autoclean]
+          - name: args
+            value: []
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    workspaces:
+      - name: git-source
+      - name: docker-credentials
+      - name: cargo-cache
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: rust-source
+    - name: cargo-cache
+      persistentVolumeClaim:
+        claimName: organic-cargo-cache-test-foreign-document
+    - name: docker-credentials
+      secret:
+        secretName: harbor-plain
+  serviceAccountName: build-bot
+  params:
+    - name: image-name
+      value: "harbor.fizz.buzz/private/organic-test-foreign-document"
+    - name: path-to-image-context
+      value: docker/organic_test/
+    - name: path-to-dockerfile
+      value: docker/organic_test/Dockerfile
diff --git a/.lighthouse/triggers.yaml b/.lighthouse/triggers.yaml
index 8ddddee..521b20c 100644
--- a/.lighthouse/triggers.yaml
+++ b/.lighthouse/triggers.yaml
@@ -16,6 +16,13 @@ spec:
       skip_branches:
         # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
         - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+    - name: rust-foreign-document-test
+      source: "pipeline-foreign-document-test.yaml"
+      # Override https-based url from lighthouse events.
+      clone_uri: "git@code.fizz.buzz:talexander/organic.git"
+      skip_branches:
+        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
+        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
     - name: rust-build
       source: "pipeline-rust-build.yaml"
       # Override https-based url from lighthouse events.

From 51c4e2b62a97554410209631bbf90e83d1ba6925 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 6 Sep 2023 16:03:59 -0400
Subject: [PATCH 3/5] Only use local folder for docker context.

---
 docker/cargo_fmt/Makefile      | 2 +-
 docker/organic_build/Makefile  | 2 +-
 docker/organic_test/Dockerfile | 2 +-
 docker/organic_test/Makefile   | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docker/cargo_fmt/Makefile b/docker/cargo_fmt/Makefile
index 8d50935..fff467b 100644
--- a/docker/cargo_fmt/Makefile
+++ b/docker/cargo_fmt/Makefile
@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:
diff --git a/docker/organic_build/Makefile b/docker/organic_build/Makefile
index c933de6..ab76282 100644
--- a/docker/organic_build/Makefile
+++ b/docker/organic_build/Makefile
@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:
diff --git a/docker/organic_test/Dockerfile b/docker/organic_test/Dockerfile
index fab4a46..927b8ed 100644
--- a/docker/organic_test/Dockerfile
+++ b/docker/organic_test/Dockerfile
@@ -43,6 +43,6 @@ FROM tester as foreign-document-test
 RUN apk add --no-cache bash
 RUN mkdir /foreign_documents
 COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
-COPY docker/organic_test/foreign_document_test_entrypoint.sh /entrypoint.sh
+COPY foreign_document_test_entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile
index e0f82ab..61280bb 100644
--- a/docker/organic_test/Makefile
+++ b/docker/organic_test/Makefile
@@ -6,11 +6,11 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester .
 
 .PHONY: build_foreign_document_test
 build_foreign_document_test:
-	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test ../../
+	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test .
 
 .PHONY: push
 push:

From b89607fc8b76527f17dddb3173e544e95be3f03c Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 6 Sep 2023 16:12:41 -0400
Subject: [PATCH 4/5] Handle CARGO_TARGET_DIR not being set.

---
 docker/organic_test/foreign_document_test_entrypoint.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/organic_test/foreign_document_test_entrypoint.sh b/docker/organic_test/foreign_document_test_entrypoint.sh
index 29a7363..8f0c90f 100644
--- a/docker/organic_test/foreign_document_test_entrypoint.sh
+++ b/docker/organic_test/foreign_document_test_entrypoint.sh
@@ -18,6 +18,9 @@ function die {
 
 function main {
     cargo build --no-default-features --features compare --profile release-lto
+    if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
+        CARGO_TARGET_DIR=$(realpath target/)
+    fi
     PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
 
     run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"

From e3d755317d74e4c565e6dca4a6119a2bd2c8a933 Mon Sep 17 00:00:00 2001
From: Tom Alexander <tom@fizz.buzz>
Date: Wed, 6 Sep 2023 16:17:42 -0400
Subject: [PATCH 5/5] Add a top-level makefile target for running the foreign
 document test.

---
 Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Makefile b/Makefile
index d8a2b07..537f634 100644
--- a/Makefile
+++ b/Makefile
@@ -42,6 +42,10 @@ dockertest:
 > $(MAKE) -C docker/organic_test
 > docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
+.PHONY: foreign_document_test
+foreign_document_test:
+> $(MAKE) -C docker/organic_test run_foreign_document_test
+
 .PHONY: dockerclean
 dockerclean:
 # Delete volumes created for running the tests in docker. This does not touch anything related to the jaeger docker container.