Merge branch 'foreign_document_test'

.lighthouse/pipeline-foreign-document-test.yaml

@@ -0,0 +1,203 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: rust-foreign-document-test
+spec:
+  pipelineSpec:
+    timeouts:
+      pipeline: "2h0m0s"
+      tasks: "1h0m40s"
+      finally: "0h30m0s"
+    params:
+      - name: image-name
+        description: The name for the built image
+        type: string
+      - name: path-to-image-context
+        description: The path to the build context
+        type: string
+      - name: path-to-dockerfile
+        description: The path to the Dockerfile
+        type: string
+    tasks:
+      - name: do-stuff
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.18
+            name: ""
+            resources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: /workspace/source
+          steps:
+            - image: alpine:3.18
+              name: do-stuff-step
+              script: |
+                #!/usr/bin/env sh
+                echo "hello world"
+      - name: report-pending
+        taskRef:
+          name: gitea-set-status
+        runAfter:
+          - fetch-repository
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          name: git-clone
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+      - name: build-image
+        taskRef:
+          name: kaniko
+        params:
+          - name: IMAGE
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+          - name: CONTEXT
+            value: $(params.path-to-image-context)
+          - name: DOCKERFILE
+            value: $(params.path-to-dockerfile)
+          - name: BUILDER_IMAGE
+            value: "gcr.io/kaniko-project/executor:v1.12.1"
+          - name: EXTRA_ARGS
+            value:
+              - --target=foreign-document-test
+              - --cache=true
+              - --cache-copy-layers
+              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - --use-new-run # Should result in a speed-up
+              - --reproducible # To remove timestamps so layer caching works.
+              - --snapshot-mode=redo
+              - --skip-unused-stages=true
+              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: dockerconfig
+            workspace: docker-credentials
+        runAfter:
+          - fetch-repository
+      - name: run-image
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        runAfter:
+          - build-image
+        params:
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          name: gitea-set-status
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: cargo-cache-autoclean
+        taskRef:
+          name: run-docker-image
+        workspaces:
+          - name: source
+            workspace: git-source
+          - name: cargo-cache
+            workspace: cargo-cache
+        params:
+          - name: command
+            value: [cargo, cache, --autoclean]
+          - name: args
+            value: []
+          - name: docker-image
+            value: "$(params.image-name):$(tasks.fetch-repository.results.commit)"
+    workspaces:
+      - name: git-source
+      - name: docker-credentials
+      - name: cargo-cache
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: rust-source
+    - name: cargo-cache
+      persistentVolumeClaim:
+        claimName: organic-cargo-cache-test-foreign-document
+    - name: docker-credentials
+      secret:
+        secretName: harbor-plain
+  serviceAccountName: build-bot
+  params:
+    - name: image-name
+      value: "harbor.fizz.buzz/private/organic-test-foreign-document"
+    - name: path-to-image-context
+      value: docker/organic_test/
+    - name: path-to-dockerfile
+      value: docker/organic_test/Dockerfile

.lighthouse/pipeline-rust-test.yaml

@@ -83,6 +83,7 @@ spec:
             value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
+              - --target=tester
               - --cache=true
               - --cache-copy-layers
               - --cache-repo=harbor.fizz.buzz/kanikocache/cache

.lighthouse/triggers.yaml

@@ -16,6 +16,13 @@ spec:
       skip_branches:
         # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
         - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+    - name: rust-foreign-document-test
+      source: "pipeline-foreign-document-test.yaml"
+      # Override https-based url from lighthouse events.
+      clone_uri: "git@code.fizz.buzz:talexander/organic.git"
+      skip_branches:
+        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
+        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
     - name: rust-build
       source: "pipeline-rust-build.yaml"
       # Override https-based url from lighthouse events.

Makefile

@@ -42,6 +42,10 @@ dockertest:
 > $(MAKE) -C docker/organic_test
 > docker run --init --rm -i -t --read-only -v "$$(readlink -f ./):/source:ro" --mount type=tmpfs,destination=/tmp --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test --no-default-features --features compare --no-fail-fast --lib --test test_loader -- --test-threads $(TESTJOBS)
 
+.PHONY: foreign_document_test
+foreign_document_test:
+> $(MAKE) -C docker/organic_test run_foreign_document_test
+
 .PHONY: dockerclean
 dockerclean:
 # Delete volumes created for running the tests in docker. This does not touch anything related to the jaeger docker container.

docker/cargo_fmt/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:

docker/organic_build/Makefile

@@ -6,7 +6,7 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile .
 
 .PHONY: push
 push:

docker/organic_test/Dockerfile

@@ -25,7 +25,7 @@ RUN make compile
 RUN make DESTDIR="/root/dist" install
 
 
-FROM rustlang/rust:nightly-alpine3.17
+FROM rustlang/rust:nightly-alpine3.17 AS tester
 ENV LANG=en_US.UTF-8
 RUN apk add --no-cache musl-dev ncurses gnutls
 RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache
@@ -33,3 +33,16 @@ COPY --from=build-emacs /root/dist/ /
 COPY --from=build-org-mode /root/dist/ /
 
 ENTRYPOINT ["cargo", "test"]
+
+
+FROM build as foreign-document-gather
+RUN mkdir /foreign_documents
+
+
+FROM tester as foreign-document-test
+RUN apk add --no-cache bash
+RUN mkdir /foreign_documents
+COPY --from=build-org-mode /root/org-mode/doc /foreign_documents/org-mode
+COPY foreign_document_test_entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

docker/organic_test/Makefile

@@ -6,7 +6,11 @@ all: build push
 
 .PHONY: build
 build:
-	docker build -t $(IMAGE_NAME) -f Dockerfile ../../
+	docker build -t $(IMAGE_NAME) -f Dockerfile --target tester .
+
+.PHONY: build_foreign_document_test
+build_foreign_document_test:
+	docker build -t $(IMAGE_NAME)-foreign-document -f Dockerfile --target foreign-document-test .
 
 .PHONY: push
 push:
@@ -34,3 +38,7 @@ run: build
 .PHONY: shell
 shell: build
 	docker run --rm -i -t --entrypoint /bin/sh --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)
+
+.PHONY: run_foreign_document_test
+run_foreign_document_test: build_foreign_document_test
+	docker run --rm --init --read-only --mount type=tmpfs,destination=/tmp -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME)-foreign-document

docker/organic_test/foreign_document_test_entrypoint.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+#
+# Run the Organic compare script against a series of documents sourced from exterior places.
+set -euo pipefail
+IFS=$'\n\t'
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+function log {
+    (>&2 echo "${@}")
+}
+
+function die {
+    local status_code="$1"
+    shift
+    (>&2 echo "${@}")
+    exit "$status_code"
+}
+
+function main {
+    cargo build --no-default-features --features compare --profile release-lto
+    if [ "${CARGO_TARGET_DIR:-}" = "" ]; then
+        CARGO_TARGET_DIR=$(realpath target/)
+    fi
+    PARSE="${CARGO_TARGET_DIR}/release-lto/parse"
+
+    run_compare "org-mode/org-guide.org" "/foreign_documents/org-mode/org-guide.org"
+    run_compare "org-mode/org-manual.org" "/foreign_documents/org-mode/org-manual.org"
+}
+
+function run_compare {
+    local name="$1"
+    local target_document="$2"
+    set +e
+    $PARSE "$target_document" &> /dev/null
+    local status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        echo "$(green_text "GOOD")  $name"
+    else
+        echo "$(red_text "FAIL")  $name"
+        return 1
+    fi
+}
+
+function green_text {
+    (IFS=' '; printf '\x1b[38;2;0;255;0m%s\x1b[0m' "${*}")
+}
+
+function red_text {
+    (IFS=' '; printf '\x1b[38;2;255;0;0m%s\x1b[0m' "${*}")
+}
+
+function yellow_text {
+    (IFS=' '; printf '\x1b[38;2;255;255;0m%s\x1b[0m' "${*}")
+}
+
+main "${@}"