From 378b6bb391dd23da0a7b149ff41ea7cce8175252 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 27 Aug 2023 17:29:55 -0400 Subject: [PATCH 1/5] Update the run targets for the Makefiles for the docker containers. This was previously using the standard docker makefile I use as a starting point for all of my docker makefiles. Now it will properly mount the source directory. --- .lighthouse/pipeline-rust-build.yaml | 16 ---------------- .lighthouse/pipeline-rust-test.yaml | 16 +--------------- .lighthouse/pipeline-rustfmt.yaml | 16 ---------------- docker/cargo_fmt/Makefile | 5 +++-- docker/organic_build/Dockerfile | 2 ++ docker/organic_build/Makefile | 5 +++-- docker/organic_test/Dockerfile | 2 ++ docker/organic_test/Makefile | 4 ++-- 8 files changed, 13 insertions(+), 53 deletions(-) diff --git a/.lighthouse/pipeline-rust-build.yaml b/.lighthouse/pipeline-rust-build.yaml index b182481..036a1b7 100644 --- a/.lighthouse/pipeline-rust-build.yaml +++ b/.lighthouse/pipeline-rust-build.yaml @@ -14,10 +14,6 @@ spec: - name: path-to-dockerfile description: The path to the Dockerfile type: string - - name: command - type: array - description: Command to run. - default: [] tasks: - name: report-pending taskRef: @@ -92,8 +88,6 @@ spec: runAfter: - build-image params: - - name: command - value: ["$(params.command[*])"] - name: args value: ["--no-default-features"] - name: docker-image @@ -109,8 +103,6 @@ spec: runAfter: - run-image-none params: - - name: command - value: ["$(params.command[*])"] - name: args value: ["--no-default-features", "--features", "tracing"] - name: docker-image @@ -126,8 +118,6 @@ spec: runAfter: - run-image-tracing params: - - name: command - value: ["$(params.command[*])"] - name: args value: ["--no-default-features", "--features", "compare"] - name: docker-image @@ -143,8 +133,6 @@ spec: runAfter: - run-image-compare params: - - name: command - value: ["$(params.command[*])"] - name: args value: [] - name: docker-image @@ -160,8 +148,6 @@ spec: runAfter: - run-image-default params: - - name: command - value: ["$(params.command[*])"] - name: args value: ["--no-default-features", "--features", "tracing,compare"] - name: docker-image @@ -256,5 +242,3 @@ spec: value: docker/organic_build/ - name: path-to-dockerfile value: docker/organic_build/Dockerfile - - name: command - value: [cargo, build] diff --git a/.lighthouse/pipeline-rust-test.yaml b/.lighthouse/pipeline-rust-test.yaml index 12fe886..0f9b60c 100644 --- a/.lighthouse/pipeline-rust-test.yaml +++ b/.lighthouse/pipeline-rust-test.yaml @@ -18,14 +18,6 @@ spec: - name: path-to-dockerfile description: The path to the Dockerfile type: string - - name: command - type: array - description: Command to run. - default: [] - - name: args - type: array - description: Arguments passed to command. - default: [] tasks: - name: do-stuff taskSpec: @@ -117,10 +109,8 @@ spec: runAfter: - build-image params: - - name: command - value: ["$(params.command[*])"] - name: args - value: ["$(params.args[*])"] + value: [--no-fail-fast, --lib, --test, test_loader] - name: docker-image value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" finally: @@ -212,7 +202,3 @@ spec: value: docker/organic_test/ - name: path-to-dockerfile value: docker/organic_test/Dockerfile - - name: command - value: [cargo, test] - - name: args - value: [--lib, --test, test_loader] diff --git a/.lighthouse/pipeline-rustfmt.yaml b/.lighthouse/pipeline-rustfmt.yaml index aa26f23..9e6d656 100644 --- a/.lighthouse/pipeline-rustfmt.yaml +++ b/.lighthouse/pipeline-rustfmt.yaml @@ -14,14 +14,6 @@ spec: - name: path-to-dockerfile description: The path to the Dockerfile type: string - - name: rustfmt-command - type: array - description: Command to run rustfmt. - default: [] - - name: rustfmt-args - type: array - description: Arguments passed to rustfmt. - default: [] - name: GIT_USER_NAME description: The username for git type: string @@ -119,10 +111,6 @@ spec: runAfter: - build-image params: - - name: command - value: ["$(params.rustfmt-command[*])"] - - name: args - value: ["$(params.rustfmt-args[*])"] - name: docker-image value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" - name: cargo-fix @@ -240,7 +228,3 @@ spec: value: docker/cargo_fmt/ - name: path-to-dockerfile value: docker/cargo_fmt/Dockerfile - - name: command - value: [cargo, fmt] - - name: args - value: [] diff --git a/docker/cargo_fmt/Makefile b/docker/cargo_fmt/Makefile index 121b50e..aa6063c 100644 --- a/docker/cargo_fmt/Makefile +++ b/docker/cargo_fmt/Makefile @@ -26,10 +26,11 @@ else @echo "REMOTE_REPO not defined, not removing from remote repo." endif +# NOTE: This target will write to folders underneath the git-root .PHONY: run run: - docker run --rm -i -t $(IMAGE_NAME) + docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME) .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) diff --git a/docker/organic_build/Dockerfile b/docker/organic_build/Dockerfile index 72de61c..fdf38f4 100644 --- a/docker/organic_build/Dockerfile +++ b/docker/organic_build/Dockerfile @@ -2,3 +2,5 @@ FROM rustlang/rust:nightly-alpine3.17 RUN apk add --no-cache musl-dev RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache + +ENTRYPOINT ["cargo", "build"] diff --git a/docker/organic_build/Makefile b/docker/organic_build/Makefile index 950cc71..2999046 100644 --- a/docker/organic_build/Makefile +++ b/docker/organic_build/Makefile @@ -26,10 +26,11 @@ else @echo "REMOTE_REPO not defined, not removing from remote repo." endif +# NOTE: This target will write to folders underneath the git-root .PHONY: run run: - docker run --rm -i -t $(IMAGE_NAME) + docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME) .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) diff --git a/docker/organic_test/Dockerfile b/docker/organic_test/Dockerfile index 7027865..6efcb46 100644 --- a/docker/organic_test/Dockerfile +++ b/docker/organic_test/Dockerfile @@ -30,3 +30,5 @@ RUN apk add --no-cache musl-dev ncurses gnutls RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache COPY --from=build-emacs /root/dist/ / COPY --from=build-org-mode /root/dist/ / + +ENTRYPOINT ["cargo", "test"] diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile index 2de6ba1..39d566a 100644 --- a/docker/organic_test/Makefile +++ b/docker/organic_test/Makefile @@ -28,8 +28,8 @@ endif .PHONY: run run: - docker run --rm -i -t $(IMAGE_NAME) + docker run --rm --init -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-fail-fast --lib --test test_loader .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/bash $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) From c17de8ef5e74cd34ad765054d723c27b9f47f1b7 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 27 Aug 2023 17:50:33 -0400 Subject: [PATCH 2/5] Set up the same mounts when running make shell. --- docker/cargo_fmt/Makefile | 2 +- docker/organic_build/Makefile | 2 +- docker/organic_test/Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cargo_fmt/Makefile b/docker/cargo_fmt/Makefile index aa6063c..c9d958d 100644 --- a/docker/cargo_fmt/Makefile +++ b/docker/cargo_fmt/Makefile @@ -33,4 +33,4 @@ run: .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME) diff --git a/docker/organic_build/Makefile b/docker/organic_build/Makefile index 2999046..709b062 100644 --- a/docker/organic_build/Makefile +++ b/docker/organic_build/Makefile @@ -33,4 +33,4 @@ run: .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME) diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile index 39d566a..28649d2 100644 --- a/docker/organic_test/Makefile +++ b/docker/organic_test/Makefile @@ -32,4 +32,4 @@ run: .PHONY: shell shell: - docker run --rm -i -t --entrypoint /bin/sh $(IMAGE_NAME) + docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) From 75dfc7f812a812a3a37b0cc9673ec000b72e9bf6 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 27 Aug 2023 17:51:57 -0400 Subject: [PATCH 3/5] Depend on build when using the docker images. --- docker/cargo_fmt/Makefile | 4 ++-- docker/organic_build/Makefile | 4 ++-- docker/organic_test/Makefile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/cargo_fmt/Makefile b/docker/cargo_fmt/Makefile index c9d958d..68bb409 100644 --- a/docker/cargo_fmt/Makefile +++ b/docker/cargo_fmt/Makefile @@ -28,9 +28,9 @@ endif # NOTE: This target will write to folders underneath the git-root .PHONY: run -run: +run: build docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME) .PHONY: shell -shell: +shell: build docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source $(IMAGE_NAME) diff --git a/docker/organic_build/Makefile b/docker/organic_build/Makefile index 709b062..9f2b7ba 100644 --- a/docker/organic_build/Makefile +++ b/docker/organic_build/Makefile @@ -28,9 +28,9 @@ endif # NOTE: This target will write to folders underneath the git-root .PHONY: run -run: +run: build docker run --rm --init -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME) .PHONY: shell -shell: +shell: build docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry $(IMAGE_NAME) diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile index 28649d2..371b661 100644 --- a/docker/organic_test/Makefile +++ b/docker/organic_test/Makefile @@ -27,9 +27,9 @@ else endif .PHONY: run -run: +run: build docker run --rm --init -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) --no-fail-fast --lib --test test_loader .PHONY: shell -shell: +shell: build docker run --rm -i -t --entrypoint /bin/sh -v "$$(readlink -f ../../):/source:ro" --workdir=/source --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target $(IMAGE_NAME) From ad389f07760c6556b735a5fb7af772f7fa58980f Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 27 Aug 2023 17:53:51 -0400 Subject: [PATCH 4/5] Remove volumes in the clean step. --- docker/organic_build/Makefile | 1 + docker/organic_test/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/organic_build/Makefile b/docker/organic_build/Makefile index 9f2b7ba..80bcb9c 100644 --- a/docker/organic_build/Makefile +++ b/docker/organic_build/Makefile @@ -25,6 +25,7 @@ ifdef REMOTE_REPO else @echo "REMOTE_REPO not defined, not removing from remote repo." endif + docker volume rm cargo-cache # NOTE: This target will write to folders underneath the git-root .PHONY: run diff --git a/docker/organic_test/Makefile b/docker/organic_test/Makefile index 371b661..d2c1aad 100644 --- a/docker/organic_test/Makefile +++ b/docker/organic_test/Makefile @@ -25,6 +25,7 @@ ifdef REMOTE_REPO else @echo "REMOTE_REPO not defined, not removing from remote repo." endif + docker volume rm rust-cache cargo-cache .PHONY: run run: build From b124317f30b0d76533de8ef7abca0540b9134e31 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 27 Aug 2023 18:03:37 -0400 Subject: [PATCH 5/5] Fix up scripts to handle the changes to the docker containers. --- scripts/run_docker_compare.bash | 2 +- scripts/run_docker_integration_test.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/run_docker_compare.bash b/scripts/run_docker_compare.bash index 127f8f1..7e56e8f 100755 --- a/scripts/run_docker_compare.bash +++ b/scripts/run_docker_compare.bash @@ -46,7 +46,7 @@ function launch_container { additional_flags+=(--env RUST_BACKTRACE=full) fi - docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test "${additional_args[@]}" + docker run "${additional_flags[@]}" --init --rm -i -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test "${additional_args[@]}" } main "${@}" diff --git a/scripts/run_docker_integration_test.bash b/scripts/run_docker_integration_test.bash index 163aefe..3539dc1 100755 --- a/scripts/run_docker_integration_test.bash +++ b/scripts/run_docker_integration_test.bash @@ -56,7 +56,7 @@ cargo test --no-fail-fast --lib --test test_loader "$test" -- --show-output EOF ) - docker run "${additional_flags[@]}" --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source organic-test sh -c "$init_script" + docker run "${additional_flags[@]}" --init --rm -v "$($REALPATH ./):/source:ro" --mount source=cargo-cache,target=/usr/local/cargo/registry --mount source=rust-cache,target=/target --env CARGO_TARGET_DIR=/target -w /source --entrypoint "" organic-test sh -c "$init_script" }