From 691f7b6566f43a8a35e7771485282addf74cec49 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 20 Oct 2024 23:38:57 -0400 Subject: [PATCH] Update dockerfiles to take advantage of BuildKit. --- .webhook_bridge/pipeline-build-hash.yaml | 68 ++++++++++++++++------ .webhook_bridge/pipeline-format.yaml | 71 +++++++++++++++++------ .webhook_bridge/pipeline-rust-clippy.yaml | 71 +++++++++++++++++------ .webhook_bridge/pipeline-rust-test.yaml | 71 +++++++++++++++++------ docker/poudboot/Dockerfile | 13 +++-- docker/poudboot_development/Dockerfile | 7 ++- 6 files changed, 219 insertions(+), 82 deletions(-) diff --git a/.webhook_bridge/pipeline-build-hash.yaml b/.webhook_bridge/pipeline-build-hash.yaml index d39f762..eaf03dd 100644 --- a/.webhook_bridge/pipeline-build-hash.yaml +++ b/.webhook_bridge/pipeline-build-hash.yaml @@ -69,42 +69,74 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" + - name: get-git-commit-time + taskSpec: + metadata: {} + stepTemplate: + image: alpine:3.20 + computeResources: + requests: + cpu: 10m + memory: 600Mi + workingDir: "$(workspaces.repo.path)" + results: + - name: unix-time + description: The time of the git commit in unix timestamp format. + steps: + - image: alpine/git:v2.34.2 + name: detect-tag-step + script: | + #!/usr/bin/env sh + set -euo pipefail + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: build-image taskRef: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 7ee31a185243ee6da13dcd26a592c585b64c80e5 - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + - name: OUTPUT + value: >- + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.fetch-repository.results.commit)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: $(params.path-to-image-context) - name: DOCKERFILE value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - - "--destination=$(params.image-name)" # Also write the :latest image - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - "--opt" + - "target=$(params.target-name)" + - --import-cache + - "type=registry,ref=$(params.image-name):buildcache" + - --export-cache + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source - name: dockerconfig workspace: docker-credentials + runAfter: + - fetch-repository finally: - name: report-success when: @@ -189,4 +221,4 @@ spec: - name: path-to-image-context value: . - name: path-to-dockerfile - value: docker/poudboot/Dockerfile + value: docker/poudboot/ diff --git a/.webhook_bridge/pipeline-format.yaml b/.webhook_bridge/pipeline-format.yaml index ef67b10..44ea2d9 100644 --- a/.webhook_bridge/pipeline-format.yaml +++ b/.webhook_bridge/pipeline-format.yaml @@ -69,41 +69,74 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" + - name: get-git-commit-time + taskSpec: + metadata: {} + stepTemplate: + image: alpine:3.20 + computeResources: + requests: + cpu: 10m + memory: 600Mi + workingDir: "$(workspaces.repo.path)" + results: + - name: unix-time + description: The time of the git commit in unix timestamp format. + steps: + - image: alpine/git:v2.34.2 + name: detect-tag-step + script: | + #!/usr/bin/env sh + set -euo pipefail + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: build-image taskRef: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 7ee31a185243ee6da13dcd26a592c585b64c80e5 - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + - name: OUTPUT + value: >- + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.fetch-repository.results.commit)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: $(params.path-to-image-context) - name: DOCKERFILE value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - "--opt" + - "target=$(params.target-name)" + - --import-cache + - "type=registry,ref=$(params.image-name):buildcache" + - --export-cache + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source - name: dockerconfig workspace: docker-credentials + runAfter: + - fetch-repository - name: run-cargo-fmt taskSpec: metadata: {} @@ -143,7 +176,7 @@ spec: - build-image params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" - name: commit-changes taskRef: resolver: git @@ -268,7 +301,7 @@ spec: workspace: cargo-cache params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" workspaces: - name: git-source - name: docker-credentials @@ -298,4 +331,4 @@ spec: - name: path-to-image-context value: docker/poudboot_development/ - name: path-to-dockerfile - value: docker/poudboot_development/Dockerfile + value: docker/poudboot_development/ diff --git a/.webhook_bridge/pipeline-rust-clippy.yaml b/.webhook_bridge/pipeline-rust-clippy.yaml index 6833ebc..7f7e9a3 100644 --- a/.webhook_bridge/pipeline-rust-clippy.yaml +++ b/.webhook_bridge/pipeline-rust-clippy.yaml @@ -69,41 +69,74 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" + - name: get-git-commit-time + taskSpec: + metadata: {} + stepTemplate: + image: alpine:3.20 + computeResources: + requests: + cpu: 10m + memory: 600Mi + workingDir: "$(workspaces.repo.path)" + results: + - name: unix-time + description: The time of the git commit in unix timestamp format. + steps: + - image: alpine/git:v2.34.2 + name: detect-tag-step + script: | + #!/usr/bin/env sh + set -euo pipefail + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: build-image taskRef: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 7ee31a185243ee6da13dcd26a592c585b64c80e5 - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + - name: OUTPUT + value: >- + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.fetch-repository.results.commit)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: $(params.path-to-image-context) - name: DOCKERFILE value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - "--opt" + - "target=$(params.target-name)" + - --import-cache + - "type=registry,ref=$(params.image-name):buildcache" + - --export-cache + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source - name: dockerconfig workspace: docker-credentials + runAfter: + - fetch-repository - name: run-cargo-clippy taskSpec: metadata: {} @@ -153,7 +186,7 @@ spec: - build-image params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" finally: - name: report-success when: @@ -247,7 +280,7 @@ spec: workspace: cargo-cache params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" workspaces: - name: git-source - name: docker-credentials @@ -277,4 +310,4 @@ spec: - name: path-to-image-context value: docker/poudboot_development/ - name: path-to-dockerfile - value: docker/poudboot_development/Dockerfile + value: docker/poudboot_development/ diff --git a/.webhook_bridge/pipeline-rust-test.yaml b/.webhook_bridge/pipeline-rust-test.yaml index 9d6b7f7..3b834a1 100644 --- a/.webhook_bridge/pipeline-rust-test.yaml +++ b/.webhook_bridge/pipeline-rust-test.yaml @@ -69,41 +69,74 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" + - name: get-git-commit-time + taskSpec: + metadata: {} + stepTemplate: + image: alpine:3.20 + computeResources: + requests: + cpu: 10m + memory: 600Mi + workingDir: "$(workspaces.repo.path)" + results: + - name: unix-time + description: The time of the git commit in unix timestamp format. + steps: + - image: alpine/git:v2.34.2 + name: detect-tag-step + script: | + #!/usr/bin/env sh + set -euo pipefail + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: build-image taskRef: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 7ee31a185243ee6da13dcd26a592c585b64c80e5 - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + - name: OUTPUT + value: >- + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.fetch-repository.results.commit)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: $(params.path-to-image-context) - name: DOCKERFILE value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - "--opt" + - "target=$(params.target-name)" + - --import-cache + - "type=registry,ref=$(params.image-name):buildcache" + - --export-cache + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source - name: dockerconfig workspace: docker-credentials + runAfter: + - fetch-repository - name: run-cargo-test taskSpec: metadata: {} @@ -143,7 +176,7 @@ spec: - build-image params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" finally: - name: report-success when: @@ -237,7 +270,7 @@ spec: workspace: cargo-cache params: - name: docker-image - value: "$(params.image-name):$(tasks.fetch-repository.results.commit)" + value: "$(tasks.build-image.results.IMAGE_URL[1])" workspaces: - name: git-source - name: docker-credentials @@ -267,4 +300,4 @@ spec: - name: path-to-image-context value: docker/poudboot_development/ - name: path-to-dockerfile - value: docker/poudboot_development/Dockerfile + value: docker/poudboot_development/ diff --git a/docker/poudboot/Dockerfile b/docker/poudboot/Dockerfile index f5c2462..1b30c01 100644 --- a/docker/poudboot/Dockerfile +++ b/docker/poudboot/Dockerfile @@ -1,13 +1,16 @@ -FROM rustlang/rust:nightly-alpine3.20 AS builder +# syntax=docker/dockerfile:1 +ARG ALPINE_VERSION="3.20" + +FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS builder RUN apk add --no-cache musl-dev RUN mkdir /root/poudboot WORKDIR /root/poudboot -COPY . . +COPY --link . . # TODO: Add static build, which currently errors due to proc_macro. RUSTFLAGS="-C target-feature=+crt-static" -RUN CARGO_TARGET_DIR=/target cargo build --profile release-lto +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked CARGO_TARGET_DIR=/target cargo build --profile release-lto -FROM alpine:3.20 AS runner +FROM alpine:$ALPINE_VERSION AS runner -COPY --from=builder /target/release-lto/poudboot /usr/bin/ +COPY --link --from=builder /target/release-lto/poudboot /usr/bin/ diff --git a/docker/poudboot_development/Dockerfile b/docker/poudboot_development/Dockerfile index da54256..2fc0bbb 100644 --- a/docker/poudboot_development/Dockerfile +++ b/docker/poudboot_development/Dockerfile @@ -1,6 +1,9 @@ -FROM rustlang/rust:nightly-alpine3.20 AS builder +# syntax=docker/dockerfile:1 +ARG ALPINE_VERSION="3.20" + +FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS builder RUN apk add --no-cache musl-dev -RUN cargo install --locked --no-default-features --features ci-autoclean cargo-cache +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked cargo install --locked --no-default-features --features ci-autoclean cargo-cache RUN rustup component add rustfmt RUN rustup component add clippy