From bcf821dc58d2ecc3a1c99497caf9c26f4cbbd3ad Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 19 Oct 2024 17:56:45 -0400 Subject: [PATCH 1/8] Try try_files. --- docker/server/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/server/nginx.conf b/docker/server/nginx.conf index 61d2562..8281478 100644 --- a/docker/server/nginx.conf +++ b/docker/server/nginx.conf @@ -31,6 +31,8 @@ http { root /srv/http/public; location / { + try_files $uri $uri/ =404; + index index.html index.htm; if (-d $request_filename) { rewrite [^/]$ $http_x_forwarded_proto://$http_host$uri/ redirect; From 0e097989826c8ed98a06bdab5c520539b3a21819 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 19 Oct 2024 18:08:37 -0400 Subject: [PATCH 2/8] Try autoindex. --- docker/server/nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/server/nginx.conf b/docker/server/nginx.conf index 8281478..229bb4a 100644 --- a/docker/server/nginx.conf +++ b/docker/server/nginx.conf @@ -32,6 +32,7 @@ http { location / { try_files $uri $uri/ =404; + autoindex on; index index.html index.htm; if (-d $request_filename) { From 79c5c7e589d1ae7d1ce55ebe2b1e5ac4bbcaf4fd Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 19 Oct 2024 18:35:06 -0400 Subject: [PATCH 3/8] Try the latest kaniko image. --- .webhook_bridge/pipeline-build-homepage-staging.yaml | 4 ++-- .webhook_bridge/pipeline-build-homepage.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.webhook_bridge/pipeline-build-homepage-staging.yaml b/.webhook_bridge/pipeline-build-homepage-staging.yaml index 604324b..96f1f06 100644 --- a/.webhook_bridge/pipeline-build-homepage-staging.yaml +++ b/.webhook_bridge/pipeline-build-homepage-staging.yaml @@ -197,7 +197,7 @@ spec: - name: DOCKERFILE value: docker/organic/Dockerfile - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.12.1" + value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - "--destination=harbor.fizz.buzz/private/homepage-build-organic" # Also write the :latest image @@ -285,7 +285,7 @@ spec: - name: DOCKERFILE value: $(params.path-to-dockerfile) - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.12.1" + value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - "--destination=$(params.image-name)" # Also write the :latest image diff --git a/.webhook_bridge/pipeline-build-homepage.yaml b/.webhook_bridge/pipeline-build-homepage.yaml index 1821e3a..69df7d2 100644 --- a/.webhook_bridge/pipeline-build-homepage.yaml +++ b/.webhook_bridge/pipeline-build-homepage.yaml @@ -229,7 +229,7 @@ spec: - name: DOCKERFILE value: docker/organic/Dockerfile - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.12.1" + value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - "--destination=harbor.fizz.buzz/private/homepage-build-organic" # Also write the :latest image @@ -317,7 +317,7 @@ spec: - name: DOCKERFILE value: $(params.path-to-dockerfile) - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.12.1" + value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - "--destination=$(params.image-name)" # Also write the :latest image From 09e8c700d2b51bde3debd4ecff8c0e1bfb1cb39a Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sat, 19 Oct 2024 23:15:35 -0400 Subject: [PATCH 4/8] Switch to a new Tekton task I've written to use buildkit instead of kaniko to build docker images. --- .../pipeline-build-homepage-staging.yaml | 66 +++++++++++++------ 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/.webhook_bridge/pipeline-build-homepage-staging.yaml b/.webhook_bridge/pipeline-build-homepage-staging.yaml index 96f1f06..c08dd8b 100644 --- a/.webhook_bridge/pipeline-build-homepage-staging.yaml +++ b/.webhook_bridge/pipeline-build-homepage-staging.yaml @@ -44,6 +44,31 @@ spec: #!/usr/bin/env sh set -euo pipefail echo -n "$(date +%s)" | tee $(results.unix-time.path) + - name: get-git-commit-time + taskSpec: + metadata: {} + stepTemplate: + image: alpine:3.20 + computeResources: + requests: + cpu: 10m + memory: 600Mi + workingDir: "$(workspaces.repo.path)" + results: + - name: unix-time + description: The time of the git commit in unix timestamp format. + steps: + - image: alpine/git:v2.34.2 + name: detect-tag-step + script: | + #!/usr/bin/env sh + set -euo pipefail + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: report-pending taskRef: resolver: git @@ -184,32 +209,35 @@ spec: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 8e4e323389e66d8365a3243f8e956136e916132e - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)" + - name: OUTPUT + value: >- + type=image,"name=harbor.fizz.buzz/private/homepage-build-organic:latest,harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: . - name: DOCKERFILE - value: docker/organic/Dockerfile - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" + value: docker/organic/ - name: EXTRA_ARGS value: - - "--destination=harbor.fizz.buzz/private/homepage-build-organic" # Also write the :latest image - - "--target=" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - --import-cache + - "type=registry,ref=harbor.fizz.buzz/private/homepage-build-organic:buildcache" + - --export-cache + - "type=registry,ref=harbor.fizz.buzz/private/homepage-build-organic:buildcache,mode=max,compression=zstd,compression-level=3,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source @@ -232,7 +260,7 @@ spec: name: build-explorer-wasm params: - name: IMAGE - value: "harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)" + value: "$(tasks.build-explorer-image.results.IMAGE_URL[1])" workspaces: - name: organic workspace: git-source-organic From 8b6836ffd9cfeb88efc1a7f71a97fdc568a5e9fe Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 20 Oct 2024 17:28:12 -0400 Subject: [PATCH 5/8] Switch to encoding the full build process in the Dockerfile. New features added in buildkit enable us to encode the full build process in the Dockerfile which makes tekton no longer a hard dependency for building the homepage. --- docker/organic/Dockerfile | 7 ---- docker/organic/Makefile | 52 ------------------------------ docker/server/Dockerfile | 68 ++++++++++++++++++++++++++++++++++----- docker/server/Makefile | 2 +- 4 files changed, 61 insertions(+), 68 deletions(-) delete mode 100644 docker/organic/Dockerfile delete mode 100644 docker/organic/Makefile diff --git a/docker/organic/Dockerfile b/docker/organic/Dockerfile deleted file mode 100644 index dfb7a02..0000000 --- a/docker/organic/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM rustlang/rust:nightly-alpine3.20 - -RUN apk add --no-cache musl-dev make bash -RUN rustup target add wasm32-unknown-unknown -RUN cargo install wasm-bindgen-cli - -CMD ["make", "wasm"] diff --git a/docker/organic/Makefile b/docker/organic/Makefile deleted file mode 100644 index 7239a4b..0000000 --- a/docker/organic/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -SHELL := bash -.ONESHELL: -.SHELLFLAGS := -eu -o pipefail -c -.DELETE_ON_ERROR: -MAKEFLAGS += --warn-undefined-variables -MAKEFLAGS += --no-builtin-rules -OS:=$(shell uname -s) - -ifeq ($(origin .RECIPEPREFIX), undefined) - $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later) -endif -.RECIPEPREFIX = > - -IMAGE_NAME:=homepage-build-organic -# REMOTE_REPO:=harbor.fizz.buzz/private -TARGET := - -.PHONY: help -help: -> @grep -h "##" $(MAKEFILE_LIST) | grep -v grep | sed -E 's/^([^:]*): *## */\1: /' - -.PHONY: build -build: ## Build the docker image. -> docker build --tag $(IMAGE_NAME) --target=$(TARGET) --file Dockerfile ../../ - -.PHONY: push -push: ## Push the docker image to a remote repository. -ifdef REMOTE_REPO -> docker tag $(IMAGE_NAME) $(REMOTE_REPO)/$(IMAGE_NAME) -> docker push $(REMOTE_REPO)/$(IMAGE_NAME) -else -> @echo "REMOTE_REPO not defined, not pushing to a remote repo." -endif - -.PHONY: clean -clean: -> docker rmi $(IMAGE_NAME) -ifdef REMOTE_REPO -> docker rmi $(REMOTE_REPO)/$(IMAGE_NAME) -else -> @echo "REMOTE_REPO not defined, not removing from remote repo." -endif - -.PHONY: run -run: build -run: ## Launch the docker image -> docker run --rm -i -t $(IMAGE_NAME) - -.PHONY: shell -shell: ## Launch an interactive shell inside the docker image. -shell: build -> docker run --rm -i -t --entrypoint /bin/bash --mount type=tmpfs,destination=/tmp $(IMAGE_NAME) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 713f521..3f888ce 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,17 +1,69 @@ -FROM harbor.fizz.buzz/private/natter:latest AS builder +# syntax=docker/dockerfile:1 +ARG ALPINE_VERSION="3.20" -COPY . /source -RUN ls /source/ -RUN natter build --config /source/natter.toml -FROM alpine:3.20 AS server + +FROM scratch AS private +ADD git@code.fizz.buzz:talexander/homepage_private.git /homepage_private + + + +FROM scratch AS explorer +ADD https://code.fizz.buzz/talexander/organic_ast_explorer.git /organic_ast_explorer + + + +FROM scratch AS organic +ADD git@code.fizz.buzz:talexander/organic.git /organic + + + +FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS organic-build +RUN apk add --no-cache musl-dev make bash +RUN rustup target add wasm32-unknown-unknown +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked cargo install wasm-bindgen-cli +COPY --link --from=organic /organic /organic +WORKDIR /organic +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked make wasm + + + +FROM node:lts-alpine$ALPINE_VERSION AS explorer-build +COPY --link --from=explorer /organic_ast_explorer /organic_ast_explorer +COPY --link --from=organic-build /organic /organic +WORKDIR /organic_ast_explorer +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/npmcache,sharing=locked npm set cache /npmcache && npm install +RUN npm run release + + + + +FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS natter-build +RUN apk add --no-cache musl-dev +ADD git@code.fizz.buzz:talexander/natter.git /natter +WORKDIR /natter +RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked CARGO_TARGET_DIR=/target cargo build --profile release-lto + + + +FROM alpine:$ALPINE_VERSION AS natter +COPY --link --from=natter-build /target/release-lto/natter /usr/bin/ +COPY --link . /source +COPY --link --from=private /homepage_private/static/* /source/static/ +COPY --link --from=explorer-build /organic_ast_explorer/dist/* /source/static/organic/ast_explorer/ +RUN --network=none --mount=type=tmpfs,target=/tmp natter build --config /source/natter.toml + + + + +FROM alpine:$ALPINE_VERSION AS server RUN apk add --no-cache bash nginx RUN addgroup web && adduser -D -G web web && install -d -D -o web -g web -m 700 /srv/http/public RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log -COPY --chown=web:web docker/server/nginx.conf /srv/http -COPY --chown=web:web docker/server/headers.include /srv/http -COPY --from=builder --chown=web:web /source/output/ /srv/http/public/ +COPY --link --chown=web:web docker/server/nginx.conf /srv/http +COPY --link --chown=web:web docker/server/headers.include /srv/http +COPY --link --from=natter --chown=web:web /source/output/ /srv/http/public/ ENTRYPOINT ["/usr/sbin/nginx", "-c", "/srv/http/nginx.conf", "-e", "stderr", "-g", "daemon off;"] diff --git a/docker/server/Makefile b/docker/server/Makefile index 539f791..fb40072 100644 --- a/docker/server/Makefile +++ b/docker/server/Makefile @@ -21,7 +21,7 @@ help: .PHONY: build build: ## Build the docker image. -> docker build --tag $(IMAGE_NAME) --target=$(TARGET) --file Dockerfile ../../ +> docker build --ssh default --tag $(IMAGE_NAME) --target=$(TARGET) --file Dockerfile ../../ .PHONY: push push: ## Push the docker image to a remote repository. From 500fd1f0c93b8b449d50962ad79ae5358b18e18b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 20 Oct 2024 17:33:59 -0400 Subject: [PATCH 6/8] Update workflow. --- .../pipeline-build-homepage-staging.yaml | 193 +----------------- docker/server/Dockerfile | 6 +- 2 files changed, 11 insertions(+), 188 deletions(-) diff --git a/.webhook_bridge/pipeline-build-homepage-staging.yaml b/.webhook_bridge/pipeline-build-homepage-staging.yaml index c08dd8b..24f2ba1 100644 --- a/.webhook_bridge/pipeline-build-homepage-staging.yaml +++ b/.webhook_bridge/pipeline-build-homepage-staging.yaml @@ -81,9 +81,6 @@ spec: value: task/gitea-set-status/0.1/gitea-set-status.yaml runAfter: - fetch-repository - - fetch-repository-private - - fetch-repository-explorer - - fetch-repository-organic params: - name: CONTEXT value: "$(params.JOB_NAME)" @@ -119,115 +116,30 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" - - name: fetch-repository-private - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-private - params: - - name: url - value: git@code.fizz.buzz:talexander/homepage_private.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: fetch-repository-explorer - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-explorer - params: - - name: url - value: git@code.fizz.buzz:talexander/organic_ast_explorer.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: fetch-repository-organic - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-organic - params: - - name: url - value: git@code.fizz.buzz:talexander/organic.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: copy-private-files - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.source.path)" - steps: - - image: alpine:3.20 - name: copy-private-files - script: | - #!/usr/bin/env sh - set -euo pipefail - cp -r "$(workspaces.source-private.path)/static/"* "$(workspaces.source.path)/static/" - workspaces: - - name: source - workspace: git-source - - name: source-private - workspace: git-source-private - runAfter: - - fetch-repository - - fetch-repository-private - - name: build-explorer-image + - name: build-image taskRef: resolver: git params: - name: url value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: 8e4e323389e66d8365a3243f8e956136e916132e + value: 3411d0cd39a749464bbf70ba40e2ca83ee9e2d02 - name: pathInRepo value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - name: OUTPUT value: >- - type=image,"name=harbor.fizz.buzz/private/homepage-build-organic:latest,harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.get-time.results.unix-time)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT - value: . + value: $(params.path-to-image-context) - name: DOCKERFILE - value: docker/organic/ + value: $(params.path-to-dockerfile) - name: EXTRA_ARGS value: - --import-cache - - "type=registry,ref=harbor.fizz.buzz/private/homepage-build-organic:buildcache" + - "type=registry,ref=$(params.image-name):buildcache" - --export-cache - - "type=registry,ref=harbor.fizz.buzz/private/homepage-build-organic:buildcache,mode=max,compression=zstd,compression-level=3,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" - --opt - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) - name: BUILDKITD_TOML @@ -245,95 +157,6 @@ spec: workspace: docker-credentials runAfter: - fetch-repository - - name: build-explorer-wasm - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.organic.path)" - steps: - - image: "$(params.IMAGE)" - name: build-explorer-wasm - params: - - name: IMAGE - value: "$(tasks.build-explorer-image.results.IMAGE_URL[1])" - workspaces: - - name: organic - workspace: git-source-organic - runAfter: - - build-explorer-image - - name: copy-explorer-files - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.source-explorer.path)" - steps: - - image: node:lts-alpine3.20 - name: copy-explorer-files - script: | - #!/usr/bin/env sh - set -euo pipefail - npm install - npm run release - mkdir -p "$(workspaces.source.path)/static/organic/ast_explorer/" - cp -r "$(workspaces.source-explorer.path)/dist/"* "$(workspaces.source.path)/static/organic/ast_explorer/" - workspaces: - - name: source - workspace: git-source - - name: source-explorer - workspace: git-source-explorer - - name: organic - workspace: git-source-organic - runAfter: - - build-explorer-wasm - - name: build-image - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml - params: - - name: IMAGE - value: "$(params.image-name):$(tasks.get-time.results.unix-time)" - - name: CONTEXT - value: $(params.path-to-image-context) - - name: DOCKERFILE - value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - - name: EXTRA_ARGS - value: - - "--destination=$(params.image-name)" # Also write the :latest image - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local - workspaces: - - name: source - workspace: git-source - - name: dockerconfig - workspace: docker-credentials - runAfter: - - copy-private-files - - copy-explorer-files finally: - name: report-success when: @@ -448,4 +271,4 @@ spec: - name: path-to-image-context value: . - name: path-to-dockerfile - value: docker/server/Dockerfile + value: docker/server diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 3f888ce..dfa06ca 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -62,8 +62,8 @@ RUN apk add --no-cache bash nginx RUN addgroup web && adduser -D -G web web && install -d -D -o web -g web -m 700 /srv/http/public RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log -COPY --link --chown=web:web docker/server/nginx.conf /srv/http -COPY --link --chown=web:web docker/server/headers.include /srv/http -COPY --link --from=natter --chown=web:web /source/output/ /srv/http/public/ +COPY --chown=web:web docker/server/nginx.conf /srv/http +COPY --chown=web:web docker/server/headers.include /srv/http +COPY --from=natter --chown=web:web /source/output/ /srv/http/public/ ENTRYPOINT ["/usr/sbin/nginx", "-c", "/srv/http/nginx.conf", "-e", "stderr", "-g", "daemon off;"] From 6dbd323979fc4adba7594b4bde52bf169437120c Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 20 Oct 2024 19:06:53 -0400 Subject: [PATCH 7/8] Convert production over to the new builtkit build. --- .webhook_bridge/pipeline-build-homepage.yaml | 235 +++---------------- 1 file changed, 33 insertions(+), 202 deletions(-) diff --git a/.webhook_bridge/pipeline-build-homepage.yaml b/.webhook_bridge/pipeline-build-homepage.yaml index 69df7d2..6650269 100644 --- a/.webhook_bridge/pipeline-build-homepage.yaml +++ b/.webhook_bridge/pipeline-build-homepage.yaml @@ -24,7 +24,7 @@ spec: description: The path to the Dockerfile type: string tasks: - - name: get-time + - name: get-git-commit-time taskSpec: metadata: {} stepTemplate: @@ -33,17 +33,22 @@ spec: requests: cpu: 10m memory: 600Mi - workingDir: "/" + workingDir: "$(workspaces.repo.path)" results: - name: unix-time - description: The current date in unix timestamp format. + description: The time of the git commit in unix timestamp format. steps: - - image: alpine:3.20 - name: get-time-step + - image: alpine/git:v2.34.2 + name: detect-tag-step script: | #!/usr/bin/env sh set -euo pipefail - echo -n "$(date +%s)" | tee $(results.unix-time.path) + echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path) + workspaces: + - name: repo + workspace: git-source + runAfter: + - fetch-repository - name: detect-tag taskSpec: metadata: {} @@ -88,9 +93,6 @@ spec: value: task/gitea-set-status/0.1/gitea-set-status.yaml runAfter: - fetch-repository - - fetch-repository-private - - fetch-repository-explorer - - fetch-repository-organic params: - name: CONTEXT value: "$(params.JOB_NAME)" @@ -126,218 +128,47 @@ spec: value: $(params.PULL_BASE_SHA) - name: deleteExisting value: "true" - - name: fetch-repository-private - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-private - params: - - name: url - value: git@code.fizz.buzz:talexander/homepage_private.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: fetch-repository-explorer - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-explorer - params: - - name: url - value: git@code.fizz.buzz:talexander/organic_ast_explorer.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: fetch-repository-organic - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/git-clone/0.9/git-clone.yaml - workspaces: - - name: output - workspace: git-source-organic - params: - - name: url - value: git@code.fizz.buzz:talexander/organic.git - - name: revision - value: main - - name: deleteExisting - value: "true" - - name: copy-private-files - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.source.path)" - steps: - - image: alpine:3.20 - name: copy-private-files - script: | - #!/usr/bin/env sh - set -euo pipefail - cp -r "$(workspaces.source-private.path)/static/"* "$(workspaces.source.path)/static/" - workspaces: - - name: source - workspace: git-source - - name: source-private - workspace: git-source-private - runAfter: - - fetch-repository - - fetch-repository-private - - name: build-explorer-image - taskRef: - resolver: git - params: - - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git - - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf - - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml - params: - - name: IMAGE - value: "harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)" - - name: CONTEXT - value: . - - name: DOCKERFILE - value: docker/organic/Dockerfile - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - - name: EXTRA_ARGS - value: - - "--destination=harbor.fizz.buzz/private/homepage-build-organic" # Also write the :latest image - - "--target=" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local - workspaces: - - name: source - workspace: git-source - - name: dockerconfig - workspace: docker-credentials - runAfter: - - fetch-repository - - name: build-explorer-wasm - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.organic.path)" - steps: - - image: "$(params.IMAGE)" - name: build-explorer-wasm - params: - - name: IMAGE - value: "harbor.fizz.buzz/private/homepage-build-organic:$(tasks.get-time.results.unix-time)" - workspaces: - - name: organic - workspace: git-source-organic - runAfter: - - build-explorer-image - - name: copy-explorer-files - taskSpec: - metadata: {} - stepTemplate: - image: alpine:3.20 - computeResources: - requests: - cpu: 10m - memory: 600Mi - workingDir: "$(workspaces.source-explorer.path)" - steps: - - image: node:lts-alpine3.20 - name: copy-explorer-files - script: | - #!/usr/bin/env sh - set -euo pipefail - npm install - npm run release - mkdir -p "$(workspaces.source.path)/static/organic/ast_explorer/" - cp -r "$(workspaces.source-explorer.path)/dist/"* "$(workspaces.source.path)/static/organic/ast_explorer/" - workspaces: - - name: source - workspace: git-source - - name: source-explorer - workspace: git-source-explorer - - name: organic - workspace: git-source-organic - runAfter: - - build-explorer-wasm - name: build-image taskRef: resolver: git params: - name: url - value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git + value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git - name: revision - value: df36b3853a5657fd883015cdbf07ad6466918acf + value: 3411d0cd39a749464bbf70ba40e2ca83ee9e2d02 - name: pathInRepo - value: task/kaniko/0.6/kaniko.yaml + value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml params: - - name: IMAGE - value: "$(params.image-name):$(tasks.detect-tag.results.tag)" + - name: OUTPUT + value: >- + type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.detect-tag.results.tag)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true - name: CONTEXT value: $(params.path-to-image-context) - name: DOCKERFILE value: $(params.path-to-dockerfile) - - name: BUILDER_IMAGE - value: "gcr.io/kaniko-project/executor:v1.23.2" - name: EXTRA_ARGS value: - - "--destination=$(params.image-name)" # Also write the :latest image - - "--target=$(params.target-name)" - - --cache=true - - --cache-copy-layers - - --cache-repo=harbor.fizz.buzz/kanikocache/cache - - --use-new-run # Should result in a speed-up - - --reproducible # To remove timestamps so layer caching works. - - --snapshot-mode=redo - - --skip-unused-stages=true - - --registry-mirror=dockerhub.dockerhub.svc.cluster.local + - --import-cache + - "type=registry,ref=$(params.image-name):buildcache" + - --export-cache + - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true" + - --opt + - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time) + - name: BUILDKITD_TOML + value: | + debug = true + [registry."docker.io"] + mirrors = ["dockerhub.dockerhub.svc.cluster.local"] + [registry."dockerhub.dockerhub.svc.cluster.local"] + http = true + insecure = true workspaces: - name: source workspace: git-source - name: dockerconfig workspace: docker-credentials runAfter: - - copy-private-files - - copy-explorer-files + - fetch-repository finally: - name: report-success when: @@ -452,4 +283,4 @@ spec: - name: path-to-image-context value: . - name: path-to-dockerfile - value: docker/server/Dockerfile + value: docker/server From 2fa772934dfbee9eecccafdff09774cc0ec41814 Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Sun, 20 Oct 2024 19:36:21 -0400 Subject: [PATCH 8/8] Remove extra volumes. --- .../pipeline-build-homepage-staging.yaml | 30 ------------------- .webhook_bridge/pipeline-build-homepage.yaml | 30 ------------------- 2 files changed, 60 deletions(-) diff --git a/.webhook_bridge/pipeline-build-homepage-staging.yaml b/.webhook_bridge/pipeline-build-homepage-staging.yaml index 24f2ba1..ff538c8 100644 --- a/.webhook_bridge/pipeline-build-homepage-staging.yaml +++ b/.webhook_bridge/pipeline-build-homepage-staging.yaml @@ -230,36 +230,6 @@ spec: requests: storage: 10Gi subPath: rust-source - - name: git-source-private - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - - name: git-source-explorer - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - - name: git-source-organic - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - name: docker-credentials secret: secretName: harbor-plain diff --git a/.webhook_bridge/pipeline-build-homepage.yaml b/.webhook_bridge/pipeline-build-homepage.yaml index 6650269..7564b84 100644 --- a/.webhook_bridge/pipeline-build-homepage.yaml +++ b/.webhook_bridge/pipeline-build-homepage.yaml @@ -242,36 +242,6 @@ spec: requests: storage: 10Gi subPath: rust-source - - name: git-source-private - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - - name: git-source-explorer - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - - name: git-source-organic - volumeClaimTemplate: - spec: - storageClassName: "nfs-client" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - subPath: git-source - name: docker-credentials secret: secretName: harbor-plain