Merge branch 'cache_nix_build'

Canonicalizing in the nix build was causing the image links to reference the nix build path. By switching to lexical normalization, the paths are being properly calculated.

.webhook_bridge/pipeline-build-homepage-staging.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   timeouts:
     pipeline: "2h0m0s"
-    tasks: "1h0m0s"
+    tasks: "1h0m40s"
     finally: "0h30m0s"
   taskRunTemplate:
     serviceAccountName: build-bot
@@ -53,7 +53,7 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         params:
@@ -123,37 +123,43 @@ spec:
             - name: url
               value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git
             - name: revision
-              value: dda7b690195b43e8b9859d1caf5dcbf48588ade1
+              value: 9c6d3bc418dc2f54d73bdfa984bce3eb6c65fd60
             - name: pathInRepo
-              value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml
+              value: task/buildah-rootless-daemonless/0.1/buildah-rootless-daemonless.yaml
         params:
           - name: OUTPUT
             value: >-
-              type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.get-time.results.unix-time)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true
+              $(params.image-name):$(tasks.get-time.results.unix-time)
           - name: CONTEXT
             value: $(params.path-to-image-context)
           - name: DOCKERFILE
             value: $(params.path-to-dockerfile)
           - name: EXTRA_ARGS
             value:
-              - "--opt"
+              - "--target"
-              - "target=$(params.target-name)"
+              - "$(params.target-name)"
-              - --import-cache
+              - --cache-from
-              - "type=registry,ref=$(params.image-name):buildcache"
+              - "$(params.image-name)-buildcache"
-              - --export-cache
+              - --cache-to
-              - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true"
+              - "$(params.image-name)-buildcache"
-              - --opt
+              - --build-arg
-              - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)
+              - "SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)"
-              - --opt
+              - --build-arg
-              - "build-arg:NIX_SUBSTITUTERS=http://ncps.nix-pull-through-cache.svc.cluster.local:80 https://cache.nixos.org"
+              - "NIX_SUBSTITUTERS=http://ncps.nix-pull-through-cache.svc.cluster.local:80 https://cache.nixos.org"
-          - name: BUILDKITD_TOML
+          - name: REGISTRIES_CONF
             value: |
-              debug = true
+              unqualified-search-registries = ["docker.io"]
-              [registry."docker.io"]
+
-                mirrors = ["dockerhub.dockerhub.svc.cluster.local"]
+              [[registry]]
-              [registry."dockerhub.dockerhub.svc.cluster.local"]
+              location = "dockerhub.dockerhub.svc.cluster.local"
-                http = true
+              insecure = true
-                insecure = true
+
+              [[registry]]
+              location = "docker.io"
+
+              [[registry.mirror]]
+              location = "dockerhub.dockerhub.svc.cluster.local" # Replace with your mirror URL
+              insecure = true
         workspaces:
           - name: source
             workspace: git-source
@@ -173,7 +179,7 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         params:
@@ -202,7 +208,7 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         params:
@@ -245,4 +251,4 @@ spec:
     - name: path-to-image-context
       value: .
     - name: path-to-dockerfile
-      value: docker/server
+      value: docker/server/Dockerfile

.webhook_bridge/pipeline-build-homepage.yaml

@@ -7,7 +7,7 @@ metadata:
 spec:
   timeouts:
     pipeline: "2h0m0s"
-    tasks: "1h0m0s"
+    tasks: "1h0m40s"
     finally: "0h30m0s"
   taskRunTemplate:
     serviceAccountName: build-bot
@@ -70,6 +70,7 @@ spec:
               script: |
                 #!/usr/bin/env sh
                 set -euo pipefail
+                export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"
                 git fetch --tags
                 current_tag=$(git tag --points-at HEAD --list 'v*.*.*')
                 if [ -z "$current_tag" ]; then
@@ -90,11 +91,9 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
-        runAfter:
-          - fetch-repository
         params:
           - name: CONTEXT
             value: "$(params.JOB_NAME)"
@@ -110,19 +109,9 @@ spec:
             value: pending
           - name: TARGET_URL
             value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+        runAfter:
+          - fetch-repository
       - name: fetch-repository
-        taskRef:
-          resolver: git
-          params:
-            - name: url
-              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
-            - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
-            - name: pathInRepo
-              value: task/git-clone/0.9/git-clone.yaml
-        workspaces:
-          - name: output
-            workspace: git-source
         params:
           - name: url
             value: $(params.REPO_URL)
@@ -130,6 +119,18 @@ spec:
             value: $(params.PULL_BASE_SHA)
           - name: deleteExisting
             value: "true"
+        taskRef:
+          params:
+            - name: url
+              value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git
+            - name: revision
+              value: "dda7b690195b43e8b9859d1caf5dcbf48588ade1"
+            - name: pathInRepo
+              value: task/git-clone/0.1/git-clone.yaml
+          resolver: git
+        workspaces:
+          - name: output
+            workspace: git-source
       - name: build-image
         taskRef:
           resolver: git
@@ -137,37 +138,43 @@ spec:
             - name: url
               value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git
             - name: revision
-              value: dda7b690195b43e8b9859d1caf5dcbf48588ade1
+              value: 9c6d3bc418dc2f54d73bdfa984bce3eb6c65fd60
             - name: pathInRepo
-              value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml
+              value: task/buildah-rootless-daemonless/0.1/buildah-rootless-daemonless.yaml
         params:
           - name: OUTPUT
             value: >-
-              type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.detect-tag.results.tag)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true
+              $(params.image-name):$(tasks.detect-tag.results.tag)
           - name: CONTEXT
             value: $(params.path-to-image-context)
           - name: DOCKERFILE
             value: $(params.path-to-dockerfile)
           - name: EXTRA_ARGS
             value:
-              - "--opt"
+              - "--target"
-              - "target=$(params.target-name)"
+              - "$(params.target-name)"
-              - --import-cache
+              - --cache-from
-              - "type=registry,ref=$(params.image-name):buildcache"
+              - "$(params.image-name)-buildcache"
-              - --export-cache
+              - --cache-to
-              - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true"
+              - "$(params.image-name)-buildcache"
-              - --opt
+              - --build-arg
-              - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)
+              - "SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)"
-              - --opt
+              - --build-arg
-              - "build-arg:NIX_SUBSTITUTERS=http://ncps.nix-pull-through-cache.svc.cluster.local:80 https://cache.nixos.org"
+              - "NIX_SUBSTITUTERS=http://ncps.nix-pull-through-cache.svc.cluster.local:80 https://cache.nixos.org"
-          - name: BUILDKITD_TOML
+          - name: REGISTRIES_CONF
             value: |
-              debug = true
+              unqualified-search-registries = ["docker.io"]
-              [registry."docker.io"]
+
-                mirrors = ["dockerhub.dockerhub.svc.cluster.local"]
+              [[registry]]
-              [registry."dockerhub.dockerhub.svc.cluster.local"]
+              location = "dockerhub.dockerhub.svc.cluster.local"
-                http = true
+              insecure = true
-                insecure = true
+
+              [[registry]]
+              location = "docker.io"
+
+              [[registry.mirror]]
+              location = "dockerhub.dockerhub.svc.cluster.local" # Replace with your mirror URL
+              insecure = true
         workspaces:
           - name: source
             workspace: git-source
@@ -187,7 +194,7 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         params:
@@ -216,7 +223,7 @@ spec:
             - name: url
               value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
-              value: f914437a46978b95f325f68d791dcf1a35738f60
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         params:
@@ -259,4 +266,4 @@ spec:
     - name: path-to-image-context
       value: .
     - name: path-to-dockerfile
-      value: docker/server
+      value: docker/server/Dockerfile

docker/server/Dockerfile

@@ -15,9 +15,13 @@ EOF
 
 RUN cp "$(nix build nixpkgs#cacert --print-out-paths)/etc/ssl/certs/ca-bundle.crt" /tmp/ca-bundle.crt
 
-COPY . /tmp/build
+COPY flake.nix flake.lock /tmp/build
 WORKDIR /tmp/build
 
+RUN --mount=type=ssh GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" nix build --no-link '.#dependencies'
+
+COPY . /tmp/build
+
 RUN --mount=type=ssh GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" nix build '.#docker_env'
 
 # Export the built closure to a folder

flake.lock

@@ -8,11 +8,11 @@
         "rust-overlay": "rust-overlay"
       },
       "locked": {
-        "lastModified": 1784333070,
+        "lastModified": 1792617478,
-        "narHash": "sha256-3WkxaHOS13vFBtQFBmGIPScQZDNLk5bqCAY9qBJASl0=",
+        "narHash": "sha256-ctFfFI7qlZxv6s8nznps7KYp/QQ1zzbMKCgLtl7YYRA=",
         "ref": "refs/heads/main",
-        "rev": "faf279292ad20783775d909b96815f7ca2843b85",
+        "rev": "1c369e4b25caf4dd9a962340a770fbd6e543820a",
-        "revCount": 327,
+        "revCount": 331,
         "type": "git",
         "url": "https://code.fizz.buzz/talexander/natter.git"
       },

flake.nix

@@ -74,6 +74,11 @@
           inherit (appliedOverlay.homepage)
             release
             ;
+          dependencies = pkgs.symlinkJoin {
+            # This is to enable the docker image to install all the dependencies, cache that layer, and then build the final site after the org source files have been loaded. Without this, every update to the blog posts would result in recompiling everything.
+            name = "homepage-dependencies";
+            paths = appliedOverlay.homepage.dependencies;
+          };
           docker_env = pkgs.stdenv.mkDerivation {
             pname = "homepage-docker-env";
             version = "0.0.0";
@@ -100,14 +105,16 @@
             natter' = (natter.overlays.default final final).natter.release;
             organic_ast_explorer' =
               (organic_ast_explorer.overlays.default final final).organic_ast_explorer.release;
-            release = homepageScope.callPackage ./nix/package.nix {
+            dependencies = {
               natter = natter';
               organic_ast_explorer = organic_ast_explorer';
               inherit homepage_private;
             };
+            release = homepageScope.callPackage ./nix/package.nix dependencies;
           in
           {
             inherit release;
+            dependencies = (builtins.attrValues dependencies);
           }
         );
       };