Reduce memory usage by using a disk for buildkit's backing.

This version does not launch the ssh agent if no keys are mounted.

.lighthouse/triggers.yaml

@@ -1,43 +0,0 @@
-apiVersion: config.lighthouse.jenkins-x.io/v1alpha1
-kind: TriggerConfig
-spec:
-  postsubmits:
-    - name: semver
-      agent: tekton-pipeline
-      branches:
-        - ^main$
-        - ^master$
-      context: homepage
-      max_concurrency: 1
-      # Override https-based url from lighthouse events.
-      clone_uri: "git@code.fizz.buzz:talexander/homepage.git"
-      pipeline_run_spec:
-        serviceAccountName: build-bot
-        pipelineRef:
-          name: semver
-          namespace: lighthouse
-        workspaces:
-          - name: git-source
-            volumeClaimTemplate:
-              spec:
-                storageClassName: "nfs-client"
-                accessModes:
-                  - ReadWriteOnce
-                resources:
-                  requests:
-                    storage: 10Gi
-            subPath: homepage-source
-        params: []
-    - name: build-homepage-staging
-      source: "pipeline-build-homepage-staging.yaml"
-      # Override https-based url from lighthouse events.
-      clone_uri: "git@code.fizz.buzz:talexander/homepage.git"
-      skip_branches:
-        # We already run on every commit, so running when the semver tags get pushed is causing needless double-processing.
-        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
-    - name: build-homepage
-      source: "pipeline-build-homepage.yaml"
-      # Override https-based url from lighthouse events.
-      clone_uri: "git@code.fizz.buzz:talexander/homepage.git"
-      branches:
-        - "^v[0-9]+\\.[0-9]+\\.[0-9]+$"

.webhook_bridge/pipeline-build-homepage-staging.yaml

@@ -1,13 +1,22 @@
-apiVersion: tekton.dev/v1beta1
+apiVersion: tekton.dev/v1
 kind: PipelineRun
 metadata:
   name: build-homepage-staging
 spec:
+  timeouts:
+    pipeline: "2h0m0s"
+    tasks: "1h0m0s"
+    finally: "0h30m0s"
+  taskRunTemplate:
+    serviceAccountName: build-bot
   pipelineSpec:
     params:
       - name: image-name
         description: The name for the built image
         type: string
+      - name: target-name
+        description: The dockerfile target to build
+        type: string
       - name: path-to-image-context
         description: The path to the build context
         type: string
@@ -19,36 +28,59 @@ spec:
         taskSpec:
           metadata: {}
           stepTemplate:
-            image: alpine:3.18
+            image: alpine:3.20
-            name: ""
+            computeResources:
-            resources:
               requests:
                 cpu: 10m
                 memory: 600Mi
-            workingDir: /workspace/source
+            workingDir: "/"
           results:
             - name: unix-time
-              description: The current date in unix timestamp format
+              description: The current date in unix timestamp format.
           steps:
-            - image: alpine:3.18
+            - image: alpine:3.20
               name: get-time-step
               script: |
                 #!/usr/bin/env sh
                 set -euo pipefail
                 echo -n "$(date +%s)" | tee $(results.unix-time.path)
+      - name: get-git-commit-time
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.20
+            computeResources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: "$(workspaces.repo.path)"
+          results:
+            - name: unix-time
+              description: The time of the git commit in unix timestamp format.
+          steps:
+            - image: alpine/git:v2.34.2
+              name: detect-tag-step
+              script: |
+                #!/usr/bin/env sh
+                set -euo pipefail
+                echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path)
+        workspaces:
+          - name: repo
+            workspace: git-source
+        runAfter:
+          - fetch-repository
       - name: report-pending
         taskRef:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         runAfter:
           - fetch-repository
-          - fetch-repository-private
         params:
           - name: CONTEXT
             value: "$(params.JOB_NAME)"
@@ -69,7 +101,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -84,89 +116,51 @@ spec:
             value: $(params.PULL_BASE_SHA)
           - name: deleteExisting
             value: "true"
-      - name: fetch-repository-private
-        taskRef:
-          resolver: git
-          params:
-            - name: url
-              value: https://github.com/tektoncd/catalog.git
-            - name: revision
-              value: df36b3853a5657fd883015cdbf07ad6466918acf
-            - name: pathInRepo
-              value: task/git-clone/0.9/git-clone.yaml
-        workspaces:
-          - name: output
-            workspace: git-source-private
-        params:
-          - name: url
-            value: git@code.fizz.buzz:talexander/homepage_private.git
-          - name: revision
-            value: main
-          - name: deleteExisting
-            value: "true"
-      - name: copy-private-files
-        taskSpec:
-          metadata: {}
-          stepTemplate:
-            image: alpine:3.18
-            name: ""
-            resources:
-              requests:
-                cpu: 10m
-                memory: 600Mi
-            workingDir: "$(workspaces.source.path)"
-          steps:
-            - image: alpine:3.18
-              name: copy-private-files
-              script: |
-                #!/usr/bin/env sh
-                set -euo pipefail
-                cp -r "$(workspaces.source-private.path)/static/"* "$(workspaces.source.path)/static/"
-        workspaces:
-          - name: source
-            workspace: git-source
-          - name: source-private
-            workspace: git-source-private
-        runAfter:
-          - get-time
-          - report-pending
       - name: build-image
         taskRef:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git
             - name: revision
-              value: df36b3853a5657fd883015cdbf07ad6466918acf
+              value: 7ee31a185243ee6da13dcd26a592c585b64c80e5
             - name: pathInRepo
-              value: task/kaniko/0.6//kaniko.yaml
+              value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml
         params:
-          - name: IMAGE
+          - name: OUTPUT
-            value: "$(params.image-name):$(tasks.get-time.results.unix-time)"
+            value: >-
+              type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.get-time.results.unix-time)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true
           - name: CONTEXT
             value: $(params.path-to-image-context)
           - name: DOCKERFILE
             value: $(params.path-to-dockerfile)
-          - name: BUILDER_IMAGE
-            value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
-              - "--destination=$(params.image-name)" # Also write the :latest image
+              - "--opt"
-              - --cache=true
+              - "target=$(params.target-name)"
-              - --cache-copy-layers
+              - --import-cache
-              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - "type=registry,ref=$(params.image-name):buildcache"
-              - --use-new-run # Should result in a speed-up
+              - --export-cache
-              - --reproducible # To remove timestamps so layer caching works.
+              - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true"
-              - --snapshot-mode=redo
+              - --opt
-              - --skip-unused-stages=true
+              - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)
-              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+          - name: BUILDKITD_TOML
+            value: |
+              debug = true
+              [registry."docker.io"]
+                mirrors = ["dockerhub.dockerhub.svc.cluster.local"]
+              [registry."dockerhub.dockerhub.svc.cluster.local"]
+                http = true
+                insecure = true
         workspaces:
           - name: source
             workspace: git-source
+          - name: buildkitd
+            workspace: buildkit-cache
           - name: dockerconfig
             workspace: docker-credentials
         runAfter:
-          - copy-private-files
+          - fetch-repository
     finally:
       - name: report-success
         when:
@@ -177,7 +171,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -206,7 +200,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -228,6 +222,7 @@ spec:
             value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
     workspaces:
       - name: git-source
+      - name: buildkit-cache
       - name: docker-credentials
   workspaces:
     - name: git-source
@@ -239,8 +234,8 @@ spec:
           resources:
             requests:
               storage: 10Gi
-      subPath: rust-source
+      subPath: git-source
-    - name: git-source-private
+    - name: buildkit-cache
       volumeClaimTemplate:
         spec:
           storageClassName: "nfs-client"
@@ -249,16 +244,16 @@ spec:
           resources:
             requests:
               storage: 10Gi
-      subPath: git-source
+      subPath: builtkit-cache
     - name: docker-credentials
       secret:
         secretName: harbor-plain
-  serviceAccountName: build-bot
-  timeout: 240h0m0s
   params:
     - name: image-name
       value: "harbor.fizz.buzz/private/homepage-staging"
+    - name: target-name
+      value: ""
     - name: path-to-image-context
       value: .
     - name: path-to-dockerfile
-      value: docker/server/Dockerfile
+      value: docker/server

.webhook_bridge/pipeline-build-homepage.yaml

@@ -1,13 +1,22 @@
-apiVersion: tekton.dev/v1beta1
+apiVersion: tekton.dev/v1
 kind: PipelineRun
 metadata:
   name: build-homepage
 spec:
+  timeouts:
+    pipeline: "2h0m0s"
+    tasks: "1h0m0s"
+    finally: "0h30m0s"
+  taskRunTemplate:
+    serviceAccountName: build-bot
   pipelineSpec:
     params:
       - name: image-name
         description: The name for the built image
         type: string
+      - name: target-name
+        description: The dockerfile target to build
+        type: string
       - name: path-to-image-context
         description: The path to the build context
         type: string
@@ -15,13 +24,37 @@ spec:
         description: The path to the Dockerfile
         type: string
     tasks:
+      - name: get-git-commit-time
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.20
+            computeResources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: "$(workspaces.repo.path)"
+          results:
+            - name: unix-time
+              description: The time of the git commit in unix timestamp format.
+          steps:
+            - image: alpine/git:v2.34.2
+              name: detect-tag-step
+              script: |
+                #!/usr/bin/env sh
+                set -euo pipefail
+                echo -n "$(git log -1 --pretty=%ct)" | tee $(results.unix-time.path)
+        workspaces:
+          - name: repo
+            workspace: git-source
+        runAfter:
+          - fetch-repository
       - name: detect-tag
         taskSpec:
           metadata: {}
           stepTemplate:
-            image: alpine:3.18
+            image: alpine:3.20
-            name: ""
+            computeResources:
-            resources:
               requests:
                 cpu: 10m
                 memory: 600Mi
@@ -53,14 +86,13 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
               value: task/gitea-set-status/0.1/gitea-set-status.yaml
         runAfter:
           - fetch-repository
-          - fetch-repository-private
         params:
           - name: CONTEXT
             value: "$(params.JOB_NAME)"
@@ -81,7 +113,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -96,88 +128,49 @@ spec:
             value: $(params.PULL_BASE_SHA)
           - name: deleteExisting
             value: "true"
-      - name: fetch-repository-private
-        taskRef:
-          resolver: git
-          params:
-            - name: url
-              value: https://github.com/tektoncd/catalog.git
-            - name: revision
-              value: df36b3853a5657fd883015cdbf07ad6466918acf
-            - name: pathInRepo
-              value: task/git-clone/0.9/git-clone.yaml
-        workspaces:
-          - name: output
-            workspace: git-source-private
-        params:
-          - name: url
-            value: git@code.fizz.buzz:talexander/homepage_private.git
-          - name: revision
-            value: main
-          - name: deleteExisting
-            value: "true"
-      - name: copy-private-files
-        taskSpec:
-          metadata: {}
-          stepTemplate:
-            image: alpine:3.18
-            name: ""
-            resources:
-              requests:
-                cpu: 10m
-                memory: 600Mi
-            workingDir: "$(workspaces.source.path)"
-          steps:
-            - image: alpine:3.18
-              name: copy-private-files
-              script: |
-                #!/usr/bin/env sh
-                set -euo pipefail
-                cp -r "$(workspaces.source-private.path)/static/"* "$(workspaces.source.path)/static/"
-        workspaces:
-          - name: source
-            workspace: git-source
-          - name: source-private
-            workspace: git-source-private
-        runAfter:
-          - report-pending
       - name: build-image
         taskRef:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/talexander/personal_tekton_catalog.git
             - name: revision
-              value: df36b3853a5657fd883015cdbf07ad6466918acf
+              value: 7ee31a185243ee6da13dcd26a592c585b64c80e5
             - name: pathInRepo
-              value: task/kaniko/0.6//kaniko.yaml
+              value: task/buildkit-rootless-daemonless/0.1/buildkit-rootless-daemonless.yaml
         params:
-          - name: IMAGE
+          - name: OUTPUT
-            value: "$(params.image-name):$(tasks.detect-tag.results.tag)"
+            value: >-
+              type=image,"name=$(params.image-name):latest,$(params.image-name):$(tasks.detect-tag.results.tag)",push=true,compression=zstd,compression-level=22,oci-mediatypes=true
           - name: CONTEXT
             value: $(params.path-to-image-context)
           - name: DOCKERFILE
             value: $(params.path-to-dockerfile)
-          - name: BUILDER_IMAGE
-            value: "gcr.io/kaniko-project/executor:v1.12.1"
           - name: EXTRA_ARGS
             value:
-              - "--destination=$(params.image-name)" # Also write the :latest image
+              - "--opt"
-              - --cache=true
+              - "target=$(params.target-name)"
-              - --cache-copy-layers
+              - --import-cache
-              - --cache-repo=harbor.fizz.buzz/kanikocache/cache
+              - "type=registry,ref=$(params.image-name):buildcache"
-              - --use-new-run # Should result in a speed-up
+              - --export-cache
-              - --reproducible # To remove timestamps so layer caching works.
+              - "type=registry,ref=$(params.image-name):buildcache,mode=max,compression=zstd,compression-level=22,rewrite-timestamp=true,image-manifest=true,oci-mediatypes=true"
-              - --snapshot-mode=redo
+              - --opt
-              - --skip-unused-stages=true
+              - build-arg:SOURCE_DATE_EPOCH=$(tasks.get-git-commit-time.results.unix-time)
-              - --registry-mirror=dockerhub.dockerhub.svc.cluster.local
+          - name: BUILDKITD_TOML
+            value: |
+              debug = true
+              [registry."docker.io"]
+                mirrors = ["dockerhub.dockerhub.svc.cluster.local"]
+              [registry."dockerhub.dockerhub.svc.cluster.local"]
+                http = true
+                insecure = true
         workspaces:
           - name: source
             workspace: git-source
           - name: dockerconfig
             workspace: docker-credentials
         runAfter:
-          - copy-private-files
+          - fetch-repository
     finally:
       - name: report-success
         when:
@@ -188,7 +181,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -217,7 +210,7 @@ spec:
           resolver: git
           params:
             - name: url
-              value: https://github.com/tektoncd/catalog.git
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
             - name: revision
               value: df36b3853a5657fd883015cdbf07ad6466918acf
             - name: pathInRepo
@@ -251,25 +244,15 @@ spec:
             requests:
               storage: 10Gi
       subPath: rust-source
-    - name: git-source-private
-      volumeClaimTemplate:
-        spec:
-          storageClassName: "nfs-client"
-          accessModes:
-            - ReadWriteOnce
-          resources:
-            requests:
-              storage: 10Gi
-      subPath: git-source
     - name: docker-credentials
       secret:
         secretName: harbor-plain
-  serviceAccountName: build-bot
-  timeout: 240h0m0s
   params:
     - name: image-name
       value: "harbor.fizz.buzz/private/homepage"
+    - name: target-name
+      value: ""
     - name: path-to-image-context
       value: .
     - name: path-to-dockerfile
-      value: docker/server/Dockerfile
+      value: docker/server

.webhook_bridge/pipeline-semver.yaml

@@ -0,0 +1,187 @@
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+metadata:
+  name: semver
+spec:
+  timeouts:
+    pipeline: "2h0m0s"
+    tasks: "1h0m0s"
+    finally: "0h30m0s"
+  taskRunTemplate:
+    serviceAccountName: build-bot
+  pipelineSpec:
+    params:
+      - name: REPO_OWNER
+        description: Owner of the repo on gitea
+        type: string
+      - name: REPO_NAME
+        description: Name of the repo on gitea
+        type: string
+      - name: PULL_BASE_SHA
+        description: The commit sha
+        type: string
+      - name: JOB_NAME
+        description: The name of the job to report to gitea
+        type: string
+    tasks:
+      - name: calculate-tag
+        runAfter:
+          - fetch-repository
+        workspaces:
+          - name: source
+            workspace: git-source
+        taskSpec:
+          metadata: {}
+          stepTemplate:
+            image: alpine:3.20
+            computeResources:
+              requests:
+                cpu: 10m
+                memory: 600Mi
+            workingDir: /workspace/source
+          results:
+            - name: tag
+              description: The tag to use for the docker container
+          steps:
+            - image: alpine/git:2.43.0
+              name: calculate-tag
+              script: |
+                #!/usr/bin/env sh
+                set -euo pipefail
+                git config --global --add safe.directory $(workspaces.source.path)
+                git fetch --tags
+                current_tag=$(git tag --points-at HEAD --list 'v*.*.*')
+                if [ -z "$current_tag" ]; then
+                  prev_tag=$(git tag --list 'v*.*.*' | sort -V -r | head -n 1)
+                  if [ -n "$prev_tag" ]; then
+                    last_bit=$(echo "$prev_tag" | cut -d '.' -f 3)
+                    incremented=$((last_bit + 1))
+                    prefix=$(echo "$prev_tag" | grep -oE 'v[0-9]*\.[0-9]*\.')
+                    final_tag="${prefix}${incremented}"
+                  else
+                    final_tag="v0.0.1"
+                  fi
+                  echo -n "${final_tag}" | tee $(results.tag.path)
+                  git tag "${final_tag}"
+                  git push origin "${final_tag}"
+                else
+                  echo -n "${current_tag}" | tee $(results.tag.path)
+                fi
+      - name: report-pending
+        taskRef:
+          resolver: git
+          params:
+            - name: url
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
+            - name: revision
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
+            - name: pathInRepo
+              value: task/gitea-set-status/0.1/gitea-set-status.yaml
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has started"
+          - name: STATE
+            value: pending
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: fetch-repository
+        taskRef:
+          resolver: git
+          params:
+            - name: url
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
+            - name: revision
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
+            - name: pathInRepo
+              value: task/git-clone/0.9/git-clone.yaml
+        workspaces:
+          - name: output
+            workspace: git-source
+        params:
+          - name: url
+            value: $(params.REPO_URL)
+          - name: revision
+            value: $(params.PULL_BASE_SHA)
+          - name: deleteExisting
+            value: "true"
+    finally:
+      - name: report-success
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Succeeded", "Completed"]
+        taskRef:
+          resolver: git
+          params:
+            - name: url
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
+            - name: revision
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
+            - name: pathInRepo
+              value: task/gitea-set-status/0.1/gitea-set-status.yaml
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has succeeded"
+          - name: STATE
+            value: success
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+      - name: report-failure
+        when:
+          - input: "$(tasks.status)"
+            operator: in
+            values: ["Failed"]
+        taskRef:
+          resolver: git
+          params:
+            - name: url
+              value: https://code.fizz.buzz/mirror/catalog.git # mirror of https://github.com/tektoncd/catalog.git
+            - name: revision
+              value: df36b3853a5657fd883015cdbf07ad6466918acf
+            - name: pathInRepo
+              value: task/gitea-set-status/0.1/gitea-set-status.yaml
+        params:
+          - name: CONTEXT
+            value: "$(params.JOB_NAME)"
+          - name: REPO_FULL_NAME
+            value: "$(params.REPO_OWNER)/$(params.REPO_NAME)"
+          - name: GITEA_HOST_URL
+            value: code.fizz.buzz
+          - name: SHA
+            value: "$(tasks.fetch-repository.results.commit)"
+          - name: DESCRIPTION
+            value: "Build $(params.JOB_NAME) has failed"
+          - name: STATE
+            value: failure
+          - name: TARGET_URL
+            value: "https://tekton.fizz.buzz/#/namespaces/$(context.pipelineRun.namespace)/pipelineruns/$(context.pipelineRun.name)"
+    workspaces:
+      - name: git-source
+  workspaces:
+    - name: git-source
+      volumeClaimTemplate:
+        spec:
+          storageClassName: "nfs-client"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      subPath: source
+  params: []

.webhook_bridge/webhook_bridge.toml

@@ -0,0 +1,19 @@
+version = "0.0.1"
+
+[[push]]
+  name = "build"
+  source = "pipeline-build-homepage.yaml"
+  clone_uri = "git@code.fizz.buzz:talexander/homepage.git"
+  branches = [ "^v[0-9]+\\.[0-9]+\\.[0-9]+$" ]
+
+[[push]]
+  name = "build-staging"
+  source = "pipeline-build-homepage-staging.yaml"
+  clone_uri = "git@code.fizz.buzz:talexander/homepage.git"
+  skip_branches = [ "^v[0-9]+\\.[0-9]+\\.[0-9]+$" ]
+
+[[push]]
+  name = "semver"
+  source = "pipeline-semver.yaml"
+  clone_uri = "git@code.fizz.buzz:talexander/homepage.git"
+  branches = [ "^main$", "^master$" ]

docker/server/Dockerfile

@@ -1,10 +1,62 @@
-FROM harbor.fizz.buzz/private/natter:latest AS builder
+# syntax=docker/dockerfile:1
+ARG ALPINE_VERSION="3.20"
 
-COPY . /source
-RUN ls /source/
-RUN natter build --config /source/natter.toml
 
-FROM alpine:3.18 AS server
+
+FROM scratch AS private
+ADD git@code.fizz.buzz:talexander/homepage_private.git /homepage_private
+
+
+
+FROM scratch AS explorer
+ADD https://code.fizz.buzz/talexander/organic_ast_explorer.git /organic_ast_explorer
+
+
+
+FROM scratch AS organic
+ADD git@code.fizz.buzz:talexander/organic.git /organic
+
+
+
+FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS organic-build
+RUN apk add --no-cache musl-dev make bash
+RUN rustup target add wasm32-unknown-unknown
+RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked cargo install wasm-bindgen-cli
+COPY --link --from=organic /organic /organic
+WORKDIR /organic
+RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked make wasm
+
+
+
+FROM node:lts-alpine$ALPINE_VERSION AS explorer-build
+COPY --link --from=explorer /organic_ast_explorer /organic_ast_explorer
+COPY --link --from=organic-build /organic /organic
+WORKDIR /organic_ast_explorer
+RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/npmcache,sharing=locked npm set cache /npmcache && npm install
+RUN npm run release
+
+
+
+
+FROM rustlang/rust:nightly-alpine$ALPINE_VERSION AS natter-build
+RUN apk add --no-cache musl-dev
+ADD git@code.fizz.buzz:talexander/natter.git /natter
+WORKDIR /natter
+RUN --mount=type=tmpfs,target=/tmp --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked CARGO_TARGET_DIR=/target cargo build --profile release-lto
+
+
+
+FROM alpine:$ALPINE_VERSION AS natter
+COPY --link --from=natter-build /target/release-lto/natter /usr/bin/
+COPY --link . /source
+COPY --link --from=private /homepage_private/static /source/static/
+COPY --link --from=explorer-build /organic_ast_explorer/dist /source/static/organic/ast_explorer/
+RUN --network=none --mount=type=tmpfs,target=/tmp natter build --config /source/natter.toml
+
+
+
+
+FROM alpine:$ALPINE_VERSION AS server
 
 RUN apk add --no-cache bash nginx
 RUN addgroup web && adduser -D -G web web && install -d -D -o web -g web -m 700 /srv/http/public
@@ -12,6 +64,6 @@ RUN ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/
 
 COPY --chown=web:web docker/server/nginx.conf /srv/http
 COPY --chown=web:web docker/server/headers.include /srv/http
-COPY --from=builder --chown=web:web /source/output/ /srv/http/public/
+COPY --from=natter --chown=web:web /source/output/ /srv/http/public/
 
 ENTRYPOINT ["/usr/sbin/nginx", "-c", "/srv/http/nginx.conf", "-e", "stderr", "-g", "daemon off;"]

docker/server/Makefile

@@ -21,7 +21,7 @@ help:
 
 .PHONY: build
 build: ## Build the docker image.
-> docker build --tag $(IMAGE_NAME) --target=$(TARGET) --file Dockerfile ../../
+> docker build --ssh default --tag $(IMAGE_NAME) --target=$(TARGET) --file Dockerfile ../../
 
 .PHONY: push
 push: ## Push the docker image to a remote repository.

docker/server/nginx.conf

@@ -31,6 +31,9 @@ http {
         root /srv/http/public;
 
         location / {
+            try_files $uri $uri/ =404;
+            autoindex on;
+
             index index.html index.htm;
             if (-d $request_filename) {
                 rewrite [^/]$ $http_x_forwarded_proto://$http_host$uri/ redirect;
@@ -48,7 +51,19 @@ http {
             default_type text/plain;
         }
 
-        location ~ /\.well-known/(?<path>openpgpkey/[^/]+/hu/) {
+        location /.well-known/openpgpkey/hu/ {
+            alias /srv/http/public/well-known/openpgpkey/fizz.buzz/hu/;
+            default_type        "application/octet-stream";
+            add_header          Access-Control-Allow-Origin * always;
+        }
+
+        location /.well-known/openpgpkey/policy {
+            alias /srv/http/public/well-known/openpgpkey/fizz.buzz/policy;
+            default_type        "application/octet-stream";
+            add_header          Access-Control-Allow-Origin * always;
+        }
+
+        location ~ /\.well-known/(?<path>openpgpkey/[^/]+/hu/.*) {
             alias /srv/http/public/well-known/$path;
             default_type        "application/octet-stream";
             add_header          Access-Control-Allow-Origin * always;

pages/index.org

@@ -1,24 +0,0 @@
-#+OPTIONS: html-postamble:nil
-#+date: <2023-12-23 Sat>
-#+author: Tom Alexander
-#+email:
-#+language: en
-#+select_tags: export
-#+exclude_tags: noexport
-
-My dev blog will appear here as soon as I finish writing articles worthy of publishing. In the mean time, please check out my repos at [[https://code.fizz.buzz/explore/repos][code.fizz.buzz]].
-
-Links:
-- My personal repos: [[https://code.fizz.buzz/explore/repos][code.fizz.buzz]]
-- LinkedIn: https://www.linkedin.com/in/tom-alexander-b6a18216/
-- GitHub: https://github.com/tomalexander
-- Resume: https://fizz.buzz/resume.pdf
-- PGP Key: https://fizz.buzz/pgp.asc
-
-* Why is your website the way it is?
-
-I used to have a developer blog hosted at this domain. I quickly developed an appreciation for the power of org-mode for writing the content of the blog but I grew tired of inconsistent build results. The static site generators at the time would function by calling out to emacs itself to parse the org-mode and export HTML which meant that updates to emacs, my elisp packages, or the static site generator could cause compatibility issues. This often lead to things like escaping issues in old blog posts going unnoticed.
-
-To solve the issue, and to seize the opportunity to gain more experience in Rust, I decided to write my own static site generator that would not depend on outside tools. So far I have written [[https://code.fizz.buzz/talexander/duster][the template engine]] and I am in the process of writing [[https://code.fizz.buzz/talexander/organic][an org-mode parser]]. When that is done, it should just be a matter of tying those two together with some minor glue to make a static site generator to create the new version of this site. Until that is done, I am using this hastily thrown-together manually-written html file as a placeholder.
-
-That isn't to say that there are no exciting things hosted on this server, just not at the root domain. For example, this server is running kubernetes that I set up manually following [[https://github.com/kelseyhightower/kubernetes-the-hard-way][kubernetes-the-hard-way]] in a bunch of [[https://man.freebsd.org/cgi/man.cgi?bhyve][bhyve VMs]] that I networked together using [[https://man.freebsd.org/cgi/man.cgi?netgraph(4)][netgraph]]. On it I host my own [[https://www.powerdns.com/][PowerDNS]] server as the authoratative DNS server for fizz.buzz. It is integrated with [[https://cert-manager.io/][cert-manager]] and [[https://github.com/kubernetes-sigs/external-dns][ExternalDNS]] so Ingresses/LoadBalancers on my cluster automatically get valid TLS certificates and update the DNS records. I have a fully open-source self-hosted gitops workflow where a commit to a git repo I'm hosting in [[https://code.fizz.buzz/][gitea]], triggers a [[https://tekton.dev/][tekton pipeline]] through [[https://github.com/jenkins-x/lighthouse][lighthouse]] to build a docker image with [[https://github.com/GoogleContainerTools/kaniko][kaniko]], which gets pushed to my self-hosted [[https://goharbor.io/][harbor]] instance, which then gets deployed to my cluster via [[https://fluxcd.io/][flux]]. The end result is I make a commit to a repo and the result is deployed to my website in minutes.

posts/2025/02/08/graph-docker-memory/files/graph.gnuplot

@@ -0,0 +1,33 @@
+set terminal svg background '#FFFFFF'
+set title 'Docker Memory Usage'
+set xdata time
+set timefmt '%s'
+set format x '%tH:%tM:%tS'
+# Please note this is in SI units (base 10), not IEC (base 2). So, for example, this would show a Gigabyte, not a Gibibyte.
+set format y '%.0s%cB'
+set datafile separator "|"
+
+plot "-" using 1:2 title 'exciting\_bohr' with lines, "-" using 1:2 title 'jovial\_chandrasekhar' with lines
+0|512000
+4|512000
+9|512000
+13|4866441
+18|3166699
+23|3128950
+27|3128950
+32|3128950
+35|32547799
+40|4329570
+e
+0|528384
+5|516096
+9|561152
+14|561152
+18|561152
+23|561152
+28|8568963
+32|8528068
+37|8528068
+40|8528068
+45|8528068
+e

posts/2025/02/08/graph-docker-memory/files/graph.svg

@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="utf-8"  standalone="no"?>
+<svg 
+ width="600" height="480"
+ viewBox="0 0 600 480"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+>
+
+<title>Gnuplot</title>
+<desc>Produced by GNUPLOT 6.0 patchlevel 2 </desc>
+
+<g id="gnuplot_canvas">
+
+<rect x="0" y="0" width="600" height="480" fill="#ffffff"/>
+<defs>
+
+	<circle id='gpDot' r='0.5' stroke-width='0.5' stroke='currentColor'/>
+	<path id='gpPt0' stroke-width='0.222' stroke='currentColor' d='M-1,0 h2 M0,-1 v2'/>
+	<path id='gpPt1' stroke-width='0.222' stroke='currentColor' d='M-1,-1 L1,1 M1,-1 L-1,1'/>
+	<path id='gpPt2' stroke-width='0.222' stroke='currentColor' d='M-1,0 L1,0 M0,-1 L0,1 M-1,-1 L1,1 M-1,1 L1,-1'/>
+	<rect id='gpPt3' stroke-width='0.222' stroke='currentColor' x='-1' y='-1' width='2' height='2'/>
+	<rect id='gpPt4' stroke-width='0.222' stroke='currentColor' fill='currentColor' x='-1' y='-1' width='2' height='2'/>
+	<circle id='gpPt5' stroke-width='0.222' stroke='currentColor' cx='0' cy='0' r='1'/>
+	<use xlink:href='#gpPt5' id='gpPt6' fill='currentColor' stroke='none'/>
+	<path id='gpPt7' stroke-width='0.222' stroke='currentColor' d='M0,-1.33 L-1.33,0.67 L1.33,0.67 z'/>
+	<use xlink:href='#gpPt7' id='gpPt8' fill='currentColor' stroke='none'/>
+	<use xlink:href='#gpPt7' id='gpPt9' stroke='currentColor' transform='rotate(180)'/>
+	<use xlink:href='#gpPt9' id='gpPt10' fill='currentColor' stroke='none'/>
+	<use xlink:href='#gpPt3' id='gpPt11' stroke='currentColor' transform='rotate(45)'/>
+	<use xlink:href='#gpPt11' id='gpPt12' fill='currentColor' stroke='none'/>
+	<path id='gpPt13' stroke-width='0.222' stroke='currentColor' d='M0,1.330 L1.265,0.411 L0.782,-1.067 L-0.782,-1.076 L-1.265,0.411 z'/>
+	<use xlink:href='#gpPt13' id='gpPt14' fill='currentColor' stroke='none'/>
+	<filter id='textbox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
+	  <feFlood flood-color='#FFFFFF' flood-opacity='1' result='bgnd'/>
+	  <feComposite in='SourceGraphic' in2='bgnd' operator='atop'/>
+	</filter>
+	<filter id='greybox' filterUnits='objectBoundingBox' x='0' y='0' height='1' width='1'>
+	  <feFlood flood-color='lightgrey' flood-opacity='1' result='grey'/>
+	  <feComposite in='SourceGraphic' in2='grey' operator='atop'/>
+	</filter>
+</defs>
+<g fill="none" color="#FFFFFF" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,444.00 L63.53,444.00 M574.82,444.00 L565.82,444.00  '/>	<g transform="translate(46.14,447.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >0B</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,388.29 L63.53,388.29 M574.82,388.29 L565.82,388.29  '/>	<g transform="translate(46.14,392.19)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >5MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,332.57 L63.53,332.57 M574.82,332.57 L565.82,332.57  '/>	<g transform="translate(46.14,336.47)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >10MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,276.86 L63.53,276.86 M574.82,276.86 L565.82,276.86  '/>	<g transform="translate(46.14,280.76)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >15MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,221.15 L63.53,221.15 M574.82,221.15 L565.82,221.15  '/>	<g transform="translate(46.14,225.05)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >20MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,165.44 L63.53,165.44 M574.82,165.44 L565.82,165.44  '/>	<g transform="translate(46.14,169.34)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >25MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,109.72 L63.53,109.72 M574.82,109.72 L565.82,109.72  '/>	<g transform="translate(46.14,113.62)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >30MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,54.01 L63.53,54.01 M574.82,54.01 L565.82,54.01  '/>	<g transform="translate(46.14,57.91)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >35MB</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,444.00 L54.53,435.00 M54.53,54.01 L54.53,63.01  '/>	<g transform="translate(54.53,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:00</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M112.34,444.00 L112.34,435.00 M112.34,54.01 L112.34,63.01  '/>	<g transform="translate(112.34,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:05</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M170.15,444.00 L170.15,435.00 M170.15,54.01 L170.15,63.01  '/>	<g transform="translate(170.15,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:10</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M227.96,444.00 L227.96,435.00 M227.96,54.01 L227.96,63.01  '/>	<g transform="translate(227.96,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:15</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M285.77,444.00 L285.77,435.00 M285.77,54.01 L285.77,63.01  '/>	<g transform="translate(285.77,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:20</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M343.58,444.00 L343.58,435.00 M343.58,54.01 L343.58,63.01  '/>	<g transform="translate(343.58,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:25</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M401.39,444.00 L401.39,435.00 M401.39,54.01 L401.39,63.01  '/>	<g transform="translate(401.39,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:30</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M459.20,444.00 L459.20,435.00 M459.20,54.01 L459.20,63.01  '/>	<g transform="translate(459.20,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:35</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M517.01,444.00 L517.01,435.00 M517.01,54.01 L517.01,63.01  '/>	<g transform="translate(517.01,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:40</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M574.82,444.00 L574.82,435.00 M574.82,54.01 L574.82,63.01  '/>	<g transform="translate(574.82,465.90)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >0:00:45</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,54.01 L54.53,444.00 L574.82,444.00 L574.82,54.01 L54.53,54.01 Z  '/></g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+	<g id="gnuplot_plot_1" ><title>exciting_bohr</title>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<g transform="translate(507.09,75.91)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >exciting_bohr</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='rgb(148,   0, 211)'  d='M515.48,72.01 L558.04,72.01 M54.53,438.30 L100.78,438.30 L158.59,438.30 L204.84,389.78 L262.65,408.71 L320.46,409.14
+		L366.70,409.14 L424.51,409.14 L459.20,81.33 L517.01,395.76  '/></g>
+	</g>
+	<g id="gnuplot_plot_2" ><title>jovial_chandrasekhar</title>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<g transform="translate(507.09,93.91)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="end">
+		<text><tspan font-family="Arial" >jovial_chandrasekhar</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='rgb(  0, 158, 115)'  d='M515.48,90.01 L558.04,90.01 M54.53,438.11 L112.34,438.25 L158.59,437.75 L216.40,437.75 L262.65,437.75 L320.46,437.75
+		L378.27,348.52 L424.51,348.98 L482.32,348.98 L517.01,348.98 L574.82,348.98  '/></g>
+	</g>
+<g fill="none" color="#FFFFFF" stroke="rgb(  0, 158, 115)" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="2.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+<g fill="none" color="black" stroke="black" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<path stroke='black'  d='M54.53,54.01 L54.53,444.00 L574.82,444.00 L574.82,54.01 L54.53,54.01 Z  '/></g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+	<g transform="translate(314.67,30.91)" stroke="none" fill="black" font-family="Arial" font-size="12.00"  text-anchor="middle">
+		<text><tspan font-family="Arial" >Docker Memory Usage</tspan></text>
+	</g>
+</g>
+<g fill="none" color="black" stroke="currentColor" stroke-width="1.00" stroke-linecap="butt" stroke-linejoin="miter">
+</g>
+</g>
+</svg>
+

posts/2025/02/08/graph-docker-memory/index.org

@@ -0,0 +1,262 @@
+#+OPTIONS: html-postamble:nil
+#+title: Graph Docker Container Memory Usage with Gnuplot
+#+date: <2025-02-08 Sat>
+#+author: Tom Alexander
+#+email:
+#+language: en
+#+select_tags: export
+#+exclude_tags: noexport
+
+Sometimes it can be useful to build a graph of docker memory usage over time. For example, I was recently working on reducing the maximum memory of a long-running script. There certainly are heavy and complex options out there like setting up Prometheus and [[https://docs.docker.com/engine/daemon/prometheus/][configuring docker to export metrics to it]] but I threw together a small python script, using only the python standard library, that outputs gnuplot code to render a graph.
+
+
+* Usage
+Invoke the python script before starting any docker containers. Then, once a docker container is started, the script will start recording memory usage. Any additional docker containers that are started while the script is running will also get recorded. When no docker containers are left, the script will export gnuplot code over stdout that can then be rendered into a graph.
+
+Each container will get its own line on the graph. All containers will have their start time aligned with the left-hand side of the graph as if they had started at the same time (so the X-axis it the number of seconds the docker container has been running, as opposed to the wall time).
+
+If you'd like, you can insert a horizontal line at whatever memory quantity you'd like by uncommenting the src_python[:exports code]{horizontal_lines} array below. This can be useful for showing a maximum limit like the paltry 32GiB offered by Cloud Run.
+
+* Example Invocation
+#+begin_src bash
+  $ ./graph_docker_memory.py | gnuplot > graph.svg
+  INFO:root:Waiting for a docker container to exist to start recording.
+  INFO:root:Recorded stat jovial_chandrasekhar: 528384 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 528384 bytes
+  INFO:root:Recorded stat exciting_bohr: 512000 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 516096 bytes
+  INFO:root:Recorded stat exciting_bohr: 512000 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 561152 bytes
+  INFO:root:Recorded stat exciting_bohr: 512000 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 561152 bytes
+  INFO:root:Recorded stat exciting_bohr: 4866441 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 561152 bytes
+  INFO:root:Recorded stat exciting_bohr: 3166699 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 561152 bytes
+  INFO:root:Recorded stat exciting_bohr: 3128950 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 8568963 bytes
+  INFO:root:Recorded stat exciting_bohr: 3128950 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 8528068 bytes
+  INFO:root:Recorded stat exciting_bohr: 3128950 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 8528068 bytes
+  INFO:root:Recorded stat exciting_bohr: 32547799 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 8528068 bytes
+  INFO:root:Recorded stat exciting_bohr: 4329570 bytes
+  INFO:root:Recorded stat jovial_chandrasekhar: 8528068 bytes
+#+end_src
+
+You can also throw src_bash[:exports code]{tee} in there to save the gnuplot file to make manual adjustments or to render in some other fashion:
+#+begin_src bash
+  ./graph_docker_memory.py | tee graph.gnuplot | gnuplot > graph.svg
+#+end_src
+* Output
+The output from the above run would be:
+
+[[./files/graph.svg]]
+
+And the gnuplot source:
+#+begin_src gnuplot
+  set terminal svg background '#FFFFFF'
+  set title 'Docker Memory Usage'
+  set xdata time
+  set timefmt '%s'
+  set format x '%tH:%tM:%tS'
+  # Please note this is in SI units (base 10), not IEC (base 2). So, for example, this would show a Gigabyte, not a Gibibyte.
+  set format y '%.0s%cB'
+  set datafile separator "|"
+
+  plot "-" using 1:2 title 'exciting\_bohr' with lines, "-" using 1:2 title 'jovial\_chandrasekhar' with lines
+  0|512000
+  4|512000
+  9|512000
+  13|4866441
+  18|3166699
+  23|3128950
+  27|3128950
+  32|3128950
+  35|32547799
+  40|4329570
+  e
+  0|528384
+  5|516096
+  9|561152
+  14|561152
+  18|561152
+  23|561152
+  28|8568963
+  32|8528068
+  37|8528068
+  40|8528068
+  45|8528068
+  e
+#+end_src
+* The script
+#+begin_src python
+  #!/usr/bin/env python
+  from __future__ import annotations
+
+  import json
+  import logging
+  import re
+  import subprocess
+  from dataclasses import dataclass
+  from datetime import datetime
+  from time import sleep
+  from typing import Collection, Final, NewType, Tuple
+
+  ContainerId = NewType("ContainerId", str)
+  ContainerName = NewType("ContainerName", str)
+
+  SAMPLE_INTERVAL_SECONDS: Final[int] = 2
+
+
+  @dataclass
+  class Sample:
+      instant: datetime
+      stats: dict[ContainerId, Stats]
+
+
+  @dataclass
+  class Stats:
+      memory_usage_bytes: int
+
+
+  def main():
+      logging.basicConfig(level=logging.INFO)
+      samples: list[Sample] = []
+      labels: dict[ContainerId, ContainerName] = {}
+      first_pass = True
+      # First wait for any docker container to exist.
+      while True:
+          sample, labels_in_sample = take_sample()
+          if labels_in_sample:
+              break
+          if first_pass:
+              first_pass = False
+              logging.info("Waiting for a docker container to exist to start recording.")
+          sleep(1)
+      # And then record memory until no containers exist.
+      while True:
+          sample, labels_in_sample = take_sample()
+          if not labels_in_sample:
+              break
+          samples.append(sample)
+          labels = {**labels, **labels_in_sample}
+          sleep(SAMPLE_INTERVAL_SECONDS)
+      if labels:
+          # Draws a red horizontal line at 32 GiB since that is the memory limit for cloud run.
+          write_plot(
+              samples,
+              labels,
+              # horizontal_lines=[(32 * 1024**3, "red", "Cloud Run Max Memory")],
+          )
+
+
+  def write_plot(
+      samples: Collection[Sample],
+      labels: dict[ContainerId, ContainerName],
+      ,*,
+      horizontal_lines: Collection[Tuple[int, str, str | None]] = [],
+  ):
+      starting_time_per_container = {
+          container_id: min(
+              (sample.instant for sample in samples if container_id in sample.stats)
+          )
+          for container_id in labels.keys()
+      }
+      print(
+          """set terminal svg background '#FFFFFF'
+  set title 'Docker Memory Usage'
+  set xdata time
+  set timefmt '%s'
+  set format x '%tH:%tM:%tS'
+  # Please note this is in SI units (base 10), not IEC (base 2). So, for example, this would show a Gigabyte, not a Gibibyte.
+  set format y '%.0s%cB'
+  set datafile separator "|"
+  """
+      )
+      for y_value, color, label in horizontal_lines:
+          print(
+              f'''set arrow from graph 0, first {y_value} to graph 1, first {y_value} nohead linewidth 2 linecolor rgb "{color}"'''
+          )
+          if label is not None:
+              print(f"""set label "{label}" at graph 0, first {y_value} offset 1,-0.5""")
+
+      # Include the horizontal lines in the range
+      if len(horizontal_lines) > 0:
+          print(f"""set yrange [*:{max(x[0] for x in horizontal_lines)}<*]""")
+      line_definitions = ", ".join(
+          [
+              f""""-" using 1:2 title '{gnuplot_escape(name)}' with lines"""
+              for container_id, name in sorted(labels.items())
+          ]
+      )
+      print("plot", line_definitions)
+      for container_id in sorted(labels.keys()):
+          start_time = int(starting_time_per_container[container_id].timestamp())
+          for sample in sorted(samples, key=lambda x: x.instant):
+              if container_id in sample.stats:
+                  print(
+                      "|".join(
+                          [
+                              str(int((sample.instant).timestamp()) - start_time),
+                              str(sample.stats[container_id].memory_usage_bytes),
+                          ]
+                      )
+                  )
+          print("e")
+
+
+  def gnuplot_escape(inp: str) -> str:
+      out = ""
+      for c in inp:
+          if c == "_":
+              out += "\\"
+          out += c
+      return out
+
+
+  def take_sample() -> Tuple[Sample, dict[ContainerId, ContainerName]]:
+      labels: dict[ContainerId, ContainerName] = {}
+      stats: dict[ContainerId, Stats] = {}
+      docker_inspect = subprocess.run(
+          ["docker", "stats", "--no-stream", "--no-trunc", "--format", "json"],
+          stdout=subprocess.PIPE,
+      )
+      for container_stat in (
+          json.loads(l) for l in docker_inspect.stdout.decode("utf8").splitlines()
+      ):
+          if not container_stat["ID"]:
+              # When containers are starting up, they sometimes have no ID and "--" as the name.
+              continue
+          labels[ContainerId(container_stat["ID"])] = ContainerName(
+              container_stat["Name"]
+          )
+          memory_usage = parse_mem_usage(container_stat["MemUsage"])
+          stats[ContainerId(container_stat["ID"])] = Stats(
+              memory_usage_bytes=memory_usage
+          )
+      for container_id, container_stat in stats.items():
+          logging.info(
+              f"Recorded stat {labels[container_id]}: {container_stat.memory_usage_bytes} bytes"
+          )
+      return Sample(instant=datetime.now(), stats=stats), labels
+
+
+  def parse_mem_usage(mem_usage: str) -> int:
+      parsed_mem_usage = re.match(
+          r"(?P<number>[0-9]+\.?[0-9]*)(?P<unit>[^\s]+)", mem_usage
+      )
+      if parsed_mem_usage is None:
+          raise Exception(f"Invalid Mem Usage: {mem_usage}")
+      number = float(parsed_mem_usage.group("number"))
+      unit = parsed_mem_usage.group("unit")
+      for multiplier, identifier in enumerate(["B", "KiB", "MiB", "GiB", "TiB"]):
+          if unit == identifier:
+              return int(number * (1024**multiplier))
+      raise Exception(f"Unrecognized unit: {unit}")
+
+
+  if __name__ == "__main__":
+      main()
+#+end_src