machine_setup/nix/kubernetes/README.org

• To-do
  • Perhaps use overlay for /etc for speedup
  • read https://nixos.org/manual/nixos/stable/
  • Performance for mini pc
• Bootstrap
  • Install cilium
  • Install flux

To-do

Perhaps use overlay for /etc for speedup

  system.etc.overlay.enable = true;

read https://nixos.org/manual/nixos/stable/

Performance for mini pc

  security.pam.loginLimits = [
    { domain = "@users"; item = "rtprio"; type = "-"; value = 1; }
  ];

Bootstrap

Install cilium

  # nix shell nixpkgs#cilium-cli
  nix shell 'nixpkgs#kubernetes-helm'

  helm repo add cilium https://helm.cilium.io/
  helm template --dry-run=client cilium cilium/cilium --version 1.18.5 --namespace kube-system \
       --set kubeProxyReplacement=true \
       --set ipam.mode=kubernetes \
       --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \
       --set k8sServicePort=6443 \
       --set ipv6.enabled=true \
       --set ipv4.enabled=false \
       --set enableIPv4BIGTCP=false \
       --set enableIPv6BIGTCP=true \
       --set routingMode=native \
       --set ipv4NativeRoutingCIDR=10.0.0.0/8 \
       --set ipv6NativeRoutingCIDR=fd00::/100

  kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose
  kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement

  #      --set hostFirewall.enabled=true
  # routingMode=native

       # --set ipv4-native-routing-cidr=10.0.0.0/8 \
       # --set ipv6-native-routing-cidr=fd00::/100
  # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \
  # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \

Install flux

  nix shell 'nixpkgs#fluxcd'

  flux bootstrap git \
    --url=ssh://git@<host>/<org>/<repository> \
    --branch=main \
    --private-key-file=<path/to/private.key> \
    --password=<key-passphrase> \
    --path=clusters/my-cluster

  nix shell 'nixpkgs#kubernetes-helm'

  helm template --dry-run=server flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-operator \
    --namespace flux-system \
    --create-namespace

  apiVersion: fluxcd.controlplane.io/v1
  kind: FluxInstance
  metadata:
    name: flux
    namespace: flux-system
    annotations:
      fluxcd.controlplane.io/reconcileEvery: "1h"
      fluxcd.controlplane.io/reconcileTimeout: "5m"
  spec:
    distribution:
      version: "2.x"
      registry: "ghcr.io/fluxcd"
      artifact: "oci://ghcr.io/controlplaneio-fluxcd/flux-operator-manifests"
    components:
      - source-controller
      - kustomize-controller
      - helm-controller
      - notification-controller
      - image-reflector-controller
      - image-automation-controller
    cluster:
      type: kubernetes
      size: medium
      multitenant: false
      networkPolicy: true
      domain: "cluster.local"
    kustomize:
      patches:
        - target:
            kind: Deployment
          patch: |
            - op: replace
              path: /spec/template/spec/nodeSelector
              value:
                kubernetes.io/os: linux
            - op: add
              path: /spec/template/spec/tolerations
              value:
                - key: "CriticalAddonsOnly"
                  operator: "Exists"
    sync:
      kind: OCIRepository
      url: "oci://ghcr.io/my-org/my-fleet-manifests"
      ref: "latest"
      path: "clusters/my-cluster"
      pullSecret: "ghcr-auth"

  apiVersion: fluxcd.controlplane.io/v1
  kind: FluxInstance
  metadata:
    name: flux
    namespace: flux-system
  spec:
    distribution:
      version: "2.7.x"
      registry: "ghcr.io/fluxcd"
    sync:
      kind: GitRepository
      url: "ssh://git@10.215.1.210:22/repos/mrmanager"
      ref: "refs/heads/nix"
      path: "clusters/my-cluster"
      pullSecret: "flux-system"

  flux create secret git flux-system \
    --url=https://gitlab.com/my-org/my-fleet.git \
    --username=git \
    --password=$GITLAB_TOKEN