From 96112bd40a43de2df6f23d5352139b19fad10edc Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 29 Dec 2025 19:11:55 -0500 Subject: [PATCH] Build the cilium manifest automatically in nix. --- nix/kubernetes/README.org | 38 +- nix/kubernetes/hosts/controller0/default.nix | 3 - nix/kubernetes/hosts/controller1/default.nix | 3 - nix/kubernetes/hosts/controller2/default.nix | 3 - nix/kubernetes/hosts/worker0/default.nix | 3 - nix/kubernetes/hosts/worker1/default.nix | 3 - nix/kubernetes/hosts/worker2/default.nix | 3 - .../files/manifests/cilium.yaml | 1687 ----------------- .../keys/package/bootstrap-script/package.nix | 4 +- .../keys/package/cilium-manifest/package.nix | 70 + nix/kubernetes/keys/scope.nix | 1 + nix/kubernetes/roles/network/default.nix | 11 +- 12 files changed, 83 insertions(+), 1746 deletions(-) delete mode 100644 nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml create mode 100644 nix/kubernetes/keys/package/cilium-manifest/package.nix diff --git a/nix/kubernetes/README.org b/nix/kubernetes/README.org index bace10e3..70f8a687 100644 --- a/nix/kubernetes/README.org +++ b/nix/kubernetes/README.org @@ -10,43 +10,11 @@ { domain = "@users"; item = "rtprio"; type = "-"; value = 1; } ]; #+end_src -* Bootstrap -** Install cilium +* Healthcheck +** Check cilium status #+begin_src bash - # nix shell nixpkgs#cilium-cli - nix shell 'nixpkgs#kubernetes-helm' - - helm repo add cilium https://helm.cilium.io/ - helm template --dry-run=client cilium cilium/cilium --version 1.18.5 --namespace kube-system \ - --set kubeProxyReplacement=true \ - --set ipam.mode=kubernetes \ - --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ - --set k8sServicePort=6443 \ - --set ipv6.enabled=true \ - --set ipv4.enabled=true \ - --set enableIPv6Masquerade=false - # --set enableIPv4BIGTCP=true \ - # --set enableIPv6BIGTCP=true - # --set routingMode=native \ - # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ - # --set ipv6NativeRoutingCIDR=fd00::/100 - - kubec - tl -n kube-system exec ds/cilium -- cilium-dbg status --verbose + kubectl -n kube-system exec ds/cilium -- cilium-dbg status --verbose kubectl -n kube-system exec ds/cilium -- cilium-dbg status | grep KubeProxyReplacement - - # --set hostFirewall.enabled=true - # routingMode=native - - # --set ipv4-native-routing-cidr=10.0.0.0/8 \ - # --set ipv6-native-routing-cidr=fd00::/100 - # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ - # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ - - - # --set encryption.enabled=true \ - # --set encryption.type=wireguard - # --set encryption.nodeEncryption=true #+end_src ** Install flux #+begin_src bash diff --git a/nix/kubernetes/hosts/controller0/default.nix b/nix/kubernetes/hosts/controller0/default.nix index 4521b7e1..bb32f505 100644 --- a/nix/kubernetes/hosts/controller0/default.nix +++ b/nix/kubernetes/hosts/controller0/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/controller1/default.nix b/nix/kubernetes/hosts/controller1/default.nix index 3af358d9..f4b5c87e 100644 --- a/nix/kubernetes/hosts/controller1/default.nix +++ b/nix/kubernetes/hosts/controller1/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/controller2/default.nix b/nix/kubernetes/hosts/controller2/default.nix index a25ddae0..e9c318ae 100644 --- a/nix/kubernetes/hosts/controller2/default.nix +++ b/nix/kubernetes/hosts/controller2/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker0/default.nix b/nix/kubernetes/hosts/worker0/default.nix index 59f80f37..6e4570f5 100644 --- a/nix/kubernetes/hosts/worker0/default.nix +++ b/nix/kubernetes/hosts/worker0/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker1/default.nix b/nix/kubernetes/hosts/worker1/default.nix index 6f951f0a..bb56099e 100644 --- a/nix/kubernetes/hosts/worker1/default.nix +++ b/nix/kubernetes/hosts/worker1/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/hosts/worker2/default.nix b/nix/kubernetes/hosts/worker2/default.nix index d595c0df..ca831440 100644 --- a/nix/kubernetes/hosts/worker2/default.nix +++ b/nix/kubernetes/hosts/worker2/default.nix @@ -51,9 +51,6 @@ address = "2620:11f:7001:7:ffff:ffff:0ad7:0101"; inherit interface; }; - nameservers = [ - "10.215.1.1" - ]; dhcpcd.enable = lib.mkForce false; useDHCP = lib.mkForce false; diff --git a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml b/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml deleted file mode 100644 index 585612e8..00000000 --- a/nix/kubernetes/keys/package/bootstrap-script/files/manifests/cilium.yaml +++ /dev/null @@ -1,1687 +0,0 @@ ---- -# Source: cilium/templates/cilium-secrets-namespace.yaml -apiVersion: v1 -kind: Namespace -metadata: - name: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium - annotations: ---- -# Source: cilium/templates/cilium-agent/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-envoy/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-envoy" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-ca-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: cilium-ca - namespace: kube-system -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - ca.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBekhjNGk4RS9iVDJpazlQemdGUHllaExyL1FWRG15eElONWVQSFFRQ0o1bnV0WThwCkUxY0hnZVcyOEhiYXJWd1N2UkFxTm1HbFdhZmQ1bXJIMjdRNmE4aVZtQ3A2WHhlbDRTK2FZUVh5K2RYMlV3TngKS2U5UTVQOFl4VDFjRlpFdEFubXJQc0tHck5vWHQ3UDQvMHRwUlU1b1pLZzBCUCt0Z2psYlB3d01LbnhUTlozLwpQa2cyNDNGdmxFQ1h1dlVzRDcvcHBYZFM0bDRFc1hqZG1tSGl6RnpUNktwNVViYnFLbGloeWh4ZEs4ZWVweHpjCjN3clBBNWVWVzY3QlJXOHU1NWVhUjJ1aThkQ2ZIMGdnWG11Ry9IYkkyTERLTkNPUWp0RUZOV1VoVDNPM29GblQKL1kvTjdpSXNVaGxPSHJ2STVDSmFoSldjbjFUR3dRbllrYXErcXdJREFRQUJBb0lCQUEySWtYNEVWWVRZUU5VQwpwYnhzQ09ZL1U4NlVrSG00TGFFODcxSTZmQjhyTXNBWmNMZ2xhNHRjMGM4dlhzVTBNZlF1aUNndENZMSs3V1Q0CjB2amVBRmRiYyt5SHhxMStFVndCN3NIV3ROTXMzZTdrL3JkQWt1WUZZL1dmN3RDT0VDREI5UzJiUmJ1ZmZIbUwKblF3REpJYkpiRDRReTRaZ215bVpUeVlmVDVkb3hTSEJVMU9GMkFPTlg1VkYya1oySDI5dVNsMlgydm9jMlZidQowUlMvZFp6VExkcTRocUFPQ1VsbER6Nzk1U29aVU4zdmsranBSOGdxN1J2WTV4MzFDdmZQTENKNFNGQ25WSWtEClZvcnNCRW51V3ZaN0tBc051Tms3ZGMvYys0STk3d3cxNEt2NWVXV3p2ak05c1RFZWF1M3liRlluTmFTMUwzbjUKR3ZzeERERUNnWUVBMGJWNEJwS0xhM2g1c3BpbHZ0UlpSMVV3WndpZG13REMxZVB6dEZHSmx0SUg1b1I0T1BaVApzWjR2ak9WU21MUndFQ1VGdkNIaWZpV0ljTjd0NG5oQ1VCNmpJQXh5aS8wU1BSU1pkN2dtMXdYNGZyWVRVTkJtCk1DZ1krZlNLbjN2R3o2ODNlbUlFbDYyOFl5amt0aWYzQXdjeHVwdk9lSjVqQVBaekNoTG8ycE1DZ1lFQStabDIKMVp1Ym10RUI5T3pJZmlhRUV3R1k5TXdkKzNKL3daUU9LUWd6SGZBRU1DQnpGcnZZMWNvWkFpeWt3NWtoSTEwWQpZRlpFQlVhdFF3L3JpRFYvR1pDRkJyRHBYTFVjeFdMcTZoTGRtYmVZZHdXVXUvakdsSTUrYnhpNkVhUnlDcys4ClpUZVFNSDNLWVFJZ2RmQ3J3N0VTSEp1OU5ualFpMmhOZkt5STRva0NnWUVBaExrN2U3MW42OTdWOXdqUmJkTnIKcGMvdTBHY05pTHFVbFZhdU9TT1oxeENhMjNSZnVuMThQdUFFN0VGL1l4SmdFbmU4QjNQU3ExQUo2SlhQTFJRNwp0QVdQN0lxMFBKRXc1K09QdGN1aEdWbTRDa29tTTNHU0cweGxjbDBwRndMNXN4d01HckxLZ1V6OS9DdzNoR29LCjFhbkorWGIrMkN3Mk1MZkoyMGhZZzIwQ2dZRUE3VWhlZDMyQjlURGpPbE5yMnJtRTc0aWlQMzVZdG5WSVRPQVMKZ2lHQWJ1S0JLTHVBamNrd3Z6VnNodXVvQ3libElQaGN0eURyYzFTWVhGdWpIdzgwY0RvNnJIeThyTnlrcUdWOApTK0I1ZUt2WUxyWklpbFpiZWxqb2kzY25WS1JQb2tXUXBXeW9EK0ZWNXRrZHdPRjJlUWc1M3FhVHYxZ2xjWkpMCnI4MWFLSGtDZ1lBSXVxaDZzOU9RRGpjckVMeklJZENiZUxxLzhEdWUzcEZjbmFIWjgxemcwY0oxNnM1MU53aHQKaUQxWC9oRS9sdkp1VTFjTEFpNlJLZEk3UXIwL0x2SkFDSTlCakp6VVVYS1hQZ3JJZU44UUxCOWNhK2lka2FheApIWmNzdnBSN1g1b2cvTVpldWVTUk1OaU9wdVJSZ0xlYlh5R29lMkhmbE05STUvazJERUU5bmc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= ---- -# Source: cilium/templates/hubble/tls-helm/server-secret.yaml -apiVersion: v1 -kind: Secret -metadata: - name: hubble-server-certs - namespace: kube-system -type: kubernetes.io/tls -data: - ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURGRENDQWZ5Z0F3SUJBZ0lSQU53OW1DamlKdysveDFOQTJ6Sm5zc0F3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTRNVEl5T0RJeQpOVEF3TVZvd0ZERVNNQkFHQTFVRUF4TUpRMmxzYVhWdElFTkJNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUF6SGM0aThFL2JUMmlrOVB6Z0ZQeWVoTHIvUVZEbXl4SU41ZVBIUVFDSjVudXRZOHAKRTFjSGdlVzI4SGJhclZ3U3ZSQXFObUdsV2FmZDVtckgyN1E2YThpVm1DcDZYeGVsNFMrYVlRWHkrZFgyVXdOeApLZTlRNVA4WXhUMWNGWkV0QW5tclBzS0dyTm9YdDdQNC8wdHBSVTVvWktnMEJQK3RnamxiUHd3TUtueFROWjMvClBrZzI0M0Z2bEVDWHV2VXNENy9wcFhkUzRsNEVzWGpkbW1IaXpGelQ2S3A1VWJicUtsaWh5aHhkSzhlZXB4emMKM3dyUEE1ZVZXNjdCUlc4dTU1ZWFSMnVpOGRDZkgwZ2dYbXVHL0hiSTJMREtOQ09RanRFRk5XVWhUM08zb0ZuVAovWS9ON2lJc1VobE9IcnZJNUNKYWhKV2NuMVRHd1FuWWthcStxd0lEQVFBQm8yRXdYekFPQmdOVkhROEJBZjhFCkJBTUNBcVF3SFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUYKTUFNQkFmOHdIUVlEVlIwT0JCWUVGRFc0WXJXYkN4c0ZlQU9nT1Vsb1A1bUllVVgrTUEwR0NTcUdTSWIzRFFFQgpDd1VBQTRJQkFRQ0VOdnBiblI4TUtaMWpBVWRlMU55elJueUhjemR2T0Jla1F1YVFkS01hOWhFMk12Y0hTQ05qClBwNUNOenNXWk5FMW5RekF6RFNDcEV5Z0lpRm5mNitYaTBBZlZncXdpcDQ2SytkbUd1d1B5dnBRNWorU1YxNHoKSmlFOElJYTdYMzQrWnlrM2ZZUjd0elpobllqQ21UcXJYbnZSbzBGZHNuU0ZvRUU4T3RRMjU5dllJeXNuRXVPVwpOTEwvSTVQeGpNQlQ1WWNkaXdtbGR2RGxwZzk5YkRUQ0w0UFJ1MldKV2VFUmhzdGIzbWdRSk9Mb0RSM1VWWmowCjBFOGxQTjJiTjZMbVZjYjB1QlVyVjJQOHR6ZmxhVnF4SzQ5SlZFQmpFVHFaSzY3TTVYY3R4WG55SHdPVEpRckIKRGhVbWpkSGNZWCtYL0t5MDNOa1dhc2ZJeWdnV3IvR3gKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWekNDQWorZ0F3SUJBZ0lSQVBsUTMyUWVldjRpclVZaXJMRmVSU0l3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSlEybHNhWFZ0SUVOQk1CNFhEVEkxTVRJeU9USXlOVEF3TVZvWERUSTJNVEl5T1RJeQpOVEF3TVZvd0tqRW9NQ1lHQTFVRUF3d2ZLaTVrWldaaGRXeDBMbWgxWW1Kc1pTMW5jbkJqTG1OcGJHbDFiUzVwCmJ6Q0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU5IMWI3VXQ0eDdaSitUUUxBWjAKdXg0VlY4V2Z5RXRGY0hUbHdxeGZGcmMzZHBrNTQzWnpwN0NSbUg3Mkw2NlQ3VW1iRElydHZsdHcxdmh5MSs2eQpaY0FqNlZIcUVDdHhEVzd0MlNjTlRzdzFkME1aV3hiVm5QeVdiWm13aW0xV1Z4ODJ4THJMUWkrY3Y4RFhaUGpBCkZldm9rV21lNURMeTBJVmIwc0thbng0cGtQdTJsZlN0U0lveVpqOXBCYWFST2NwYXlVemNCZFU2RHBGdDlMRDIKdTIyTnVzbm00ZUh2bm9wRm5FMVI2UnprbVRGNHh4R2IyeTVweHdMTTV6RzhkZERJL3hhd3hNcHdENVVFYlRXWQorUjhaVGh5a242RHpmaitwcGI5dWZocmNOalZXRzNhY0o1elg4cElKU25ya1VTQmpLdm1ieVh3WEt4T0c4eGYzClltMENBd0VBQWFPQmpUQ0JpakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdIUVlEVlIwbEJCWXdGQVlJS3dZQkJRVUgKQXdFR0NDc0dBUVVGQndNQ01Bd0dBMVVkRXdFQi93UUNNQUF3SHdZRFZSMGpCQmd3Rm9BVU5iaGl0WnNMR3dWNApBNkE1U1dnL21ZaDVSZjR3S2dZRFZSMFJCQ013SVlJZktpNWtaV1poZFd4MExtaDFZbUpzWlMxbmNuQmpMbU5wCmJHbDFiUzVwYnpBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQVRrb24wcTVDdzJPT2JCcy9Nb0g2a041WXJidTAKeTR1c1hVc0x4V3oyNDFadisvY3JZQ0x4d092bGZNTGNnU0crL0xtTkIrMGpGc0RnMnRhQk9zejNHbzRtUzRrQwprV1QxUENSQ2o5MFdZU2hzRHNMdUV5MlI1bndTYmZCL3NHc3MxZnNwcDNhTldXSUNjRjJlZmh1TXRsREdMcEtRClFKQytlbHNYSFNoNkpWMWROTXhFYkhQVk9kQzBobUswNHRJMjJ0Tytsc0Iwd090S3VCbUJUb2VtR1ZScUtRbmEKem81NUtLc2cyQ1FwODhSTUJ4OUk2aXJjMThqSmFJeFhaRTE5RGMzWkRiNk42ZzVGc2hNeHhUK3F5cEt3VUVWego1SGRQNks5NjhyS3RjZnZXSXpieDVsYnIvY3hrZWZ0ZTE5SThXbmlxTzl4SlkxN0duVXV1aU5DREx3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= - tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBMGZWdnRTM2pIdGtuNU5Bc0JuUzdIaFZYeFovSVMwVndkT1hDckY4V3R6ZDJtVG5qCmRuT25zSkdZZnZZdnJwUHRTWnNNaXUyK1czRFcrSExYN3JKbHdDUHBVZW9RSzNFTmJ1M1pKdzFPekRWM1F4bGIKRnRXYy9KWnRtYkNLYlZaWEh6YkV1c3RDTDV5L3dOZGsrTUFWNitpUmFaN2tNdkxRaFZ2U3dwcWZIaW1RKzdhVgo5SzFJaWpKbVAya0ZwcEU1eWxySlROd0YxVG9Pa1czMHNQYTdiWTI2eWViaDRlK2Vpa1djVFZIcEhPU1pNWGpICkVadmJMbW5IQXN6bk1ieDEwTWovRnJERXluQVBsUVJ0TlpqNUh4bE9IS1Nmb1BOK1A2bWx2MjUrR3R3Mk5WWWIKZHB3bm5OZnlrZ2xLZXVSUklHTXErWnZKZkJjckU0YnpGL2RpYlFJREFRQUJBb0lCQUROMzhKbmY5VkN2R0IzNQp3QWtYSFhXYThzakJ5Z1pWcitNZ0hiQkdvUmxwMGJ0dkd1a0RJZ1RoTkJwUGZGSFY5bkc0UGdOaWsycjBCdWFoCnRIaFJxQ3JKUXQxdlBPYUJVTE0wUUNyWUVzeHhnTnBrZ1ZWZ0tSc3NQeExSV2FYQzZCbUJRZ211WVB1dDk3MnMKcjVaZzlCN0FCTUY1RDJURDdFVDh4UFlxbEQzK3JjTUpIelZmWkhyWXhkSzZsaldHelBkYkhrQ251VUdpS0lHWgp6VTI4cVNJLzBzeld6SEU0K2NrZFplc2dTUXovMVV2WUJ3Qk1DM0k3V3dvSVlZbXV1RVhreHdiTy9iMjltSVJHClhyaWY1M1FBNDA4a0YvODFic3piUmFuT01YRUhrN2NnSlFKUzVwaXV2UVV1KzA4bzJWZEhuZUJoQ1VqK1JWWDcKZElxWEhuRUNnWUVBOGpJaGdjMGtxTlA2Z3dldjJxSkszbGd3UzhqR3pWQVlJZDIzQkNuQWtQRGp5czROdVY2QQpZaEVGejNGWkpLV0J4a3pXeDVlNXZpeHJOR0RMUnNGVDgxR1ZTZUtnZGljR3NxYTZNL0Q2d2FRZXh6T2RTRFJNCk15cjBKeWs3dmRVTjdEVHovbTl4UjNzdkluM0dpMGhXaU1ELzFJUWpJOEhyLzkraVc1UkMwQ2tDZ1lFQTNlenYKZXhHM3BsUXhDbmNJbUlyaU5SS1NiRTljODluNGQ5VVJzcEs3cFVZVWQvSFExZmlTZ2ZpL2twZnBQb0dXREtOTAppOTA4WFN3MWNxTTZCTXVWTnlmT2xEKzRQaTZXb3phU1pHOTI4Ym8zNFBzbjhSeEZqSjJ3NTZyVC9iU2p5WUllClR3NUNpQzJmQUhGQmlUTkV0TEwzY09EMzlxcmF1NWR3MVZjMGVLVUNnWUFQd1JFSzUzUTJBeXZ0Y0FlUldqTXkKaVZ3QzRmbUVpMncyYjd5aTZiQmIvVDlrQnNrL3dKVHJUQjRyb3p6Z05GL2ZyVW5mUzlCS1BZdHZxY2d2UHc0ZAo2cldpUzdxU1ZQR0xsMnJQVENLVGpBQndocnY1WVdWL2dwREZKMXA2ZTZ4ZkxBYWZDMUs4Q3BoRFR4a21JRHQ0Cnc4MGdYc1FHWkd2Y2hnaUNtbjlLR1FLQmdGRUVMMFQ1YWRieHcxbHpyUktyR1B1UkJSMC9OOHJaMXdoQUk0N1MKWEdod2xnWlhwQXFKRFVzZmVTaFdCdE5IMFhSWnBMbXhrVmplUzhERzcrWlNQKzM3dlVHSHBZWWwwZDVSak0zWApsMCtWME5KMFBkZWFuNVUrK0JjSzJRczBoOXFIZ3ZNUFhLQ0VMeGlsUCt5TFo5aWp3UXRYUlk2cVB1SGUzbFV6CjJiYjFBb0dBWUhNazlYV29rOU5MRS81c1N1c2QyWFJqVjJVN1dGZnV5K21lKzJGVU1PdGliUUMrNHgwNnBLMlUKanRMc2VZU0thcURTQ1F5bG15SXlLVG5VNlZ0YnY1Z0FvRWhQZ2tYV2p2SWhPbDduamhHWXlyOFpLcWFvaTdOZwpNa29BQUg3KzBsL2VzeVkzSmZ3UlVyNUhoNGhmaDg1bHFFaDZjejlkTGhOa0U0VTV0TkU9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== ---- -# Source: cilium/templates/cilium-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-config - namespace: kube-system -data: - - # Identity allocation mode selects how identities are shared between cilium - # nodes by setting how they are stored. The options are "crd", "kvstore" or - # "doublewrite-readkvstore" / "doublewrite-readcrd". - # - "crd" stores identities in kubernetes as CRDs (custom resource definition). - # These can be queried with: - # kubectl get ciliumid - # - "kvstore" stores identities in an etcd kvstore, that is - # configured below. Cilium versions before 1.6 supported only the kvstore - # backend. Upgrades from these older cilium versions should continue using - # the kvstore by commenting out the identity-allocation-mode below, or - # setting it to "kvstore". - # - "doublewrite" modes store identities in both the kvstore and CRDs. This is useful - # for seamless migrations from the kvstore mode to the crd mode. Consult the - # documentation for more information on how to perform the migration. - identity-allocation-mode: crd - - identity-heartbeat-timeout: "30m0s" - identity-gc-interval: "15m0s" - cilium-endpoint-gc-interval: "5m0s" - nodes-gc-interval: "5m0s" - - # If you want to run cilium in debug mode change this value to true - debug: "false" - debug-verbose: "" - metrics-sampling-interval: "5m" - # The agent can be put into the following three policy enforcement modes - # default, always and never. - # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes - enable-policy: "default" - policy-cidr-match-mode: "" - # If you want metrics enabled in cilium-operator, set the port for - # which the Cilium Operator will have their metrics exposed. - # NOTE that this will open the port on the nodes where Cilium operator pod - # is scheduled. - operator-prometheus-serve-addr: ":9963" - enable-metrics: "true" - enable-policy-secrets-sync: "true" - policy-secrets-only-from-secrets-namespace: "true" - policy-secrets-namespace: "cilium-secrets" - - # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 - # address. - enable-ipv4: "true" - - # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 - # address. - enable-ipv6: "true" - # Users who wish to specify their own custom CNI configuration file must set - # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. - custom-cni-conf: "false" - enable-bpf-clock-probe: "false" - # If you want cilium monitor to aggregate tracing for packets, set this level - # to "low", "medium", or "maximum". The higher the level, the less packets - # that will be seen in monitor output. - monitor-aggregation: medium - - # The monitor aggregation interval governs the typical time between monitor - # notification events for each allowed connection. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-interval: "5s" - - # The monitor aggregation flags determine which TCP flags which, upon the - # first observation, cause monitor notifications to be generated. - # - # Only effective when monitor aggregation is set to "medium" or higher. - monitor-aggregation-flags: all - # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic - # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. - bpf-map-dynamic-size-ratio: "0.0025" - # bpf-policy-map-max specifies the maximum number of entries in endpoint - # policy map (per endpoint) - bpf-policy-map-max: "16384" - # bpf-policy-stats-map-max specifies the maximum number of entries in global - # policy stats map - bpf-policy-stats-map-max: "65536" - # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, - # backend and affinity maps. - bpf-lb-map-max: "65536" - bpf-lb-external-clusterip: "false" - bpf-lb-source-range-all-types: "false" - bpf-lb-algorithm-annotation: "false" - bpf-lb-mode-annotation: "false" - - bpf-distributed-lru: "false" - bpf-events-drop-enabled: "true" - bpf-events-policy-verdict-enabled: "true" - bpf-events-trace-enabled: "true" - - # Pre-allocation of map entries allows per-packet latency to be reduced, at - # the expense of up-front memory allocation for the entries in the maps. The - # default value below will minimize memory usage in the default installation; - # users who are sensitive to latency may consider setting this to "true". - # - # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore - # this option and behave as though it is set to "true". - # - # If this value is modified, then during the next Cilium startup the restore - # of existing endpoints and tracking of ongoing connections may be disrupted. - # As a result, reply packets may be dropped and the load-balancing decisions - # for established connections may change. - # - # If this option is set to "false" during an upgrade from 1.3 or earlier to - # 1.4 or later, then it may cause one-time disruptions during the upgrade. - preallocate-bpf-maps: "false" - - # Name of the cluster. Only relevant when building a mesh of clusters. - cluster-name: "default" - # Unique ID of the cluster. Must be unique across all conneted clusters and - # in the range of 1 and 255. Only relevant when building a mesh of clusters. - cluster-id: "0" - - # Encapsulation mode for communication between nodes - # Possible values: - # - disabled - # - vxlan (default) - # - geneve - - routing-mode: "tunnel" - tunnel-protocol: "vxlan" - tunnel-source-port-range: "0-0" - service-no-backend-response: "reject" - - - # Enables L7 proxy for L7 policy enforcement and visibility - enable-l7-proxy: "true" - enable-ipv4-masquerade: "true" - enable-ipv4-big-tcp: "false" - enable-ipv6-big-tcp: "false" - enable-ipv6-masquerade: "false" - enable-tcx: "true" - datapath-mode: "veth" - enable-masquerade-to-route-source: "false" - - enable-xt-socket-fallback: "true" - install-no-conntrack-iptables-rules: "false" - iptables-random-fully: "false" - - auto-direct-node-routes: "false" - direct-routing-skip-unreachable: "false" - - - - kube-proxy-replacement: "true" - kube-proxy-replacement-healthz-bind-address: "" - bpf-lb-sock: "false" - nodeport-addresses: "" - enable-health-check-nodeport: "true" - enable-health-check-loadbalancer-ip: "false" - node-port-bind-protection: "true" - enable-auto-protect-node-port-range: "true" - bpf-lb-acceleration: "disabled" - enable-svc-source-range-check: "true" - enable-l2-neigh-discovery: "false" - k8s-require-ipv4-pod-cidr: "false" - k8s-require-ipv6-pod-cidr: "false" - enable-k8s-networkpolicy: "true" - enable-endpoint-lockdown-on-policy-overflow: "false" - # Tell the agent to generate and write a CNI configuration file - write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist - cni-exclusive: "true" - cni-log-file: "/var/run/cilium/cilium-cni.log" - enable-endpoint-health-checking: "true" - enable-health-checking: "true" - health-check-icmp-failure-threshold: "3" - enable-well-known-identities: "false" - enable-node-selector-labels: "false" - synchronize-k8s-nodes: "true" - operator-api-serve-addr: "127.0.0.1:9234" - - enable-hubble: "true" - # UNIX domain socket for Hubble server to listen to. - hubble-socket-path: "/var/run/cilium/hubble.sock" - hubble-network-policy-correlation-enabled: "true" - # An additional address for Hubble server to listen to (e.g. ":4244"). - hubble-listen-address: ":4244" - hubble-disable-tls: "false" - hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt - hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key - hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt - ipam: "kubernetes" - ipam-cilium-node-update-rate: "15s" - - default-lb-service-ipam: "lbipam" - egress-gateway-reconciliation-trigger-interval: "1s" - enable-vtep: "false" - vtep-endpoint: "" - vtep-cidr: "" - vtep-mask: "" - vtep-mac: "" - procfs: "/host/proc" - bpf-root: "/sys/fs/bpf" - cgroup-root: "/run/cilium/cgroupv2" - - identity-management-mode: "agent" - enable-sctp: "false" - remove-cilium-node-taints: "true" - set-cilium-node-taints: "true" - set-cilium-is-up-condition: "true" - unmanaged-pod-watcher-interval: "15" - # default DNS proxy to transparent mode in non-chaining modes - dnsproxy-enable-transparent-mode: "true" - dnsproxy-socket-linger-timeout: "10" - tofqdns-dns-reject-response-code: "refused" - tofqdns-enable-dns-compression: "true" - tofqdns-endpoint-max-ip-per-hostname: "1000" - tofqdns-idle-connection-grace-period: "0s" - tofqdns-max-deferred-connection-deletes: "10000" - tofqdns-proxy-response-max-delay: "100ms" - tofqdns-preallocate-identities: "true" - agent-not-ready-taint-key: "node.cilium.io/agent-not-ready" - - mesh-auth-enabled: "true" - mesh-auth-queue-size: "1024" - mesh-auth-rotated-identities-queue-size: "1024" - mesh-auth-gc-interval: "5m0s" - - proxy-xff-num-trusted-hops-ingress: "0" - proxy-xff-num-trusted-hops-egress: "0" - proxy-connect-timeout: "2" - proxy-initial-fetch-timeout: "30" - proxy-max-requests-per-connection: "0" - proxy-max-connection-duration-seconds: "0" - proxy-idle-timeout-seconds: "60" - proxy-max-concurrent-retries: "128" - http-retry-count: "3" - http-stream-idle-timeout: "300" - - external-envoy-proxy: "true" - envoy-base-id: "0" - envoy-access-log-buffer-size: "4096" - envoy-keep-cap-netbindservice: "false" - max-connected-clusters: "255" - clustermesh-enable-endpoint-sync: "false" - clustermesh-enable-mcs-api: "false" - policy-default-local-cluster: "false" - - nat-map-stats-entries: "32" - nat-map-stats-interval: "30s" - enable-internal-traffic-policy: "true" - enable-lb-ipam: "true" - enable-non-default-deny-policies: "true" - enable-source-ip-verification: "true" - -# Extra config allows adding arbitrary properties to the cilium config. -# By putting it at the end of the ConfigMap, it's also possible to override existing properties. ---- -# Source: cilium/templates/cilium-envoy/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: cilium-envoy-config - namespace: kube-system -data: - # Keep the key name as bootstrap-config.json to avoid breaking changes - bootstrap-config.json: | - {"admin":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}},"applicationLogConfig":{"logFormat":{"textFormat":"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"}},"bootstrapExtensions":[{"name":"envoy.bootstrap.internal_listener","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener"}}],"dynamicResources":{"cdsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"},"ldsConfig":{"apiConfigSource":{"apiType":"GRPC","grpcServices":[{"envoyGrpc":{"clusterName":"xds-grpc-cilium"}}],"setNodeOnFirstMessageOnly":true,"transportApiVersion":"V3"},"initialFetchTimeout":"30s","resourceApiVersion":"V3"}},"node":{"cluster":"ingress-cluster","id":"host~127.0.0.1~no-id~localdomain"},"overloadManager":{"resourceMonitors":[{"name":"envoy.resource_monitors.global_downstream_max_connections","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig","max_active_downstream_connections":"50000"}}]},"staticResources":{"clusters":[{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"egress-cluster","type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"useDownstreamProtocolConfig":{}}}},{"circuitBreakers":{"thresholds":[{"maxRetries":128}]},"cleanupInterval":"2.500s","connectTimeout":"2s","lbPolicy":"CLUSTER_PROVIDED","name":"ingress-cluster-tls","transportSocket":{"name":"cilium.tls_wrapper","typedConfig":{"@type":"type.googleapis.com/cilium.UpstreamTlsWrapperContext"}},"type":"ORIGINAL_DST","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","commonHttpProtocolOptions":{"idleTimeout":"60s","maxConnectionDuration":"0s","maxRequestsPerConnection":0},"upstreamHttpProtocolOptions":{},"useDownstreamProtocolConfig":{}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"xds-grpc-cilium","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/xds.sock"}}}}]}]},"name":"xds-grpc-cilium","type":"STATIC","typedExtensionProtocolOptions":{"envoy.extensions.upstreams.http.v3.HttpProtocolOptions":{"@type":"type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions","explicitHttpConfig":{"http2ProtocolOptions":{}}}}},{"connectTimeout":"2s","loadAssignment":{"clusterName":"/envoy-admin","endpoints":[{"lbEndpoints":[{"endpoint":{"address":{"pipe":{"path":"/var/run/cilium/envoy/sockets/admin.sock"}}}}]}]},"name":"/envoy-admin","type":"STATIC"}],"listeners":[{"additionalAddresses":[{"address":{"socketAddress":{"address":"::","portValue":9964}}}],"address":{"socketAddress":{"address":"0.0.0.0","portValue":9964}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtualHosts":[{"domains":["*"],"name":"prometheus_metrics_route","routes":[{"match":{"prefix":"/metrics"},"name":"prometheus_metrics_route","route":{"cluster":"/envoy-admin","prefixRewrite":"/stats/prometheus"}}]}]},"statPrefix":"envoy-prometheus-metrics-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-prometheus-metrics-listener"},{"additionalAddresses":[{"address":{"socketAddress":{"address":"::1","portValue":9878}}}],"address":{"socketAddress":{"address":"127.0.0.1","portValue":9878}},"filterChains":[{"filters":[{"name":"envoy.filters.network.http_connection_manager","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager","httpFilters":[{"name":"envoy.filters.http.router","typedConfig":{"@type":"type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"}}],"internalAddressConfig":{"cidrRanges":[{"addressPrefix":"10.0.0.0","prefixLen":8},{"addressPrefix":"172.16.0.0","prefixLen":12},{"addressPrefix":"192.168.0.0","prefixLen":16},{"addressPrefix":"127.0.0.1","prefixLen":32},{"addressPrefix":"::1","prefixLen":128}]},"routeConfig":{"virtual_hosts":[{"domains":["*"],"name":"health","routes":[{"match":{"prefix":"/healthz"},"name":"health","route":{"cluster":"/envoy-admin","prefixRewrite":"/ready"}}]}]},"statPrefix":"envoy-health-listener","streamIdleTimeout":"300s"}}]}],"name":"envoy-health-listener"}]}} ---- -# Source: cilium/templates/cilium-agent/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - networking.k8s.io - resources: - - networkpolicies - verbs: - - get - - list - - watch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - services - - pods - - endpoints - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list - - watch - # This is used when validating policies in preflight. This will need to stay - # until we figure out how to avoid "get" inside the preflight, and then - # should be removed ideally. - - get -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumbgppeeringpolicies - - ciliumbgpnodeconfigs - - ciliumbgpadvertisements - - ciliumbgppeerconfigs - - ciliumclusterwideenvoyconfigs - - ciliumclusterwidenetworkpolicies - - ciliumegressgatewaypolicies - - ciliumendpoints - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumidentities - - ciliumlocalredirectpolicies - - ciliumnetworkpolicies - - ciliumnodes - - ciliumnodeconfigs - - ciliumcidrgroups - - ciliuml2announcementpolicies - - ciliumpodippools - verbs: - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - - ciliumendpoints - - ciliumnodes - verbs: - - create -- apiGroups: - - cilium.io - # To synchronize garbage collection of such resources - resources: - - ciliumidentities - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - verbs: - - delete - - get -- apiGroups: - - cilium.io - resources: - - ciliumnodes - - ciliumnodes/status - verbs: - - get - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints/status - - ciliumendpoints - - ciliuml2announcementpolicies/status - - ciliumbgpnodeconfigs/status - verbs: - - patch ---- -# Source: cilium/templates/cilium-operator/clusterrole.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - # to automatically delete [core|kube]dns pods so that are starting to being - # managed by Cilium - - delete -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - cilium-config - verbs: - # allow patching of the configmap to set annotations - - patch -- apiGroups: - - "" - resources: - - nodes - verbs: - - list - - watch -- apiGroups: - - "" - resources: - # To remove node taints - - nodes - # To set NetworkUnavailable false on startup - - nodes/status - verbs: - - patch -- apiGroups: - - discovery.k8s.io - resources: - - endpointslices - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform LB IP allocation for BGP - - services/status - verbs: - - update - - patch -- apiGroups: - - "" - resources: - # to check apiserver connectivity - - namespaces - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - # to perform the translation of a CNP that contains `ToGroup` to its endpoints - - services - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies - - ciliumclusterwidenetworkpolicies - verbs: - # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups' - - create - - update - - deletecollection - # To update the status of the CNPs and CCNPs - - patch - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumnetworkpolicies/status - - ciliumclusterwidenetworkpolicies/status - verbs: - # Update the auto-generated CNPs and CCNPs status. - - patch - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpoints - - ciliumidentities - verbs: - # To perform garbage collection of such resources - - delete - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumidentities - verbs: - # To synchronize garbage collection of such resources - - update -- apiGroups: - - cilium.io - resources: - - ciliumnodes - verbs: - - create - - update - - get - - list - - watch - # To perform CiliumNode garbage collector - - delete -- apiGroups: - - cilium.io - resources: - - ciliumnodes/status - verbs: - - update -- apiGroups: - - cilium.io - resources: - - ciliumendpointslices - - ciliumenvoyconfigs - - ciliumbgppeerconfigs - - ciliumbgpadvertisements - - ciliumbgpnodeconfigs - verbs: - - create - - update - - get - - list - - watch - - delete - - patch -- apiGroups: - - cilium.io - resources: - - ciliumbgpclusterconfigs/status - - ciliumbgppeerconfigs/status - verbs: - - update -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - update - resourceNames: - - ciliumloadbalancerippools.cilium.io - - ciliumbgppeeringpolicies.cilium.io - - ciliumbgpclusterconfigs.cilium.io - - ciliumbgppeerconfigs.cilium.io - - ciliumbgpadvertisements.cilium.io - - ciliumbgpnodeconfigs.cilium.io - - ciliumbgpnodeconfigoverrides.cilium.io - - ciliumclusterwideenvoyconfigs.cilium.io - - ciliumclusterwidenetworkpolicies.cilium.io - - ciliumegressgatewaypolicies.cilium.io - - ciliumendpoints.cilium.io - - ciliumendpointslices.cilium.io - - ciliumenvoyconfigs.cilium.io - - ciliumidentities.cilium.io - - ciliumlocalredirectpolicies.cilium.io - - ciliumnetworkpolicies.cilium.io - - ciliumnodes.cilium.io - - ciliumnodeconfigs.cilium.io - - ciliumcidrgroups.cilium.io - - ciliuml2announcementpolicies.cilium.io - - ciliumpodippools.cilium.io - - ciliumgatewayclassconfigs.cilium.io -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools - - ciliumpodippools - - ciliumbgppeeringpolicies - - ciliumbgpclusterconfigs - - ciliumbgpnodeconfigoverrides - - ciliumbgppeerconfigs - verbs: - - get - - list - - watch -- apiGroups: - - cilium.io - resources: - - ciliumpodippools - verbs: - - create -- apiGroups: - - cilium.io - resources: - - ciliumloadbalancerippools/status - verbs: - - patch -# For cilium-operator running in HA mode. -# -# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election -# between multiple running instances. -# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less -# common and fewer objects in the cluster watch "all Leases". -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update ---- -# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: cilium-operator - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cilium-operator -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-agent/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch ---- -# Source: cilium/templates/cilium-operator/role.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - update - - patch ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-config-agent - namespace: kube-system - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-config-agent -subjects: - - kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-agent/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium" - namespace: kube-system ---- -# Source: cilium/templates/cilium-operator/rolebinding.yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cilium-operator-tlsinterception-secrets - namespace: "cilium-secrets" - labels: - app.kubernetes.io/part-of: cilium -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: cilium-operator-tlsinterception-secrets -subjects: -- kind: ServiceAccount - name: "cilium-operator" - namespace: kube-system ---- -# Source: cilium/templates/cilium-envoy/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: cilium-envoy - namespace: kube-system - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9964" - labels: - k8s-app: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - io.cilium/app: proxy -spec: - clusterIP: None - type: ClusterIP - selector: - k8s-app: cilium-envoy - ports: - - name: envoy-metrics - port: 9964 - protocol: TCP - targetPort: envoy-metrics ---- -# Source: cilium/templates/hubble/peer-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: hubble-peer - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: hubble-peer - -spec: - selector: - k8s-app: cilium - ports: - - name: peer-service - port: 443 - protocol: TCP - targetPort: 4244 - internalTrafficPolicy: Local ---- -# Source: cilium/templates/cilium-agent/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium - namespace: kube-system - labels: - k8s-app: cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-agent -spec: - selector: - matchLabels: - k8s-app: cilium - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - kubectl.kubernetes.io/default-container: cilium-agent - labels: - k8s-app: cilium - app.kubernetes.io/name: cilium-agent - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - seccompProfile: - type: Unconfined - containers: - - name: cilium-agent - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - cilium-agent - args: - - --config-dir=/tmp/cilium/config-map - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - failureThreshold: 300 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - - name: "require-k8s-connectivity" - value: "false" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9879 - scheme: HTTP - httpHeaders: - - name: "brief" - value: "true" - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_CLUSTERMESH_CONFIG - value: /var/lib/cilium/clustermesh/ - - name: GOMEMLIMIT - valueFrom: - resourceFieldRef: - resource: limits.memory - divisor: '1' - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - - name: KUBE_CLIENT_BACKOFF_BASE - value: "1" - - name: KUBE_CLIENT_BACKOFF_DURATION - value: "120" - lifecycle: - postStart: - exec: - command: - - "bash" - - "-c" - - | - set -o errexit - set -o pipefail - set -o nounset - - # When running in AWS ENI mode, it's likely that 'aws-node' has - # had a chance to install SNAT iptables rules. These can result - # in dropped traffic, so we should attempt to remove them. - # We do it using a 'postStart' hook since this may need to run - # for nodes which might have already been init'ed but may still - # have dangling rules. This is safe because there are no - # dependencies on anything that is part of the startup script - # itself, and can be safely run multiple times per node (e.g. in - # case of a restart). - if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; - then - echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore - fi - echo 'Done!' - - preStop: - exec: - command: - - /cni-uninstall.sh - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - CHOWN - - KILL - - NET_ADMIN - - NET_RAW - - IPC_LOCK - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - - DAC_OVERRIDE - - FOWNER - - SETGID - - SETUID - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - # Unprivileged containers need to mount /proc/sys/net from the host - # to have write access - - mountPath: /host/proc/sys/net - name: host-proc-sys-net - # Unprivileged containers need to mount /proc/sys/kernel from the host - # to have write access - - mountPath: /host/proc/sys/kernel - name: host-proc-sys-kernel - - name: bpf-maps - mountPath: /sys/fs/bpf - # Unprivileged containers can't set mount propagation to bidirectional - # in this case we will mount the bpf fs from an init container that - # is privileged and set the mount propagation from host to container - # in Cilium. - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium - - name: cilium-netns - mountPath: /var/run/cilium/netns - mountPropagation: HostToContainer - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d - - name: clustermesh-secrets - mountPath: /var/lib/cilium/clustermesh - readOnly: true - # Needed to be able to load kernel modules - - name: lib-modules - mountPath: /lib/modules - readOnly: true - - name: xtables-lock - mountPath: /run/xtables.lock - - name: hubble-tls - mountPath: /var/lib/cilium/tls/hubble - readOnly: true - - name: tmp - mountPath: /tmp - - initContainers: - - name: config - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - cilium-dbg - - build-config - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - volumeMounts: - - name: tmp - mountPath: /tmp - terminationMessagePolicy: FallbackToLogsOnError - # Required to mount cgroup2 filesystem on the underlying Kubernetes node. - # We use nsenter command with host's cgroup and mount namespaces enabled. - - name: mount-cgroup - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - env: - - name: CGROUP_ROOT - value: /run/cilium/cgroupv2 - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh and mount that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-mount /hostbin/cilium-mount; - nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT; - rm /hostbin/cilium-mount - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - - name: apply-sysctl-overwrites - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - env: - - name: BIN_PATH - value: /opt/cni/bin - command: - - sh - - -ec - # The statically linked Go program binary is invoked to avoid any - # dependency on utilities like sh that can be missing on certain - # distros installed on the underlying host. Copy the binary to the - # same directory where we install cilium cni plugin so that exec permissions - # are available. - - | - cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix; - nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix"; - rm /hostbin/cilium-sysctlfix - volumeMounts: - - name: hostproc - mountPath: /hostproc - - name: cni-path - mountPath: /hostbin - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - drop: - - ALL - # Mount the bpf fs if it is not mounted. We will perform this task - # from a privileged container because the mount propagation bidirectional - # only works from privileged containers. - - name: mount-bpf-fs - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - args: - - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf' - command: - - /bin/bash - - -c - - -- - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - privileged: true - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: Bidirectional - - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - /init-container.sh - env: - - name: CILIUM_ALL_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-state - optional: true - - name: CILIUM_BPF_STATE - valueFrom: - configMapKeyRef: - name: cilium-config - key: clean-cilium-bpf-state - optional: true - - name: WRITE_CNI_CONF_WHEN_READY - valueFrom: - configMapKeyRef: - name: cilium-config - key: write-cni-conf-when-ready - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - terminationMessagePolicy: FallbackToLogsOnError - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_MODULE - - SYS_ADMIN - - SYS_RESOURCE - drop: - - ALL - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - # Required to mount cgroup filesystem from the host to cilium agent pod - - name: cilium-cgroup - mountPath: /run/cilium/cgroupv2 - mountPropagation: HostToContainer - - name: cilium-run - mountPath: /var/run/cilium # wait-for-kube-proxy - # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - - name: install-cni-binaries - image: "quay.io/cilium/cilium:v1.18.5@sha256:2c92fb05962a346eaf0ce11b912ba434dc10bd54b9989e970416681f4a069628" - imagePullPolicy: IfNotPresent - command: - - "/install-plugin.sh" - resources: - requests: - cpu: 100m - memory: 10Mi - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: cni-path - mountPath: /host/opt/cni/bin # .Values.cni.install - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - # For sharing configuration between the "config" initContainer and the agent - - name: tmp - emptyDir: {} - # To keep state between restarts / upgrades - - name: cilium-run - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - # To exec into pod network namespaces - - name: cilium-netns - hostPath: - path: /var/run/netns - type: DirectoryOrCreate - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate - # To mount cgroup2 filesystem on the host or apply sysctlfix - - name: hostproc - hostPath: - path: /proc - type: Directory - # To keep state between restarts / upgrades for cgroup2 filesystem - - name: cilium-cgroup - hostPath: - path: /run/cilium/cgroupv2 - type: DirectoryOrCreate - # To install cilium cni plugin in the host - - name: cni-path - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - # To install cilium cni configuration in the host - - name: etc-cni-netd - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - # To be able to load kernel modules - - name: lib-modules - hostPath: - path: /lib/modules - # To access iptables concurrently with other processes (e.g. kube-proxy) - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: FileOrCreate - # Sharing socket with Cilium Envoy on the same node by using a host path - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - # To read the clustermesh configuration - - name: clustermesh-secrets - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: cilium-clustermesh - optional: true - # note: items are not explicitly listed here, since the entries of this secret - # depend on the peers configured, and that would cause a restart of all agents - # at every addition/removal. Leaving the field empty makes each secret entry - # to be automatically projected into the volume as a file whose name is the key. - - secret: - name: clustermesh-apiserver-remote-cert - optional: true - items: - - key: tls.key - path: common-etcd-client.key - - key: tls.crt - path: common-etcd-client.crt - - key: ca.crt - path: common-etcd-client-ca.crt - # note: we configure the volume for the kvstoremesh-specific certificate - # regardless of whether KVStoreMesh is enabled or not, so that it can be - # automatically mounted in case KVStoreMesh gets subsequently enabled, - # without requiring an agent restart. - - secret: - name: clustermesh-apiserver-local-cert - optional: true - items: - - key: tls.key - path: local-etcd-client.key - - key: tls.crt - path: local-etcd-client.crt - - key: ca.crt - path: local-etcd-client-ca.crt - - name: host-proc-sys-net - hostPath: - path: /proc/sys/net - type: Directory - - name: host-proc-sys-kernel - hostPath: - path: /proc/sys/kernel - type: Directory - - name: hubble-tls - projected: - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - sources: - - secret: - name: hubble-server-certs - optional: true - items: - - key: tls.crt - path: server.crt - - key: tls.key - path: server.key - - key: ca.crt - path: client-ca.crt ---- -# Source: cilium/templates/cilium-envoy/daemonset.yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: cilium-envoy - namespace: kube-system - labels: - k8s-app: cilium-envoy - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-envoy - name: cilium-envoy -spec: - selector: - matchLabels: - k8s-app: cilium-envoy - updateStrategy: - rollingUpdate: - maxUnavailable: 2 - type: RollingUpdate - template: - metadata: - annotations: - labels: - k8s-app: cilium-envoy - name: cilium-envoy - app.kubernetes.io/name: cilium-envoy - app.kubernetes.io/part-of: cilium - spec: - securityContext: - appArmorProfile: - type: Unconfined - containers: - - name: cilium-envoy - image: "quay.io/cilium/cilium-envoy:v1.34.12-1765374555-6a93b0bbba8d6dc75b651cbafeedb062b2997716@sha256:3108521821c6922695ff1f6ef24b09026c94b195283f8bfbfc0fa49356a156e1" - imagePullPolicy: IfNotPresent - command: - - /usr/bin/cilium-envoy-starter - args: - - '--' - - '-c /var/run/cilium/envoy/bootstrap-config.json' - - '--base-id 0' - - '--log-level info' - startupProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - failureThreshold: 105 - periodSeconds: 2 - successThreshold: 1 - initialDelaySeconds: 5 - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 10 - timeoutSeconds: 5 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9878 - scheme: HTTP - periodSeconds: 30 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: envoy-metrics - containerPort: 9964 - hostPort: 9964 - protocol: TCP - securityContext: - seLinuxOptions: - level: s0 - type: spc_t - capabilities: - add: - - NET_ADMIN - - SYS_ADMIN - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - volumeMounts: - - name: envoy-sockets - mountPath: /var/run/cilium/envoy/sockets - readOnly: false - - name: envoy-artifacts - mountPath: /var/run/cilium/envoy/artifacts - readOnly: true - - name: envoy-config - mountPath: /var/run/cilium/envoy/ - readOnly: true - - name: bpf-maps - mountPath: /sys/fs/bpf - mountPropagation: HostToContainer - restartPolicy: Always - priorityClassName: system-node-critical - serviceAccountName: "cilium-envoy" - automountServiceAccountToken: true - terminationGracePeriodSeconds: 1 - hostNetwork: true - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: cilium.io/no-schedule - operator: NotIn - values: - - "true" - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium - topologyKey: kubernetes.io/hostname - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: cilium-envoy - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - operator: Exists - volumes: - - name: envoy-sockets - hostPath: - path: "/var/run/cilium/envoy/sockets" - type: DirectoryOrCreate - - name: envoy-artifacts - hostPath: - path: "/var/run/cilium/envoy/artifacts" - type: DirectoryOrCreate - - name: envoy-config - configMap: - name: "cilium-envoy-config" - # note: the leading zero means this number is in octal representation: do not remove it - defaultMode: 0400 - items: - - key: bootstrap-config.json - path: bootstrap-config.json - # To keep state between restarts / upgrades - # To keep state between restarts / upgrades for bpf maps - - name: bpf-maps - hostPath: - path: /sys/fs/bpf - type: DirectoryOrCreate ---- -# Source: cilium/templates/cilium-operator/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cilium-operator - namespace: kube-system - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator -spec: - # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go - # for more details. - replicas: 2 - selector: - matchLabels: - io.cilium/app: operator - name: cilium-operator - # ensure operator update on single node k8s clusters, by using rolling update with maxUnavailable=100% in case - # of one replica and no user configured Recreate strategy. - # otherwise an update might get stuck due to the default maxUnavailable=50% in combination with the - # podAntiAffinity which prevents deployments of multiple operator replicas on the same node. - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 50% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/port: "9963" - prometheus.io/scrape: "true" - labels: - io.cilium/app: operator - name: cilium-operator - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: cilium-operator - spec: - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: cilium-operator - image: "quay.io/cilium/operator-generic:v1.18.5@sha256:36c3f6f14c8ced7f45b40b0a927639894b44269dd653f9528e7a0dc363a4eb99" - imagePullPolicy: IfNotPresent - command: - - cilium-operator-generic - args: - - --config-dir=/tmp/cilium/config-map - - --debug=$(CILIUM_DEBUG) - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CILIUM_K8S_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: CILIUM_DEBUG - valueFrom: - configMapKeyRef: - key: debug - name: cilium-config - optional: true - - name: KUBERNETES_SERVICE_HOST - value: "2620:11f:7001:7:ffff:ffff:ad7:1dd" - - name: KUBERNETES_SERVICE_PORT - value: "6443" - ports: - - name: prometheus - containerPort: 9963 - hostPort: 9963 - protocol: TCP - livenessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 3 - readinessProbe: - httpGet: - host: "127.0.0.1" - path: /healthz - port: 9234 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 5 - timeoutSeconds: 3 - failureThreshold: 5 - volumeMounts: - - name: cilium-config-path - mountPath: /tmp/cilium/config-map - readOnly: true - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - terminationMessagePolicy: FallbackToLogsOnError - hostNetwork: true - restartPolicy: Always - priorityClassName: system-cluster-critical - serviceAccountName: "cilium-operator" - automountServiceAccountToken: true - # In HA mode, cilium-operator pods must not be scheduled on the same - # node as they will clash with each other. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - io.cilium/app: operator - topologyKey: kubernetes.io/hostname - nodeSelector: - kubernetes.io/os: linux - tolerations: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - key: node-role.kubernetes.io/master - operator: Exists - - key: node.kubernetes.io/not-ready - operator: Exists - - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - - key: node.cilium.io/agent-not-ready - operator: Exists - - volumes: - # To read the configuration from the config map - - name: cilium-config-path - configMap: - name: cilium-config diff --git a/nix/kubernetes/keys/package/bootstrap-script/package.nix b/nix/kubernetes/keys/package/bootstrap-script/package.nix index 298ac62b..9190b7af 100644 --- a/nix/kubernetes/keys/package/bootstrap-script/package.nix +++ b/nix/kubernetes/keys/package/bootstrap-script/package.nix @@ -8,12 +8,10 @@ # installCheckPhase # distPhase { - config, lib, stdenv, writeShellScript, k8s, - openssh, ... }: let @@ -30,7 +28,7 @@ let lib.concatMapStringsSep "," lib.escapeShellArg ( [ ./files/manifests/initial_clusterrole.yaml - ./files/manifests/cilium.yaml + "${k8s.cilium-manifest}/cilium.yaml" ./files/manifests/coredns.yaml ./files/manifests/flux_namespace.yaml ./files/manifests/flux.yaml diff --git a/nix/kubernetes/keys/package/cilium-manifest/package.nix b/nix/kubernetes/keys/package/cilium-manifest/package.nix new file mode 100644 index 00000000..f8c7a09f --- /dev/null +++ b/nix/kubernetes/keys/package/cilium-manifest/package.nix @@ -0,0 +1,70 @@ +# unpackPhase +# patchPhase +# configurePhase +# buildPhase +# checkPhase +# installPhase +# fixupPhase +# installCheckPhase +# distPhase +{ + stdenv, + openssl, + fetchFromGitHub, + kubernetes-helm, + ... +}: +stdenv.mkDerivation ( + finalAttrs: + let + version = "1.18.5"; + in + { + name = "cilium-manifest"; + nativeBuildInputs = [ + openssl + kubernetes-helm + ]; + buildInputs = [ ]; + + src = fetchFromGitHub { + owner = "cilium"; + repo = "cilium"; + tag = "v${version}"; + hash = "sha256-348inOOQ/fgwTYnaSHrQ363xGYnx2UPts3D4ycDRsWE="; + }; + + buildPhase = '' + helm template --dry-run=client cilium $src/install/kubernetes/cilium --version 1.18.5 --namespace kube-system \ + --set kubeProxyReplacement=true \ + --set ipam.mode=kubernetes \ + --set k8sServiceHost="2620:11f:7001:7:ffff:ffff:ad7:1dd" \ + --set k8sServicePort=6443 \ + --set ipv6.enabled=true \ + --set ipv4.enabled=true \ + --set enableIPv6Masquerade=false \ + | tee $NIX_BUILD_TOP/cilium.yaml + ''; + + # --set enableIPv4BIGTCP=false \ + # --set enableIPv6BIGTCP=false \ + # --set routingMode=native \ + # --set ipv4NativeRoutingCIDR=10.0.0.0/8 \ + # --set ipv6NativeRoutingCIDR=2620:11f:7001:7:ffff::/96 \ + + # --set hostFirewall.enabled=true + # --set routingMode=native + + # --set 'ipam.operator.clusterPoolIPv4PodCIDRList=["10.0.0.0/8"]' \ + # --set 'ipam.operator.clusterPoolIPv6PodCIDRList=["fd00::/100"]' \ + + # --set encryption.enabled=true \ + # --set encryption.type=wireguard + # --set encryption.nodeEncryption=true + + installPhase = '' + mkdir -p "$out" + cp $NIX_BUILD_TOP/cilium.yaml $out/ + ''; + } +) diff --git a/nix/kubernetes/keys/scope.nix b/nix/kubernetes/keys/scope.nix index 0aa26956..784b20cb 100644 --- a/nix/kubernetes/keys/scope.nix +++ b/nix/kubernetes/keys/scope.nix @@ -207,6 +207,7 @@ makeScope newScope ( } ); encryption_config = (callPackage ./package/k8s-encryption-key/package.nix additional_vars); + cilium-manifest = (callPackage ./package/cilium-manifest/package.nix additional_vars); all_keys = (callPackage ./package/k8s-keys/package.nix additional_vars); deploy_script = (callPackage ./package/deploy-script/package.nix additional_vars); bootstrap_script = (callPackage ./package/bootstrap-script/package.nix additional_vars); diff --git a/nix/kubernetes/roles/network/default.nix b/nix/kubernetes/roles/network/default.nix index c75d0d1e..a25e70dc 100644 --- a/nix/kubernetes/roles/network/default.nix +++ b/nix/kubernetes/roles/network/default.nix @@ -30,16 +30,21 @@ config = lib.mkIf config.me.network.enable { networking.dhcpcd.enable = lib.mkDefault false; networking.useDHCP = lib.mkDefault false; + # Nameservers configured in host-specific files. + # networking.nameservers = [ + # "194.242.2.2#doh.mullvad.net" + # "2a07:e340::2#doh.mullvad.net" + # ]; networking.nameservers = [ - "194.242.2.2#doh.mullvad.net" - "2a07:e340::2#doh.mullvad.net" + "10.215.1.1" + "2620:11f:7001:7:ffff:ffff:0ad7:0101" ]; services.resolved = { enable = true; # dnssec = "true"; domains = [ "~." ]; fallbackDns = [ ]; - dnsovertls = "true"; + # dnsovertls = "true"; }; # Without this, systemd-resolved will send DNS requests for .home.arpa to the per-link DNS server (172.16.0.1) which does not support DNS-over-TLS. This leads to the connection hanging and timing out. This causes firefox startup to take an extra 10+ seconds.