From 56a414357666fcbab71e75e3a005e4031c711fe6 Mon Sep 17 00:00:00 2001 From: Kireque <5346530+kireque@users.noreply.github.com> Date: Fri, 20 Dec 2024 15:16:41 +0100 Subject: [PATCH] feat(talconfig): update --- kubernetes/main/talos/talconfig.yaml | 218 ++++++++++++++++----------- 1 file changed, 128 insertions(+), 90 deletions(-) diff --git a/kubernetes/main/talos/talconfig.yaml b/kubernetes/main/talos/talconfig.yaml index 808348214..e3fa16e3b 100644 --- a/kubernetes/main/talos/talconfig.yaml +++ b/kubernetes/main/talos/talconfig.yaml @@ -1,21 +1,23 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/budimanjojo/talhelper/master/pkg/config/schemas/talconfig.json -clusterName: ${clusterName} -endpoint: "https://${clusterEndpointIP}:6443" +clusterName: &clusterName main + +endpoint: https://main.k8s.home.econline.nl:6443 # renovate: depName=ghcr.io/siderolabs/installer datasource=docker -talosVersion: v1.9.0 +talosVersion: v1.8.4 # renovate: depName=ghcr.io/siderolabs/kubelet datasource=docker kubernetesVersion: v1.32.0 +additionalApiServerCertSans: &sans + - main.k8s.home.econline.nl + - &talosControlplaneVip 10.1.1.30 + - 127.0.0.1 # KubePrism +additionalMachineCertSans: *sans + cniConfig: name: none -additionalApiServerCertSans: &san - - ${clusterName}.${clusterDNSSuffix} - - "127.0.0.1" # KubePrism -additionalMachineCertSans: *san - nodes: - hostname: delta.home.econline.nl ipAddress: 10.1.1.31 @@ -23,6 +25,10 @@ nodes: installDiskSelector: size: "<= 512GB" model: "CT500MX500SSD1" + machineDisks: + - device: /dev/disk/by-id/ata-CT500MX500SSD1_2402E88C6FB4 + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -32,7 +38,7 @@ nodes: driver: e1000e dhcp: true vip: - ip: ${clusterEndpointIP} + ip: *talosControlplaneVip vlans: - &vlan40 vlanId: 40 @@ -40,18 +46,6 @@ nodes: dhcp: true dhcpOptions: routeMetric: 4096 - patches: - - &kubelet_extra_mounts |- - machine: - kubelet: - extraMounts: - - destination: /var/openebs/local - type: bind - source: /var/openebs/local - options: - - rbind - - rshared - - rw - hostname: enigma.home.econline.nl ipAddress: 10.1.1.32 @@ -59,6 +53,10 @@ nodes: installDiskSelector: size: "<= 512GB" model: "CT500MX500SSD1" + machineDisks: + - device: /dev/disk/by-id/ata-CT500MX500SSD1_2235E6604278 + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -68,18 +66,20 @@ nodes: driver: e1000e dhcp: true vip: - ip: ${clusterEndpointIP} + ip: *talosControlplaneVip vlans: - *vlan40 - patches: - - *kubelet_extra_mounts - hostname: felix.home.econline.nl ipAddress: 10.1.1.33 controlPlane: true installDiskSelector: - size: "<= 500GB" + size: "<= 5125GB" model: "VK0480GDJXV" + machineDisks: + - device: /dev/disk/by-id/ata-VK0480GDJXV_BTWL5131051T480QGN + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -89,29 +89,62 @@ nodes: driver: e1000e dhcp: true vip: - ip: ${clusterEndpointIP} + ip: *talosControlplaneVip vlans: - *vlan40 - patches: - - *kubelet_extra_mounts controlPlane: + nodeLabels: + topology.kubernetes.io/region: *clusterName + topology.kubernetes.io/zone: m + schematic: customization: extraKernelArgs: + - intel_iommu=on + - iommu=pt - net.ifnames=1 systemExtensions: officialExtensions: - - siderolabs/intel-ucode + - siderolabs/gasket-driver - siderolabs/i915-ucode + - siderolabs/intel-ucode + - siderolabs/mei patches: + # Configure containerd + - |- + machine: + files: + - op: create + path: /etc/cri/conf.d/20-customization.part + content: | + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + enable_unprivileged_ports = true + enable_unprivileged_icmp = true + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + discard_unpacked_layers = false + # Disable search domain everywhere - |- machine: network: disableSearchDomain: true + # Enable cluster discovery + - |- + cluster: + discovery: + enabled: true + registries: + kubernetes: + disabled: false + service: + disabled: true + # Disable Host DNS - |- machine: @@ -121,6 +154,26 @@ controlPlane: resolveMemberNames: true forwardKubeDNSToHost: false + # Configure kubelet + - |- + machine: + kubelet: + extraArgs: + rotate-server-certificates: "true" + extraConfig: + maxPods: 150 + extraMounts: + - destination: /var/mnt/extra + type: bind + source: /var/mnt/extra + options: + - rbind + - rshared + - rw + nodeIP: + validSubnets: + - 10.1.1.0/24 + # Enable KubePrism - |- machine: @@ -129,42 +182,37 @@ controlPlane: enabled: true port: 7445 - # Cluster configuration + # Configure cluster - |- cluster: allowSchedulingOnMasters: true - proxy: - disabled: true coreDNS: disabled: true - discovery: - enabled: true - registries: - kubernetes: - disabled: false - service: - disabled: true + proxy: + disabled: true + scheduler: + config: + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + profiles: + - schedulerName: default-scheduler + pluginConfig: + - name: PodTopologySpread + args: + defaultingType: List + defaultConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway - # ETCD configuration + # Configure etcd - |- cluster: etcd: advertisedSubnets: - 10.1.1.0/24 - # Configure containerd - - |- - machine: - files: - - op: create - path: /etc/cri/conf.d/20-customization.part - content: | - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - enable_unprivileged_ports = true - enable_unprivileged_icmp = true - - # Configure nfs mount options + # Configure NFS mount options - | machine: files: @@ -176,10 +224,33 @@ controlPlane: nfsvers=4 hard=True noatime=True - nodiratime=True - rsize=131072 - wsize=131072 - nconnect=8 + nconnect=16 + + # Configure static host entries + - |- + machine: + network: + extraHostEntries: + - ip: 10.1.1.30 + aliases: + - main.k8s.home.econline.nl + + # Configure custom sysctls + - |- + machine: + sysctls: + fs.inotify.max_user_instances: 8192 # Watchdog + fs.inotify.max_user_watches: 1048576 # Watchdog + net.core.rmem_max: 67108864 # Cloudflared / QUIC + net.core.wmem_max: 67108864 # Cloudflared / QUIC + + # Configure udev rules + - |- + machine: + udev: + rules: + # Intel GPU + - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660" # Disable default API server admission plugins. - |- @@ -196,36 +267,3 @@ controlPlane: - os:admin allowedKubernetesNamespaces: - system-upgrade - - # Kubelet configuration - - |- - machine: - kubelet: - extraArgs: - rotate-server-certificates: "true" - extraConfig: - maxPods: 150 - nodeIP: - validSubnets: - - 10.1.1.0/24 - - # Custom sysctls - - |- - machine: - sysctls: - fs.inotify.max_queued_events: "65536" - fs.inotify.max_user_instances: "8192" - fs.inotify.max_user_watches: "524288" - net.core.rmem_max: "7500000" - net.core.wmem_max: "7500000" - - # Various udev rules - - |- - machine: - udev: - rules: - # Intel GPU - - SUBSYSTEM=="drm", KERNEL=="renderD*", GROUP="44", MODE="0660" - # Google Coral USB Accelerator - - SUBSYSTEMS=="usb", ATTRS{idVendor}=="1a6e", ATTRS{idProduct}=="089a", GROUP="20", MODE="0660" - - SUBSYSTEMS=="usb", ATTRS{idVendor}=="18d1", ATTRS{idProduct}=="9302", GROUP="20", MODE="0660"