From a453892fd1859a49b2953f78cb4712d8f29d55ca Mon Sep 17 00:00:00 2001 From: CoS Date: Sun, 7 Jun 2026 17:57:47 -0700 Subject: [PATCH] fix(strimzi): pin operator deployment to n97-mini (K3s pod-network bug workaround) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Strimzi cluster operator's admin client was timing out with 'No resolvable bootstrap urls given in bootstrap.servers' and 'KafkaQuorumCheck cannot be initialised' errors. Investigation showed: - Operator pods scheduled to n97-node-2 (the worker) have broken outbound network connectivity: cannot reach kube-dns (10.43.0.10:53), in-cluster service IPs, or cross-node pod IPs. - Operator pods scheduled to n97-mini (the control plane) have full network connectivity and reconcile both Kafka clusters successfully. - Fresh busybox pods in the same namespace can reach everything — the bug is specific to operator pods (and possibly pods that started when the operator was at full leader+controllers state). This is the same pod-specific K3s networking bug that bit the camel-k operator in marcus/engineering#60, but the Strimzi operator can't use the camel-k fix (hostNetwork: true + ClusterFirstWithHostNet) because Strimzi hard-codes its health probe port to 8080, which is already bound on both nodes (kuma + something else on n97-mini, IPv6 wildcard on n97-node-2). Fix: pin the operator to n97-mini via spec.nodeName. This is a workaround for the K3s pod-specific network bug; the right fix is in K3s itself. When that bug is fixed upstream, the nodeName can be removed. Changes: - strimzi-operator-pinned.yaml: complete operator deployment manifest with nodeName, hostAliases for the bootstrap FQDN, no UseKRaft feature gate (0.45.2 doesn't support it), replicaCount: 1 with Recreate strategy. - Recreate strategy is mandatory: any rolling update would still hit the same bug because the new pod would still be scheduled to n97-node-2 during the brief window before nodeName takes effect. Verification (2026-06-08 00:56 UTC): - Operator pod 'strimzi-cluster-operator-9bd6cbcd6-tqgjb' on n97-mini is the leader, 1/1 Running, 0 restarts. - Both Kafka CRs (kafka/kafka, scobleclaw/scoble-kafka) report READY=True. - E2E test: produced 8 messages via scoble-kafka:9092 alias, all consumed back successfully. Refs: marcus/engineering#62 --- .../provisioning/strimzi-operator-pinned.yaml | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml diff --git a/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml b/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml new file mode 100644 index 0000000..e79c506 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: strimzi-cluster-operator + namespace: kafka + labels: + app: strimzi +spec: + # Replicas MUST be 1 for the current K3s pod-specific network bug workaround. + # See audit-sink/provisioning/README.md and marcus/engineering#62. + replicas: 1 + selector: + matchLabels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + strategy: + type: Recreate + template: + metadata: + labels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + spec: + # Pin the operator to n97-mini (the control-plane node) because of a K3s + # pod-specific network bug that breaks the operator's outbound networking + # when scheduled to the worker node n97-node-2. The bug affects DNS, in-cluster + # service IPs, and cross-node pod IPs from operator pods only — fresh pods in + # the same namespace are fine. + # + # DO NOT remove this nodeName without first re-validating that the operator + # pod can make outbound TCP connections from n97-node-2 to other pod IPs + # (test: `kubectl exec deploy/strimzi-cluster-operator -- timeout 5 sh -c "echo > /dev/tcp/10.43.0.10/53"`). + nodeName: n97-mini + serviceAccountName: strimzi-cluster-operator + # hostAliases: hard-coded bootstrap address (the kube-dns resolution of + # -kafka-bootstrap..svc.cluster.local:9091 fails from + # the operator's Java DNS resolver). This is belt-and-suspenders — the + # kube-dns path does work now that the operator is on n97-mini. + hostAliases: + - ip: 10.43.212.41 # scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local + hostnames: + - scoble-kafka-kafka-bootstrap + - scoble-kafka-kafka-bootstrap.scobleclaw + - scoble-kafka-kafka-bootstrap.scobleclaw.svc + - scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local + containers: + - name: strimzi-cluster-operator + image: quay.io/strimzi/operator:0.45.2 + imagePullPolicy: IfNotPresent + args: + - /opt/strimzi/bin/cluster_operator_run.sh + ports: + - containerPort: 8080 + name: http + protocol: TCP + livenessProbe: + httpGet: + path: /healthy + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + startupProbe: + httpGet: + path: /healthy + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 30 + env: + - name: STRIMZI_NAMESPACE + value: "kafka,scobleclaw,scoble-policy" + - name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS + value: "120000" + - name: STRIMZI_OPERATION_TIMEOUT_MS + value: "300000" + - name: STRIMZI_DEFAULT_KAFKA_EXPORTER_IMAGE + value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_DEFAULT_CRUISE_CONTROL_IMAGE + value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_CONNECT_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_MIRROR_MAKER_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_DEFAULT_TOPIC_OPERATOR_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_USER_OPERATOR_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_KAFKA_INIT_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_KAFKA_BRIDGE_IMAGE + value: quay.io/strimzi/kafka-bridge:0.31.2 + - name: STRIMZI_DEFAULT_KANIKO_EXECUTOR_IMAGE + value: quay.io/strimzi/kaniko-executor:0.45.2 + - name: STRIMZI_DEFAULT_MAVEN_BUILDER + value: quay.io/strimzi/maven-builder:0.45.2 + - name: STRIMZI_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: STRIMZI_FEATURE_GATES + value: "" + - name: STRIMZI_LEADER_ELECTION_ENABLED + value: "true" + - name: STRIMZI_LEADER_ELECTION_LEASE_NAME + value: strimzi-cluster-operator + - name: STRIMZI_LEADER_ELECTION_LEASE_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: STRIMZI_LEADER_ELECTION_IDENTITY + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + resources: + requests: + cpu: 200m + memory: 384Mi + limits: + cpu: 1 + memory: 384Mi + volumeMounts: + - name: strimzi-tmp + mountPath: /tmp + - name: co-config-volume + mountPath: /opt/strimzi/custom-config/ + volumes: + - name: strimzi-tmp + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: co-config-volume + configMap: + name: strimzi-cluster-operator + defaultMode: 420