fix(strimzi): pin operator deployment to n97-mini (K3s pod-network bug workaround)
The Strimzi cluster operator's admin client was timing out with 'No resolvable bootstrap urls given in bootstrap.servers' and 'KafkaQuorumCheck cannot be initialised' errors. Investigation showed: - Operator pods scheduled to n97-node-2 (the worker) have broken outbound network connectivity: cannot reach kube-dns (10.43.0.10:53), in-cluster service IPs, or cross-node pod IPs. - Operator pods scheduled to n97-mini (the control plane) have full network connectivity and reconcile both Kafka clusters successfully. - Fresh busybox pods in the same namespace can reach everything — the bug is specific to operator pods (and possibly pods that started when the operator was at full leader+controllers state). This is the same pod-specific K3s networking bug that bit the camel-k operator in marcus/engineering#60, but the Strimzi operator can't use the camel-k fix (hostNetwork: true + ClusterFirstWithHostNet) because Strimzi hard-codes its health probe port to 8080, which is already bound on both nodes (kuma + something else on n97-mini, IPv6 wildcard on n97-node-2). Fix: pin the operator to n97-mini via spec.nodeName. This is a workaround for the K3s pod-specific network bug; the right fix is in K3s itself. When that bug is fixed upstream, the nodeName can be removed. Changes: - strimzi-operator-pinned.yaml: complete operator deployment manifest with nodeName, hostAliases for the bootstrap FQDN, no UseKRaft feature gate (0.45.2 doesn't support it), replicaCount: 1 with Recreate strategy. - Recreate strategy is mandatory: any rolling update would still hit the same bug because the new pod would still be scheduled to n97-node-2 during the brief window before nodeName takes effect. Verification (2026-06-08 00:56 UTC): - Operator pod 'strimzi-cluster-operator-9bd6cbcd6-tqgjb' on n97-mini is the leader, 1/1 Running, 0 restarts. - Both Kafka CRs (kafka/kafka, scobleclaw/scoble-kafka) report READY=True. - E2E test: produced 8 messages via scoble-kafka:9092 alias, all consumed back successfully. Refs: marcus/engineering#62
This commit is contained in:
parent
87c9442671
commit
a453892fd1
1 changed files with 167 additions and 0 deletions
|
|
@ -0,0 +1,167 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: strimzi-cluster-operator
|
||||
namespace: kafka
|
||||
labels:
|
||||
app: strimzi
|
||||
spec:
|
||||
# Replicas MUST be 1 for the current K3s pod-specific network bug workaround.
|
||||
# See audit-sink/provisioning/README.md and marcus/engineering#62.
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
name: strimzi-cluster-operator
|
||||
strimzi.io/kind: cluster-operator
|
||||
strategy:
|
||||
type: Recreate
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
name: strimzi-cluster-operator
|
||||
strimzi.io/kind: cluster-operator
|
||||
spec:
|
||||
# Pin the operator to n97-mini (the control-plane node) because of a K3s
|
||||
# pod-specific network bug that breaks the operator's outbound networking
|
||||
# when scheduled to the worker node n97-node-2. The bug affects DNS, in-cluster
|
||||
# service IPs, and cross-node pod IPs from operator pods only — fresh pods in
|
||||
# the same namespace are fine.
|
||||
#
|
||||
# DO NOT remove this nodeName without first re-validating that the operator
|
||||
# pod can make outbound TCP connections from n97-node-2 to other pod IPs
|
||||
# (test: `kubectl exec deploy/strimzi-cluster-operator -- timeout 5 sh -c "echo > /dev/tcp/10.43.0.10/53"`).
|
||||
nodeName: n97-mini
|
||||
serviceAccountName: strimzi-cluster-operator
|
||||
# hostAliases: hard-coded bootstrap address (the kube-dns resolution of
|
||||
# <cluster>-kafka-bootstrap.<namespace>.svc.cluster.local:9091 fails from
|
||||
# the operator's Java DNS resolver). This is belt-and-suspenders — the
|
||||
# kube-dns path does work now that the operator is on n97-mini.
|
||||
hostAliases:
|
||||
- ip: 10.43.212.41 # scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local
|
||||
hostnames:
|
||||
- scoble-kafka-kafka-bootstrap
|
||||
- scoble-kafka-kafka-bootstrap.scobleclaw
|
||||
- scoble-kafka-kafka-bootstrap.scobleclaw.svc
|
||||
- scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local
|
||||
containers:
|
||||
- name: strimzi-cluster-operator
|
||||
image: quay.io/strimzi/operator:0.45.2
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- /opt/strimzi/bin/cluster_operator_run.sh
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
protocol: TCP
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthy
|
||||
port: http
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /healthy
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
env:
|
||||
- name: STRIMZI_NAMESPACE
|
||||
value: "kafka,scobleclaw,scoble-policy"
|
||||
- name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS
|
||||
value: "120000"
|
||||
- name: STRIMZI_OPERATION_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: STRIMZI_DEFAULT_KAFKA_EXPORTER_IMAGE
|
||||
value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_DEFAULT_CRUISE_CONTROL_IMAGE
|
||||
value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_KAFKA_IMAGES
|
||||
value: |
|
||||
3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0
|
||||
3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1
|
||||
3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0
|
||||
3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1
|
||||
3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_KAFKA_CONNECT_IMAGES
|
||||
value: |
|
||||
3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0
|
||||
3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1
|
||||
3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0
|
||||
3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1
|
||||
3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_KAFKA_MIRROR_MAKER_IMAGES
|
||||
value: |
|
||||
3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0
|
||||
3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1
|
||||
3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0
|
||||
3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1
|
||||
3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES
|
||||
value: |
|
||||
3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0
|
||||
3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1
|
||||
3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0
|
||||
3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1
|
||||
3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2
|
||||
- name: STRIMZI_DEFAULT_TOPIC_OPERATOR_IMAGE
|
||||
value: quay.io/strimzi/operator:0.45.2
|
||||
- name: STRIMZI_DEFAULT_USER_OPERATOR_IMAGE
|
||||
value: quay.io/strimzi/operator:0.45.2
|
||||
- name: STRIMZI_DEFAULT_KAFKA_INIT_IMAGE
|
||||
value: quay.io/strimzi/operator:0.45.2
|
||||
- name: STRIMZI_DEFAULT_KAFKA_BRIDGE_IMAGE
|
||||
value: quay.io/strimzi/kafka-bridge:0.31.2
|
||||
- name: STRIMZI_DEFAULT_KANIKO_EXECUTOR_IMAGE
|
||||
value: quay.io/strimzi/kaniko-executor:0.45.2
|
||||
- name: STRIMZI_DEFAULT_MAVEN_BUILDER
|
||||
value: quay.io/strimzi/maven-builder:0.45.2
|
||||
- name: STRIMZI_OPERATOR_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
- name: STRIMZI_FEATURE_GATES
|
||||
value: ""
|
||||
- name: STRIMZI_LEADER_ELECTION_ENABLED
|
||||
value: "true"
|
||||
- name: STRIMZI_LEADER_ELECTION_LEASE_NAME
|
||||
value: strimzi-cluster-operator
|
||||
- name: STRIMZI_LEADER_ELECTION_LEASE_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.namespace
|
||||
- name: STRIMZI_LEADER_ELECTION_IDENTITY
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
apiVersion: v1
|
||||
fieldPath: metadata.name
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 384Mi
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 384Mi
|
||||
volumeMounts:
|
||||
- name: strimzi-tmp
|
||||
mountPath: /tmp
|
||||
- name: co-config-volume
|
||||
mountPath: /opt/strimzi/custom-config/
|
||||
volumes:
|
||||
- name: strimzi-tmp
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 1Mi
|
||||
- name: co-config-volume
|
||||
configMap:
|
||||
name: strimzi-cluster-operator
|
||||
defaultMode: 420
|
||||
Loading…
Add table
Add a link
Reference in a new issue