Initial cluster config: NixOS flake + k3s + 5 node definitions

- Multi-host flake.nix (3x N97 x86_64 + 2x RPi4 aarch64)
- Shared modules: k3s-cluster, tailscale, meos-gateway
- Node configs: n97-1 (control plane), n97-2/n97-3 (Ollama workers), rpi4-1/rpi4-2 (ARM workers)
- K8s manifests: Forgejo, Postgres (CNPG), Umami, OpenBrain, MeOS Dashboard, MCP servers
- Bootstrap and DB migration scripts
- Migration plan doc (7-day phased rollout)

README.md

@@ -1,3 +1,64 @@
-# devops
+# sscoble/devops
 
-Infra as code for Scobleclaw multi-node cluster
+Infrastructure as code for the Scobleclaw cluster.
+
+## Hardware
+
+| Node      | Platform  | Arch    | Role          | RAM  | Storage |
+|-----------|-----------|---------|---------------|------|---------|
+| n97-1     | GMK N97   | x86_64  | k3s server    | 12GB | 233GB   |
+| n97-2     | GMK N97   | x86_64  | k3s worker    | 12GB | ~233GB  |
+| n97-3     | GMK N97   | x86_64  | k3s worker    | 12GB | ~233GB  |
+| rpi4-1    | RPi 4B    | aarch64 | k3s worker    | 8GB  | SD/USB  |
+| rpi4-2    | RPi 4B    | aarch64 | k3s worker    | 8GB  | SD/USB  |
+
+## Network
+
+- **Mesh**: Tailscale (all nodes)
+- **DNS**: os.scoble.me → n97-1 (Tailscale IP), SSL via Let's Encrypt + Cloudflare DNS
+- **K8s API**: Tailscale-only, no public exposure
+
+## Repo Layout
+
+```
+├── README.md
+├── flake.nix              # NixOS flake — all 5 nodes
+├── hardware-n97.nix       # shared N97 perf tuning
+├── hardware-rpi4.nix      # shared RPi4 config
+├── modules/
+│   ├── k3s-cluster.nix    # shared k3s agent config
+│   ├── meos-gateway.nix   # nginx + acme + homepage
+│   ├── ollama-worker.nix  # Ollama on worker nodes
+│   └── tailscale.nix      # shared Tailscale config
+├── nodes/
+│   ├── n97-1.nix          # control plane + gateway
+│   ├── n97-2.nix          # Ollama worker x86
+│   ├── n97-3.nix          # Ollama worker x86
+│   ├── rpi4-1.nix         # ARM worker (light services)
+│   └── rpi4-2.nix         # ARM worker (umami)
+├── apps/                  # K8s manifests + Helm values
+│   ├── forgejo/
+│   ├── postgres/
+│   ├── umami/
+│   ├── openbrain/
+│   ├── meos-dashboard/
+│   └── mcp-servers/
+├── scripts/
+│   ├── bootstrap-node.sh  # initial node setup
+│   └── migrate-db.sh      # Postgres migration from bare to K8s
+└── docs/
+    └── migration.md       # transition plan
+```
+
+## Quickstart
+
+```bash
+# Build and deploy a node
+nixos-rebuild switch --flake .#n97-1 --target-host root@100.x.x.x
+
+# Access cluster
+kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get nodes
+
+# Deploy app
+kubectl apply -k apps/forgejo/
+```

apps/forgejo/deployment.yaml

@@ -0,0 +1,103 @@
+# Forgejo on k3s
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: forgejo
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: forgejo-data
+  namespace: forgejo
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forgejo
+  namespace: forgejo
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: forgejo
+  template:
+    metadata:
+      labels:
+        app: forgejo
+    spec:
+      containers:
+        - name: forgejo
+          image: codeberg.org/forgejo/forgejo:11.0
+          ports:
+            - containerPort: 3000
+              name: http
+            - containerPort: 22
+              name: ssh
+          env:
+            - name: USER_UID
+              value: "1000"
+            - name: USER_GID
+              value: "1000"
+            - name: FORGEJO__database__DB_TYPE
+              value: postgres
+            - name: FORGEJO__database__HOST
+              value: postgres-scobleclaw-rw.database.svc.cluster.local:5432
+            - name: FORGEJO__database__NAME
+              value: forgejo
+            - name: FORGEJO__database__USER
+              value: forgejo
+            - name: FORGEJO__database__PASSWD
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-users
+                  key: forgejo-password
+            - name: FORGEJO__server__ROOT_URL
+              value: "https://os.scoble.me/forgejo/"
+            - name: FORGEJO__server__HTTP_PORT
+              value: "3000"
+            - name: FORGEJO__server__DOMAIN
+              value: os.scoble.me
+            - name: FORGEJO__server__DISABLE_SSH
+              value: "false"
+            - name: FORGEJO__server__SSH_PORT
+              value: "2222"
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "250m"
+            limits:
+              memory: "512Mi"
+              cpu: "1"
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: forgejo-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: forgejo
+  namespace: forgejo
+spec:
+  ports:
+    - name: http
+      port: 3000
+      targetPort: 3000
+    - name: ssh
+      port: 22
+      targetPort: 22
+  selector:
+    app: forgejo

apps/mcp-servers/deployment.yaml

@@ -0,0 +1,83 @@
+# MCP Servers — lightweight Node.js workloads for ARM worker (rpi4-1)
+# forgejo-mcp, lean-experiment-mcp, wardrobe-ai, homeschool
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mcp-servers
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forgejo-mcp
+  namespace: mcp-servers
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: forgejo-mcp
+  template:
+    metadata:
+      labels:
+        app: forgejo-mcp
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+        - key: "arm64"
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: forgejo-mcp
+          image: node:22-slim
+          command: ["node", "/app/index.js"]
+          workingDir: /app
+          env:
+            - name: FORGEJO_URL
+              value: "http://forgejo.forgejo.svc.cluster.local:3000"
+            - name: FORGEJO_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: forgejo-tokens
+                  key: agent-cos
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "128Mi"
+              cpu: "100m"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: lean-experiment-mcp
+  namespace: mcp-servers
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: lean-experiment-mcp
+  template:
+    metadata:
+      labels:
+        app: lean-experiment-mcp
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+        - key: "arm64"
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: lean-mcp
+          image: node:22-slim
+          command: ["node", "/app/dist/index.js"]
+          workingDir: /app
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "128Mi"
+              cpu: "100m"

apps/meos-dashboard/deployment.yaml

@@ -0,0 +1,46 @@
+# MeOS Dashboard — stateless React app, can run on any node
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: meos-dashboard
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: meos-dashboard
+  namespace: meos-dashboard
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: meos-dashboard
+  template:
+    metadata:
+      labels:
+        app: meos-dashboard
+    spec:
+      containers:
+        - name: dashboard
+          image: ghcr.io/scobleclaw/meos-dashboard:latest
+          ports:
+            - containerPort: 3000
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "128Mi"
+              cpu: "250m"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: meos-dashboard
+  namespace: meos-dashboard
+spec:
+  ports:
+    - port: 3000
+      targetPort: 3000
+  selector:
+    app: meos-dashboard

apps/openbrain/deployment.yaml

@@ -0,0 +1,59 @@
+# OpenBrain MCP — deployed to ARM worker (rpi4-1)
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openbrain
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: openbrain
+  namespace: openbrain
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: openbrain
+  template:
+    metadata:
+      labels:
+        app: openbrain
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+        - key: "arm64"
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: openbrain
+          image: node:22-slim
+          command: ["node", "/app/index.js"]
+          workingDir: /app
+          ports:
+            - containerPort: 3000
+          env:
+            - name: NODE_ENV
+              value: production
+            - name: PORT
+              value: "3000"
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "256Mi"
+              cpu: "250m"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: openbrain
+  namespace: openbrain
+spec:
+  ports:
+    - port: 3000
+      targetPort: 3000
+  selector:
+    app: openbrain

apps/postgres/cluster.yaml

@@ -0,0 +1,59 @@
+# CloudNative PG operator (recommended for Postgres on K8s)
+# https://cloudnative-pg.io/
+#
+# Install: kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.25/releases/cnpg-1.25.1.yaml
+#
+# Then create the cluster below.
+---
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgres-scobleclaw
+  namespace: database
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgresql:17.4
+
+  storage:
+    size: 5Gi
+    storageClass: local-path
+
+  resources:
+    requests:
+      memory: "256Mi"
+      cpu: "100m"
+    limits:
+      memory: "512Mi"
+      cpu: "500m"
+
+  postgresql:
+    parameters:
+      max_connections: "200"
+      shared_buffers: "128MB"
+
+  databases:
+    - name: forgejo
+      owner: forgejo
+    - name: umami
+      owner: umami
+
+  users:
+    - name: forgejo
+      password: "<set-via-secret>"
+    - name: umami
+      password: "<set-via-secret>"
+
+  enableSuperuserAccess: false
+
+  monitoring:
+    enablePodMonitor: false
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: postgres-users
+  namespace: database
+type: Opaque
+stringData:
+  forgejo-password: "<change-me>"
+  umami-password: "<change-me>"

apps/umami/deployment.yaml

@@ -0,0 +1,67 @@
+# Umami analytics — deployed to ARM worker (rpi4-2)
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: umami
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: umami
+  namespace: umami
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: umami
+  template:
+    metadata:
+      labels:
+        app: umami
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: arm64
+      tolerations:
+        - key: "arm64"
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: umami
+          image: ghcr.io/umami-software/umami:postgresql-v2
+          ports:
+            - containerPort: 3000
+          env:
+            - name: DATABASE_TYPE
+              value: postgresql
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: umami-db
+                  key: url
+            - name: APP_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: umami-secret
+                  key: secret
+            - name: DISABLE_TELEMETRY
+              value: "1"
+          resources:
+            requests:
+              memory: "128Mi"
+              cpu: "100m"
+            limits:
+              memory: "256Mi"
+              cpu: "500m"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: umami
+  namespace: umami
+spec:
+  ports:
+    - port: 3000
+      targetPort: 3000
+  selector:
+    app: umami

docs/migration.md

@@ -0,0 +1,69 @@
+# Migration Plan: Bare Metal → k3s Cluster
+
+## Overview
+
+Migrate from single-node N97 (nixos) to 5-node k3s cluster with zero downtime.
+
+## Phase 1: Foundation (Day 1)
+
+1. **Create devops repo** ✅ (sscoble/devops)
+2. **Bootstrap k3s on n97-1** — single-node control plane alongside existing services
+3. **Install CNPG operator** — `kubectl apply -f cnpg-crds.yaml`
+4. **Deploy local-path provisioner** — for persistent volumes on each node
+5. **Install nginx ingress controller** — `helm install nginx-ingress ingress-nginx`
+
+At this point, k3s is running but all services are still bare metal. No traffic shifts yet.
+
+## Phase 2: Non-Critical Services (Day 2-3)
+
+Move services that don't affect the core workflow:
+
+1. **MeOS Dashboard → k3s** — deploy manifest, update nginx upstream
+2. **MCP servers → rpi4-1** — forgejo-mcp, lean-experiment, wardrobe-ai
+3. **Taint ARM nodes** — prevent x86 pods from landing on Pi 4s
+
+## Phase 3: Data Layer (Day 4-5)
+
+1. **Deploy CNPG Postgres cluster** — `kubectl apply -f apps/postgres/cluster.yaml`
+2. **Run migration** — `./scripts/migrate-db.sh`
+3. **Verify data integrity** — row counts, test queries
+4. **Update Forgejo to point at CNPG** — change DATABASE_URL env var
+5. **Disable bare-metal PostgreSQL** — set `services.postgresql.enable = false`
+
+## Phase 4: Core Services (Day 6-7)
+
+1. **Forgejo → k3s** — deploy manifest, cut over nginx upstream
+2. **Umami → rpi4-2** — deploy manifest, verify analytics still collecting
+3. **Homepage dashboard** — stays on n97-1 (lightweight, already working)
+
+## Phase 5: Scale Out (Day 8+)
+
+1. **Add n97-2 as worker** — `./scripts/bootstrap-node.sh n97-2 100.x.x.x agent`
+2. **Add n97-3 as worker** — same
+3. **Deploy Ollama on n97-2/n97-3** — systemd services, expose as ClusterIP
+4. **Add rpi4-1/rpi4-2 as workers** — bootstrap with ARM64 support
+5. **Configure node taints** — ARM workers for light work, N97s for heavy
+
+## What Stays Bare Metal
+
+- **OpenClaw Gateway** — manages processes, not containers. Stay outside k8s.
+- **Nginx + ACME** — TLS termination on n97-1, routes to k8s Ingress
+- **Ollama** — GPU/NPU passthrough is painful in k8s. Run as systemd services on workers.
+- **Storj static hosting** — build + deploy pipeline stays in Nix flake
+- **X11/LightDM** — only on n97-1 if you use the desktop
+
+## Rollback Strategy
+
+Each phase can be rolled back independently:
+- **Phase 2**: Switch nginx upstreams back to bare-metal ports
+- **Phase 3**: Re-enable `services.postgresql.enable = true`, restore from dump
+- **Phase 4**: Same as Phase 2 — nginx upstream swap
+- **Phase 5**: `kubectl drain <node> && kubectl delete node <node>`
+
+## Post-Migration
+
+- Add `nodeAffinity` rules for Ollama pods (n97-2, n97-3 only)
+- Set up `PodDisruptionBudgets` for critical services
+- Consider HA control plane (3-server etcd) if uptime matters
+- Add monitoring: Prometheus + Grafana on rpi4-2
+- Set up Velero for K8s backups

flake.nix

@@ -0,0 +1,48 @@
+{
+  description = "Scobleclaw Multi-Node Cluster — NixOS + k3s";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-25.11";
+  };
+
+  outputs = { self, nixpkgs }:
+    let
+      shared-modules = [
+        ./modules/tailscale.nix
+        ./modules/k3s-cluster.nix
+      ];
+
+      mkSystem = system: hostModule: nixpkgs.lib.nixosSystem {
+        inherit system;
+        modules = shared-modules ++ [ hostModule ];
+        specialArgs = { inherit self; };
+      };
+    in {
+      nixosConfigurations = {
+        n97-1 = mkSystem "x86_64-linux" [
+          ./hardware-n97.nix
+          ./nodes/n97-1.nix
+        ];
+
+        n97-2 = mkSystem "x86_64-linux" [
+          ./hardware-n97.nix
+          ./nodes/n97-2.nix
+        ];
+
+        n97-3 = mkSystem "x86_64-linux" [
+          ./hardware-n97.nix
+          ./nodes/n97-3.nix
+        ];
+
+        rpi4-1 = mkSystem "aarch64-linux" [
+          ./hardware-rpi4.nix
+          ./nodes/rpi4-1.nix
+        ];
+
+        rpi4-2 = mkSystem "aarch64-linux" [
+          ./hardware-rpi4.nix
+          ./nodes/rpi4-2.nix
+        ];
+      };
+    };
+}

hardware-n97.nix

@@ -0,0 +1,29 @@
+{ config, lib, pkgs, ... }:
+
+{
+  # Intel N97 Mini PC performance tuning
+  boot = {
+    kernelPackages = pkgs.linuxPackages_latest;
+    loader = {
+      systemd-boot.enable = true;
+      efi.canTouchEfiVariables = true;
+    };
+    initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ];
+  };
+
+  hardware = {
+    cpu.intel.updateMicrocode = true;
+    enableRedistributableFirmware = true;
+    opengl.enable = true;
+  };
+
+  services.xserver.videoDrivers = [ "intel" ];
+  powerManagement.enable = true;
+  services.thermald.enable = true;
+
+  # Local storage dir for k3s persistent volumes
+  systemd.tmpfiles.rules = [
+    "d /data/k3s 0755 root root -"
+    "d /data/ollama 0755 root root -"
+  ];
+}

hardware-rpi4.nix

@@ -0,0 +1,23 @@
+{ config, lib, pkgs, ... }:
+
+{
+  # Raspberry Pi 4 — aarch64 worker node
+  boot = {
+    kernelPackages = pkgs.linuxPackages_rpi4;
+    loader = {
+      grub.enable = false;
+      generic-extlinux-compatible.enable = true;
+    };
+  };
+
+  # ARM-specific performance
+  hardware.enableRedistributableFirmware = true;
+
+  # Local storage
+  systemd.tmpfiles.rules = [
+    "d /data/k3s 0755 root root -"
+  ];
+
+  # RPi 4 throttling protection
+  powerManagement.cpuFreqGovernor = "ondemand";
+}

modules/k3s-cluster.nix

@@ -0,0 +1,76 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.k3s-cluster;
+in {
+  options.services.k3s-cluster = {
+    enable = mkEnableOption "k3s cluster node";
+    role = mkOption {
+      type = types.enum [ "server" "agent" ];
+      default = "agent";
+      description = "k3s role: server (control plane) or agent (worker).";
+    };
+    serverAddr = mkOption {
+      type = types.str;
+      default = "https://127.0.0.1:6443";
+      description = "k3s server URL for agents to join.";
+    };
+    tailscaleIP = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "This node's Tailscale IP (for --advertise-address).";
+    };
+    tokenFile = mkOption {
+      type = types.nullOr types.str;
+      default = "/var/lib/k3s/cluster-token";
+      description = "File containing the cluster shared secret.";
+    };
+    withOllama = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Enable Ollama on this node (GPU passthrough).";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.k3s = {
+      enable = true;
+      role = cfg.role;
+
+      extraFlags = with cfg; concatStringsSep " " (
+        [ "--flannel-iface=tailscale0" ]
+        ++ optionals (tailscaleIP != null) [
+          "--advertise-address=${tailscaleIP}"
+          "--node-ip=${tailscaleIP}"
+        ]
+        ++ optionals (role == "server") [
+          "--cluster-init"
+          "--disable=traefik"
+          "--disable=servicelb"
+          "--kube-controller-manager-arg=bind-address=0.0.0.0"
+        ]
+      );
+
+      inherit tokenFile;
+
+      # Allow pods on control plane (we need the resources)
+      inherit (cfg) serverAddr;
+    };
+
+    # Open k3s API port on control plane
+    networking.firewall.allowedTCPPorts = optionals (cfg.role == "server") [ 6443 ];
+
+    # ── Ollama (optional, on worker nodes with GPU/NPU) ──
+    services.ollama = mkIf cfg.withOllama {
+      enable = true;
+      acceleration = "opencl";  # Intel N97 iGPU
+      environmentVariables = {
+        OLLAMA_KEEP_ALIVE = "24h";
+        OLLAMA_HOST = "0.0.0.0:11434";
+      };
+      openFirewall = true;
+    };
+  };
+}

modules/meos-gateway.nix

@@ -0,0 +1,94 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.meos-gateway;
+  certDir = config.security.acme.certs."os.scoble.me".directory;
+in {
+  options.services.meos-gateway = {
+    enable = mkEnableOption "MeOS Gateway — nginx reverse proxy for os.scoble.me";
+    tailscaleIP = mkOption {
+      type = types.str;
+      description = "n97-1 Tailscale IP for nginx to bind to.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # ── Nginx — SSL-terminating reverse proxy ──
+    services.nginx = {
+      enable = true;
+      virtualHosts."os.scoble.me" = {
+        addSSL = true;
+        sslCertificate = "${certDir}/fullchain.pem";
+        sslCertificateKey = "${certDir}/key.pem";
+        listen = [
+          { addr = cfg.tailscaleIP; port = 443; ssl = true; }
+        ];
+
+        locations = {
+          "/forgejo/" = {
+            proxyPass = "http://k3s-forgejo.kube-system.svc.cluster.local:3000/";
+            proxyWebsockets = true;
+            extraConfig = ''
+              proxy_set_header Host $host;
+              proxy_set_header X-Real-IP $remote_addr;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto $scheme;
+              proxy_set_header X-Forwarded-Prefix /forgejo;
+            '';
+          };
+          "/umami" = {
+            proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069";
+            proxyWebsockets = true;
+            extraConfig = ''
+              proxy_set_header Host $host;
+              proxy_set_header X-Real-IP $remote_addr;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto $scheme;
+            '';
+          };
+          "/umami/" = {
+            proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069";
+            proxyWebsockets = true;
+            extraConfig = ''
+              proxy_set_header Host $host;
+              proxy_set_header X-Real-IP $remote_addr;
+              proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+              proxy_set_header X-Forwarded-Proto $scheme;
+            '';
+          };
+          "/dashboard/" = {
+            proxyPass = "http://k3s-dashboard.kube-system.svc.cluster.local:3000/";
+            proxyWebsockets = true;
+          };
+          "/" = {
+            proxyPass = "http://127.0.0.1:8082";
+            proxyWebsockets = true;
+          };
+        };
+      };
+
+      # HTTP -> HTTPS redirect
+      virtualHosts."os.scoble.me-http" = {
+        listen = [ { addr = cfg.tailscaleIP; port = 80; } ];
+        serverName = "os.scoble.me";
+        extraConfig = "return 301 https://$server_name$request_uri;";
+      };
+    };
+
+    # ACME with Cloudflare DNS challenge
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = "scott@scoble.me";
+      certs."os.scoble.me" = {
+        dnsProvider = "cloudflare";
+        environmentFile = "/var/lib/acme/cloudflare.env";
+        group = "nginx";
+      };
+    };
+
+    # Firewall
+    networking.firewall.allowedTCPPorts = [ 80 443 ];
+  };
+}

modules/tailscale.nix

@@ -0,0 +1,34 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.tailscale-node;
+in {
+  options.services.tailscale-node = {
+    enable = mkEnableOption "Tailscale mesh node";
+    authKeyFile = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "Path to file containing Tailscale pre-auth key (for unattended setup).";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.tailscale = {
+      enable = true;
+      interfaceName = "tailscale0";
+      authKeyFile = cfg.authKeyFile;
+      extraUpFlags = "--accept-routes --accept-dns=false";
+    };
+
+    # Persist Tailscale state across reboots
+    systemd.services.tailscale = {
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig.SupplementaryGroups = [ "nginx" ];
+    };
+
+    networking.firewall.trustedInterfaces = [ "tailscale0" ];
+  };
+}

nodes/n97-1.nix

@@ -0,0 +1,68 @@
+{ config, lib, pkgs, ... }:
+
+let
+  # n97-1 Tailscale IP — update after initial Tailscale auth
+  tailscaleIP = "100.x.x.x";
+in {
+  imports = [
+    ../modules/meos-gateway.nix
+  ];
+
+  networking.hostName = "n97-1";
+
+  # ── k3s server (single-node control plane) ──
+  services.k3s-cluster = {
+    enable = true;
+    role = "server";
+    tailscaleIP = tailscaleIP;
+    tokenFile = "/var/lib/k3s/cluster-token";
+  };
+
+  # ── Gateway (nginx + ACME) ──
+  services.meos-gateway = {
+    enable = true;
+    tailscaleIP = tailscaleIP;
+  };
+
+  # ── PostgreSQL (for Forgejo + Umami, transition to K8s) ──
+  services.postgresql = {
+    enable = true;
+    package = pkgs.postgresql_17;
+    extraPlugins = [ pkgs.postgresql_17.pkgs.pgvector ];
+    dataDir = "/data/postgresql";
+  };
+
+  # ── Forgejo (bare metal during transition, then K8s) ──
+  services.forgejo = {
+    enable = true;
+    database.type = "postgres";
+    database.name = "forgejo";
+    database.user = "forgejo";
+    settings.server.ROOT_URL = "https://os.scoble.me/forgejo/";
+    lfs.enable = true;
+  };
+
+  # ── OpenClaw gateway stays bare metal ──
+  # (managed separately via systemd user service)
+
+  # ── Firewall ──
+  networking.firewall = {
+    enable = true;
+    trustedInterfaces = [ "tailscale0" ];
+    allowedTCPPorts = [ 80 443 6443 ];
+  };
+
+  # ── User ──
+  users.users.sscoble = {
+    isNormalUser = true;
+    extraGroups = [ "wheel" "docker" ];
+  };
+
+  security.sudo.extraRules = [{
+    users = [ "sscoble" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "sscoble" ];
+}

nodes/n97-2.nix

@@ -0,0 +1,42 @@
+{ config, lib, pkgs, ... }:
+
+let
+  tailscaleIP = "100.x.x.x";
+in {
+  networking.hostName = "n97-2";
+
+  services.k3s-cluster = {
+    enable = true;
+    role = "agent";
+    serverAddr = "https://100.x.x.x:6443";  # n97-1 IP
+    tailscaleIP = tailscaleIP;
+    tokenFile = "/var/lib/k3s/cluster-token";
+    withOllama = true;
+  };
+
+  # Ollama models to serve
+  services.ollama.models = [
+    "deepseek-v4-pro:cloud"  # heavy reasoning
+    "kimi-k2.6:cloud"        # backup reasoning
+    "glm-5.1:cloud"          # fallback
+  ];
+
+  networking.firewall = {
+    enable = true;
+    trustedInterfaces = [ "tailscale0" ];
+    allowedTCPPorts = [ 11434 ];
+  };
+
+  users.users.sscoble = {
+    isNormalUser = true;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.extraRules = [{
+    users = [ "sscoble" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "sscoble" ];
+}

nodes/n97-3.nix

@@ -0,0 +1,42 @@
+{ config, lib, pkgs, ... }:
+
+let
+  tailscaleIP = "100.x.x.x";
+in {
+  networking.hostName = "n97-3";
+
+  services.k3s-cluster = {
+    enable = true;
+    role = "agent";
+    serverAddr = "https://100.x.x.x:6443";  # n97-1 IP
+    tailscaleIP = tailscaleIP;
+    tokenFile = "/var/lib/k3s/cluster-token";
+    withOllama = true;
+  };
+
+  services.ollama.models = [
+    "qwen3-coder-next:cloud"  # targeted code
+    "deepseek-v4-flash:cloud"  # fast default
+    "nomic-embed-text"        # embeddings
+    "mxbai-embed-large"       # embeddings
+  ];
+
+  networking.firewall = {
+    enable = true;
+    trustedInterfaces = [ "tailscale0" ];
+    allowedTCPPorts = [ 11434 ];
+  };
+
+  users.users.sscoble = {
+    isNormalUser = true;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.extraRules = [{
+    users = [ "sscoble" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "sscoble" ];
+}

nodes/rpi4-1.nix

@@ -0,0 +1,36 @@
+{ config, lib, pkgs, ... }:
+
+let
+  tailscaleIP = "100.x.x.x";
+in {
+  networking.hostName = "rpi4-1";
+
+  services.k3s-cluster = {
+    enable = true;
+    role = "agent";
+    serverAddr = "https://100.x.x.x:6443";  # n97-1 IP
+    tailscaleIP = tailscaleIP;
+    tokenFile = "/var/lib/k3s/cluster-token";
+  };
+
+  # ARM worker — light services only
+  # Taints/tolerations will be set via kubectl to prevent x86 pods landing here
+
+  networking.firewall = {
+    enable = true;
+    trustedInterfaces = [ "tailscale0" ];
+  };
+
+  users.users.sscoble = {
+    isNormalUser = true;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.extraRules = [{
+    users = [ "sscoble" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "sscoble" ];
+}

nodes/rpi4-2.nix

@@ -0,0 +1,36 @@
+{ config, lib, pkgs, ... }:
+
+let
+  tailscaleIP = "100.x.x.x";
+in {
+  networking.hostName = "rpi4-2";
+
+  services.k3s-cluster = {
+    enable = true;
+    role = "agent";
+    serverAddr = "https://100.x.x.x:6443";  # n97-1 IP
+    tailscaleIP = tailscaleIP;
+    tokenFile = "/var/lib/k3s/cluster-token";
+  };
+
+  # ARM worker — Umami and monitoring
+  # Taints/tolerations set via kubectl
+
+  networking.firewall = {
+    enable = true;
+    trustedInterfaces = [ "tailscale0" ];
+  };
+
+  users.users.sscoble = {
+    isNormalUser = true;
+    extraGroups = [ "wheel" ];
+  };
+
+  security.sudo.extraRules = [{
+    users = [ "sscoble" ];
+    commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }];
+  }];
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.trusted-users = [ "sscoble" ];
+}

scripts/bootstrap-node.sh

@@ -0,0 +1,58 @@
+# Bootstrap a new node for the Scobleclaw k3s cluster
+#
+# Usage: ./bootstrap-node.sh <hostname> <tailscale-ip> <role: server|agent>
+#
+# Prerequisites:
+# - NixOS installed on target node
+# - SSH access to root@<tailscale-ip>
+# - This repo cloned locally
+#
+set -euo pipefail
+
+HOSTNAME="${1:?Usage: bootstrap-node.sh <hostname> <tailscale-ip> <role>}"
+TSIP="${2:?Need tailscale IP}"
+ROLE="${3:-agent}"
+REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+
+echo "==> Bootstrapping $HOSTNAME ($TSIP) as k3s $ROLE"
+
+# 1. Generate cluster token if it doesn't exist
+TOKEN_FILE="/var/lib/k3s/cluster-token"
+if [ ! -f "$TOKEN_FILE" ]; then
+  echo "==> Generating k3s cluster token"
+  sudo mkdir -p /var/lib/k3s
+  head -c 32 /dev/urandom | base64 | sudo tee "$TOKEN_FILE" > /dev/null
+  sudo chmod 600 "$TOKEN_FILE"
+fi
+
+# 2. Copy token to target node
+echo "==> Copying cluster token to $HOSTNAME"
+ssh "root@$TSIP" "mkdir -p /var/lib/k3s"
+scp "$TOKEN_FILE" "root@$TSIP:$TOKEN_FILE"
+
+# 3. Copy NixOS config
+echo "==> Copying NixOS configuration"
+ssh "root@$TSIP" "mkdir -p /etc/nixos/nodes /etc/nixos/modules"
+scp "$REPO_DIR/flake.nix" "root@$TSIP:/etc/nixos/flake.nix"
+scp "$REPO_DIR/hardware-n97.nix" "root@$TSIP:/etc/nixos/hardware-n97.nix" 2>/dev/null || true
+scp "$REPO_DIR/hardware-rpi4.nix" "root@$TSIP:/etc/nixos/hardware-rpi4.nix" 2>/dev/null || true
+scp "$REPO_DIR/modules/"*.nix "root@$TSIP:/etc/nixos/modules/"
+scp "$REPO_DIR/nodes/$HOSTNAME.nix" "root@$TSIP:/etc/nixos/nodes/"
+
+# 4. Build and switch
+echo "==> Building NixOS configuration for $HOSTNAME"
+ssh "root@$TSIP" "nixos-rebuild switch --flake /etc/nixos#$HOSTNAME"
+
+# 5. Wait for k3s to come up
+echo "==> Waiting for k3s to start..."
+sleep 10
+
+if [ "$ROLE" = "server" ]; then
+  echo "==> k3s server is running. Get kubeconfig:"
+  echo "    scp root@$TSIP:/etc/rancher/k3s/k3s.yaml ~/.kube/config-scobleclaw"
+  echo "    sed -i 's/127.0.0.1/$TSIP/g' ~/.kube/config-scobleclaw"
+else
+  echo "==> k3s agent joined. Verify with: kubectl get nodes"
+fi
+
+echo "==> Done! $HOSTNAME is ready."

scripts/migrate-db.sh

@@ -0,0 +1,49 @@
+# Migrate PostgreSQL from bare metal (n97-1) to CloudNative PG (k3s)
+#
+# Prerequisites:
+# - CNPG operator installed in k3s
+# - postgres-scobleclaw cluster created (apps/postgres/cluster.yaml)
+# - Both clusters accessible from n97-1
+#
+set -euo pipefail
+
+SOURCE_HOST="127.0.0.1"
+SOURCE_PORT="5432"
+CNPG_HOST="postgres-scobleclaw-rw.database.svc.cluster.local"
+CNPG_PORT="5432"
+
+echo "==> Starting PostgreSQL migration"
+echo "    Source: $SOURCE_HOST:$SOURCE_PORT (bare metal)"
+echo "    Target: $CNPG_HOST:$CNPG_PORT (k3s/CNPG)"
+
+# 1. Dump all databases
+echo "==> Dumping databases from bare metal..."
+sudo -u postgres pg_dumpall --clean --if-exists > /tmp/pg-migration-dump.sql
+
+# 2. Wait for CNPG cluster to be ready
+echo "==> Waiting for CNPG cluster to be ready..."
+kubectl -n database wait --for=condition=Ready cluster/postgres-scobleclaw --timeout=300s
+
+# 3. Get CNPG superuser password
+CNPG_PASSWORD=$(kubectl -n database get secret postgres-scobleclaw-superuser -o jsonpath='{.data.password}' | base64 -d)
+
+# 4. Restore dump into CNPG
+echo "==> Restoring dump into CNPG..."
+PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres < /tmp/pg-migration-dump.sql
+
+# 5. Verify
+echo "==> Verifying migration..."
+DB_COUNT=$(PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres -t -c "SELECT count(*) FROM pg_database WHERE datname IN ('forgejo','umami');" | tr -d ' ')
+echo "    Databases found: $DB_COUNT (expected: 2)"
+
+if [ "$DB_COUNT" = "2" ]; then
+  echo "==> Migration successful! You can now disable bare-metal PostgreSQL:"
+  echo "    Set services.postgresql.enable = false; in nodes/n97-1.nix"
+  echo "    And rebuild: nixos-rebuild switch --flake .#n97-1"
+else
+  echo "==> WARNING: Not all databases migrated. Check manually."
+  exit 1
+fi
+
+# Cleanup
+rm -f /tmp/pg-migration-dump.sql