From 7b5ffe6b115a96d09f78e3e888eb1c1d13bafd1f Mon Sep 17 00:00:00 2001 From: "Marcus (agent-cos)" Date: Mon, 4 May 2026 12:22:09 -0700 Subject: [PATCH] Initial cluster config: NixOS flake + k3s + 5 node definitions - Multi-host flake.nix (3x N97 x86_64 + 2x RPi4 aarch64) - Shared modules: k3s-cluster, tailscale, meos-gateway - Node configs: n97-1 (control plane), n97-2/n97-3 (Ollama workers), rpi4-1/rpi4-2 (ARM workers) - K8s manifests: Forgejo, Postgres (CNPG), Umami, OpenBrain, MeOS Dashboard, MCP servers - Bootstrap and DB migration scripts - Migration plan doc (7-day phased rollout) --- README.md | 65 +++++++++++++++++- apps/forgejo/deployment.yaml | 103 ++++++++++++++++++++++++++++ apps/mcp-servers/deployment.yaml | 83 ++++++++++++++++++++++ apps/meos-dashboard/deployment.yaml | 46 +++++++++++++ apps/openbrain/deployment.yaml | 59 ++++++++++++++++ apps/postgres/cluster.yaml | 59 ++++++++++++++++ apps/umami/deployment.yaml | 67 ++++++++++++++++++ docs/migration.md | 69 +++++++++++++++++++ flake.nix | 48 +++++++++++++ hardware-n97.nix | 29 ++++++++ hardware-rpi4.nix | 23 +++++++ modules/k3s-cluster.nix | 76 ++++++++++++++++++++ modules/meos-gateway.nix | 94 +++++++++++++++++++++++++ modules/tailscale.nix | 34 +++++++++ nodes/n97-1.nix | 68 ++++++++++++++++++ nodes/n97-2.nix | 42 ++++++++++++ nodes/n97-3.nix | 42 ++++++++++++ nodes/rpi4-1.nix | 36 ++++++++++ nodes/rpi4-2.nix | 36 ++++++++++ scripts/bootstrap-node.sh | 58 ++++++++++++++++ scripts/migrate-db.sh | 49 +++++++++++++ 21 files changed, 1184 insertions(+), 2 deletions(-) create mode 100644 apps/forgejo/deployment.yaml create mode 100644 apps/mcp-servers/deployment.yaml create mode 100644 apps/meos-dashboard/deployment.yaml create mode 100644 apps/openbrain/deployment.yaml create mode 100644 apps/postgres/cluster.yaml create mode 100644 apps/umami/deployment.yaml create mode 100644 docs/migration.md create mode 100644 flake.nix create mode 100644 hardware-n97.nix create mode 100644 hardware-rpi4.nix create mode 100644 modules/k3s-cluster.nix create mode 100644 modules/meos-gateway.nix create mode 100644 modules/tailscale.nix create mode 100644 nodes/n97-1.nix create mode 100644 nodes/n97-2.nix create mode 100644 nodes/n97-3.nix create mode 100644 nodes/rpi4-1.nix create mode 100644 nodes/rpi4-2.nix create mode 100755 scripts/bootstrap-node.sh create mode 100755 scripts/migrate-db.sh diff --git a/README.md b/README.md index ea974e3..3ddcf52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,64 @@ -# devops +# sscoble/devops -Infra as code for Scobleclaw multi-node cluster \ No newline at end of file +Infrastructure as code for the Scobleclaw cluster. + +## Hardware + +| Node | Platform | Arch | Role | RAM | Storage | +|-----------|-----------|---------|---------------|------|---------| +| n97-1 | GMK N97 | x86_64 | k3s server | 12GB | 233GB | +| n97-2 | GMK N97 | x86_64 | k3s worker | 12GB | ~233GB | +| n97-3 | GMK N97 | x86_64 | k3s worker | 12GB | ~233GB | +| rpi4-1 | RPi 4B | aarch64 | k3s worker | 8GB | SD/USB | +| rpi4-2 | RPi 4B | aarch64 | k3s worker | 8GB | SD/USB | + +## Network + +- **Mesh**: Tailscale (all nodes) +- **DNS**: os.scoble.me → n97-1 (Tailscale IP), SSL via Let's Encrypt + Cloudflare DNS +- **K8s API**: Tailscale-only, no public exposure + +## Repo Layout + +``` +├── README.md +├── flake.nix # NixOS flake — all 5 nodes +├── hardware-n97.nix # shared N97 perf tuning +├── hardware-rpi4.nix # shared RPi4 config +├── modules/ +│ ├── k3s-cluster.nix # shared k3s agent config +│ ├── meos-gateway.nix # nginx + acme + homepage +│ ├── ollama-worker.nix # Ollama on worker nodes +│ └── tailscale.nix # shared Tailscale config +├── nodes/ +│ ├── n97-1.nix # control plane + gateway +│ ├── n97-2.nix # Ollama worker x86 +│ ├── n97-3.nix # Ollama worker x86 +│ ├── rpi4-1.nix # ARM worker (light services) +│ └── rpi4-2.nix # ARM worker (umami) +├── apps/ # K8s manifests + Helm values +│ ├── forgejo/ +│ ├── postgres/ +│ ├── umami/ +│ ├── openbrain/ +│ ├── meos-dashboard/ +│ └── mcp-servers/ +├── scripts/ +│ ├── bootstrap-node.sh # initial node setup +│ └── migrate-db.sh # Postgres migration from bare to K8s +└── docs/ + └── migration.md # transition plan +``` + +## Quickstart + +```bash +# Build and deploy a node +nixos-rebuild switch --flake .#n97-1 --target-host root@100.x.x.x + +# Access cluster +kubectl --kubeconfig /etc/rancher/k3s/k3s.yaml get nodes + +# Deploy app +kubectl apply -k apps/forgejo/ +``` diff --git a/apps/forgejo/deployment.yaml b/apps/forgejo/deployment.yaml new file mode 100644 index 0000000..da57ef8 --- /dev/null +++ b/apps/forgejo/deployment.yaml @@ -0,0 +1,103 @@ +# Forgejo on k3s +--- +apiVersion: v1 +kind: Namespace +metadata: + name: forgejo +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: forgejo-data + namespace: forgejo +spec: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forgejo + namespace: forgejo +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: forgejo + template: + metadata: + labels: + app: forgejo + spec: + containers: + - name: forgejo + image: codeberg.org/forgejo/forgejo:11.0 + ports: + - containerPort: 3000 + name: http + - containerPort: 22 + name: ssh + env: + - name: USER_UID + value: "1000" + - name: USER_GID + value: "1000" + - name: FORGEJO__database__DB_TYPE + value: postgres + - name: FORGEJO__database__HOST + value: postgres-scobleclaw-rw.database.svc.cluster.local:5432 + - name: FORGEJO__database__NAME + value: forgejo + - name: FORGEJO__database__USER + value: forgejo + - name: FORGEJO__database__PASSWD + valueFrom: + secretKeyRef: + name: postgres-users + key: forgejo-password + - name: FORGEJO__server__ROOT_URL + value: "https://os.scoble.me/forgejo/" + - name: FORGEJO__server__HTTP_PORT + value: "3000" + - name: FORGEJO__server__DOMAIN + value: os.scoble.me + - name: FORGEJO__server__DISABLE_SSH + value: "false" + - name: FORGEJO__server__SSH_PORT + value: "2222" + volumeMounts: + - name: data + mountPath: /data + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "1" + volumes: + - name: data + persistentVolumeClaim: + claimName: forgejo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: forgejo + namespace: forgejo +spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + - name: ssh + port: 22 + targetPort: 22 + selector: + app: forgejo \ No newline at end of file diff --git a/apps/mcp-servers/deployment.yaml b/apps/mcp-servers/deployment.yaml new file mode 100644 index 0000000..cdb2daa --- /dev/null +++ b/apps/mcp-servers/deployment.yaml @@ -0,0 +1,83 @@ +# MCP Servers — lightweight Node.js workloads for ARM worker (rpi4-1) +# forgejo-mcp, lean-experiment-mcp, wardrobe-ai, homeschool +--- +apiVersion: v1 +kind: Namespace +metadata: + name: mcp-servers +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forgejo-mcp + namespace: mcp-servers +spec: + replicas: 1 + selector: + matchLabels: + app: forgejo-mcp + template: + metadata: + labels: + app: forgejo-mcp + spec: + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: "arm64" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: forgejo-mcp + image: node:22-slim + command: ["node", "/app/index.js"] + workingDir: /app + env: + - name: FORGEJO_URL + value: "http://forgejo.forgejo.svc.cluster.local:3000" + - name: FORGEJO_TOKEN + valueFrom: + secretKeyRef: + name: forgejo-tokens + key: agent-cos + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lean-experiment-mcp + namespace: mcp-servers +spec: + replicas: 1 + selector: + matchLabels: + app: lean-experiment-mcp + template: + metadata: + labels: + app: lean-experiment-mcp + spec: + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: "arm64" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: lean-mcp + image: node:22-slim + command: ["node", "/app/dist/index.js"] + workingDir: /app + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "100m" \ No newline at end of file diff --git a/apps/meos-dashboard/deployment.yaml b/apps/meos-dashboard/deployment.yaml new file mode 100644 index 0000000..937f728 --- /dev/null +++ b/apps/meos-dashboard/deployment.yaml @@ -0,0 +1,46 @@ +# MeOS Dashboard — stateless React app, can run on any node +--- +apiVersion: v1 +kind: Namespace +metadata: + name: meos-dashboard +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: meos-dashboard + namespace: meos-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: meos-dashboard + template: + metadata: + labels: + app: meos-dashboard + spec: + containers: + - name: dashboard + image: ghcr.io/scobleclaw/meos-dashboard:latest + ports: + - containerPort: 3000 + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "250m" +--- +apiVersion: v1 +kind: Service +metadata: + name: meos-dashboard + namespace: meos-dashboard +spec: + ports: + - port: 3000 + targetPort: 3000 + selector: + app: meos-dashboard \ No newline at end of file diff --git a/apps/openbrain/deployment.yaml b/apps/openbrain/deployment.yaml new file mode 100644 index 0000000..21da3d8 --- /dev/null +++ b/apps/openbrain/deployment.yaml @@ -0,0 +1,59 @@ +# OpenBrain MCP — deployed to ARM worker (rpi4-1) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openbrain +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: openbrain + namespace: openbrain +spec: + replicas: 1 + selector: + matchLabels: + app: openbrain + template: + metadata: + labels: + app: openbrain + spec: + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: "arm64" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: openbrain + image: node:22-slim + command: ["node", "/app/index.js"] + workingDir: /app + ports: + - containerPort: 3000 + env: + - name: NODE_ENV + value: production + - name: PORT + value: "3000" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "250m" +--- +apiVersion: v1 +kind: Service +metadata: + name: openbrain + namespace: openbrain +spec: + ports: + - port: 3000 + targetPort: 3000 + selector: + app: openbrain \ No newline at end of file diff --git a/apps/postgres/cluster.yaml b/apps/postgres/cluster.yaml new file mode 100644 index 0000000..b7177ab --- /dev/null +++ b/apps/postgres/cluster.yaml @@ -0,0 +1,59 @@ +# CloudNative PG operator (recommended for Postgres on K8s) +# https://cloudnative-pg.io/ +# +# Install: kubectl apply --server-side -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.25/releases/cnpg-1.25.1.yaml +# +# Then create the cluster below. +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: postgres-scobleclaw + namespace: database +spec: + instances: 1 + imageName: ghcr.io/cloudnative-pg/postgresql:17.4 + + storage: + size: 5Gi + storageClass: local-path + + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + + postgresql: + parameters: + max_connections: "200" + shared_buffers: "128MB" + + databases: + - name: forgejo + owner: forgejo + - name: umami + owner: umami + + users: + - name: forgejo + password: "" + - name: umami + password: "" + + enableSuperuserAccess: false + + monitoring: + enablePodMonitor: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-users + namespace: database +type: Opaque +stringData: + forgejo-password: "" + umami-password: "" diff --git a/apps/umami/deployment.yaml b/apps/umami/deployment.yaml new file mode 100644 index 0000000..a41bc31 --- /dev/null +++ b/apps/umami/deployment.yaml @@ -0,0 +1,67 @@ +# Umami analytics — deployed to ARM worker (rpi4-2) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: umami +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: umami + namespace: umami +spec: + replicas: 1 + selector: + matchLabels: + app: umami + template: + metadata: + labels: + app: umami + spec: + nodeSelector: + kubernetes.io/arch: arm64 + tolerations: + - key: "arm64" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: umami + image: ghcr.io/umami-software/umami:postgresql-v2 + ports: + - containerPort: 3000 + env: + - name: DATABASE_TYPE + value: postgresql + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: umami-db + key: url + - name: APP_SECRET + valueFrom: + secretKeyRef: + name: umami-secret + key: secret + - name: DISABLE_TELEMETRY + value: "1" + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" +--- +apiVersion: v1 +kind: Service +metadata: + name: umami + namespace: umami +spec: + ports: + - port: 3000 + targetPort: 3000 + selector: + app: umami \ No newline at end of file diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..7a0cd69 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,69 @@ +# Migration Plan: Bare Metal → k3s Cluster + +## Overview + +Migrate from single-node N97 (nixos) to 5-node k3s cluster with zero downtime. + +## Phase 1: Foundation (Day 1) + +1. **Create devops repo** ✅ (sscoble/devops) +2. **Bootstrap k3s on n97-1** — single-node control plane alongside existing services +3. **Install CNPG operator** — `kubectl apply -f cnpg-crds.yaml` +4. **Deploy local-path provisioner** — for persistent volumes on each node +5. **Install nginx ingress controller** — `helm install nginx-ingress ingress-nginx` + +At this point, k3s is running but all services are still bare metal. No traffic shifts yet. + +## Phase 2: Non-Critical Services (Day 2-3) + +Move services that don't affect the core workflow: + +1. **MeOS Dashboard → k3s** — deploy manifest, update nginx upstream +2. **MCP servers → rpi4-1** — forgejo-mcp, lean-experiment, wardrobe-ai +3. **Taint ARM nodes** — prevent x86 pods from landing on Pi 4s + +## Phase 3: Data Layer (Day 4-5) + +1. **Deploy CNPG Postgres cluster** — `kubectl apply -f apps/postgres/cluster.yaml` +2. **Run migration** — `./scripts/migrate-db.sh` +3. **Verify data integrity** — row counts, test queries +4. **Update Forgejo to point at CNPG** — change DATABASE_URL env var +5. **Disable bare-metal PostgreSQL** — set `services.postgresql.enable = false` + +## Phase 4: Core Services (Day 6-7) + +1. **Forgejo → k3s** — deploy manifest, cut over nginx upstream +2. **Umami → rpi4-2** — deploy manifest, verify analytics still collecting +3. **Homepage dashboard** — stays on n97-1 (lightweight, already working) + +## Phase 5: Scale Out (Day 8+) + +1. **Add n97-2 as worker** — `./scripts/bootstrap-node.sh n97-2 100.x.x.x agent` +2. **Add n97-3 as worker** — same +3. **Deploy Ollama on n97-2/n97-3** — systemd services, expose as ClusterIP +4. **Add rpi4-1/rpi4-2 as workers** — bootstrap with ARM64 support +5. **Configure node taints** — ARM workers for light work, N97s for heavy + +## What Stays Bare Metal + +- **OpenClaw Gateway** — manages processes, not containers. Stay outside k8s. +- **Nginx + ACME** — TLS termination on n97-1, routes to k8s Ingress +- **Ollama** — GPU/NPU passthrough is painful in k8s. Run as systemd services on workers. +- **Storj static hosting** — build + deploy pipeline stays in Nix flake +- **X11/LightDM** — only on n97-1 if you use the desktop + +## Rollback Strategy + +Each phase can be rolled back independently: +- **Phase 2**: Switch nginx upstreams back to bare-metal ports +- **Phase 3**: Re-enable `services.postgresql.enable = true`, restore from dump +- **Phase 4**: Same as Phase 2 — nginx upstream swap +- **Phase 5**: `kubectl drain && kubectl delete node ` + +## Post-Migration + +- Add `nodeAffinity` rules for Ollama pods (n97-2, n97-3 only) +- Set up `PodDisruptionBudgets` for critical services +- Consider HA control plane (3-server etcd) if uptime matters +- Add monitoring: Prometheus + Grafana on rpi4-2 +- Set up Velero for K8s backups \ No newline at end of file diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..82964f6 --- /dev/null +++ b/flake.nix @@ -0,0 +1,48 @@ +{ + description = "Scobleclaw Multi-Node Cluster — NixOS + k3s"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-25.11"; + }; + + outputs = { self, nixpkgs }: + let + shared-modules = [ + ./modules/tailscale.nix + ./modules/k3s-cluster.nix + ]; + + mkSystem = system: hostModule: nixpkgs.lib.nixosSystem { + inherit system; + modules = shared-modules ++ [ hostModule ]; + specialArgs = { inherit self; }; + }; + in { + nixosConfigurations = { + n97-1 = mkSystem "x86_64-linux" [ + ./hardware-n97.nix + ./nodes/n97-1.nix + ]; + + n97-2 = mkSystem "x86_64-linux" [ + ./hardware-n97.nix + ./nodes/n97-2.nix + ]; + + n97-3 = mkSystem "x86_64-linux" [ + ./hardware-n97.nix + ./nodes/n97-3.nix + ]; + + rpi4-1 = mkSystem "aarch64-linux" [ + ./hardware-rpi4.nix + ./nodes/rpi4-1.nix + ]; + + rpi4-2 = mkSystem "aarch64-linux" [ + ./hardware-rpi4.nix + ./nodes/rpi4-2.nix + ]; + }; + }; +} diff --git a/hardware-n97.nix b/hardware-n97.nix new file mode 100644 index 0000000..a1f7fb2 --- /dev/null +++ b/hardware-n97.nix @@ -0,0 +1,29 @@ +{ config, lib, pkgs, ... }: + +{ + # Intel N97 Mini PC performance tuning + boot = { + kernelPackages = pkgs.linuxPackages_latest; + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ]; + }; + + hardware = { + cpu.intel.updateMicrocode = true; + enableRedistributableFirmware = true; + opengl.enable = true; + }; + + services.xserver.videoDrivers = [ "intel" ]; + powerManagement.enable = true; + services.thermald.enable = true; + + # Local storage dir for k3s persistent volumes + systemd.tmpfiles.rules = [ + "d /data/k3s 0755 root root -" + "d /data/ollama 0755 root root -" + ]; +} diff --git a/hardware-rpi4.nix b/hardware-rpi4.nix new file mode 100644 index 0000000..aacc29c --- /dev/null +++ b/hardware-rpi4.nix @@ -0,0 +1,23 @@ +{ config, lib, pkgs, ... }: + +{ + # Raspberry Pi 4 — aarch64 worker node + boot = { + kernelPackages = pkgs.linuxPackages_rpi4; + loader = { + grub.enable = false; + generic-extlinux-compatible.enable = true; + }; + }; + + # ARM-specific performance + hardware.enableRedistributableFirmware = true; + + # Local storage + systemd.tmpfiles.rules = [ + "d /data/k3s 0755 root root -" + ]; + + # RPi 4 throttling protection + powerManagement.cpuFreqGovernor = "ondemand"; +} diff --git a/modules/k3s-cluster.nix b/modules/k3s-cluster.nix new file mode 100644 index 0000000..1b50529 --- /dev/null +++ b/modules/k3s-cluster.nix @@ -0,0 +1,76 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.k3s-cluster; +in { + options.services.k3s-cluster = { + enable = mkEnableOption "k3s cluster node"; + role = mkOption { + type = types.enum [ "server" "agent" ]; + default = "agent"; + description = "k3s role: server (control plane) or agent (worker)."; + }; + serverAddr = mkOption { + type = types.str; + default = "https://127.0.0.1:6443"; + description = "k3s server URL for agents to join."; + }; + tailscaleIP = mkOption { + type = types.nullOr types.str; + default = null; + description = "This node's Tailscale IP (for --advertise-address)."; + }; + tokenFile = mkOption { + type = types.nullOr types.str; + default = "/var/lib/k3s/cluster-token"; + description = "File containing the cluster shared secret."; + }; + withOllama = mkOption { + type = types.bool; + default = false; + description = "Enable Ollama on this node (GPU passthrough)."; + }; + }; + + config = mkIf cfg.enable { + services.k3s = { + enable = true; + role = cfg.role; + + extraFlags = with cfg; concatStringsSep " " ( + [ "--flannel-iface=tailscale0" ] + ++ optionals (tailscaleIP != null) [ + "--advertise-address=${tailscaleIP}" + "--node-ip=${tailscaleIP}" + ] + ++ optionals (role == "server") [ + "--cluster-init" + "--disable=traefik" + "--disable=servicelb" + "--kube-controller-manager-arg=bind-address=0.0.0.0" + ] + ); + + inherit tokenFile; + + # Allow pods on control plane (we need the resources) + inherit (cfg) serverAddr; + }; + + # Open k3s API port on control plane + networking.firewall.allowedTCPPorts = optionals (cfg.role == "server") [ 6443 ]; + + # ── Ollama (optional, on worker nodes with GPU/NPU) ── + services.ollama = mkIf cfg.withOllama { + enable = true; + acceleration = "opencl"; # Intel N97 iGPU + environmentVariables = { + OLLAMA_KEEP_ALIVE = "24h"; + OLLAMA_HOST = "0.0.0.0:11434"; + }; + openFirewall = true; + }; + }; +} diff --git a/modules/meos-gateway.nix b/modules/meos-gateway.nix new file mode 100644 index 0000000..524b99d --- /dev/null +++ b/modules/meos-gateway.nix @@ -0,0 +1,94 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.meos-gateway; + certDir = config.security.acme.certs."os.scoble.me".directory; +in { + options.services.meos-gateway = { + enable = mkEnableOption "MeOS Gateway — nginx reverse proxy for os.scoble.me"; + tailscaleIP = mkOption { + type = types.str; + description = "n97-1 Tailscale IP for nginx to bind to."; + }; + }; + + config = mkIf cfg.enable { + # ── Nginx — SSL-terminating reverse proxy ── + services.nginx = { + enable = true; + virtualHosts."os.scoble.me" = { + addSSL = true; + sslCertificate = "${certDir}/fullchain.pem"; + sslCertificateKey = "${certDir}/key.pem"; + listen = [ + { addr = cfg.tailscaleIP; port = 443; ssl = true; } + ]; + + locations = { + "/forgejo/" = { + proxyPass = "http://k3s-forgejo.kube-system.svc.cluster.local:3000/"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Prefix /forgejo; + ''; + }; + "/umami" = { + proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + "/umami/" = { + proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069"; + proxyWebsockets = true; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + ''; + }; + "/dashboard/" = { + proxyPass = "http://k3s-dashboard.kube-system.svc.cluster.local:3000/"; + proxyWebsockets = true; + }; + "/" = { + proxyPass = "http://127.0.0.1:8082"; + proxyWebsockets = true; + }; + }; + }; + + # HTTP -> HTTPS redirect + virtualHosts."os.scoble.me-http" = { + listen = [ { addr = cfg.tailscaleIP; port = 80; } ]; + serverName = "os.scoble.me"; + extraConfig = "return 301 https://$server_name$request_uri;"; + }; + }; + + # ACME with Cloudflare DNS challenge + security.acme = { + acceptTerms = true; + defaults.email = "scott@scoble.me"; + certs."os.scoble.me" = { + dnsProvider = "cloudflare"; + environmentFile = "/var/lib/acme/cloudflare.env"; + group = "nginx"; + }; + }; + + # Firewall + networking.firewall.allowedTCPPorts = [ 80 443 ]; + }; +} diff --git a/modules/tailscale.nix b/modules/tailscale.nix new file mode 100644 index 0000000..a20310b --- /dev/null +++ b/modules/tailscale.nix @@ -0,0 +1,34 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.tailscale-node; +in { + options.services.tailscale-node = { + enable = mkEnableOption "Tailscale mesh node"; + authKeyFile = mkOption { + type = types.nullOr types.str; + default = null; + description = "Path to file containing Tailscale pre-auth key (for unattended setup)."; + }; + }; + + config = mkIf cfg.enable { + services.tailscale = { + enable = true; + interfaceName = "tailscale0"; + authKeyFile = cfg.authKeyFile; + extraUpFlags = "--accept-routes --accept-dns=false"; + }; + + # Persist Tailscale state across reboots + systemd.services.tailscale = { + after = [ "network-online.target" ]; + wants = [ "network-online.target" ]; + serviceConfig.SupplementaryGroups = [ "nginx" ]; + }; + + networking.firewall.trustedInterfaces = [ "tailscale0" ]; + }; +} diff --git a/nodes/n97-1.nix b/nodes/n97-1.nix new file mode 100644 index 0000000..ae9276e --- /dev/null +++ b/nodes/n97-1.nix @@ -0,0 +1,68 @@ +{ config, lib, pkgs, ... }: + +let + # n97-1 Tailscale IP — update after initial Tailscale auth + tailscaleIP = "100.x.x.x"; +in { + imports = [ + ../modules/meos-gateway.nix + ]; + + networking.hostName = "n97-1"; + + # ── k3s server (single-node control plane) ── + services.k3s-cluster = { + enable = true; + role = "server"; + tailscaleIP = tailscaleIP; + tokenFile = "/var/lib/k3s/cluster-token"; + }; + + # ── Gateway (nginx + ACME) ── + services.meos-gateway = { + enable = true; + tailscaleIP = tailscaleIP; + }; + + # ── PostgreSQL (for Forgejo + Umami, transition to K8s) ── + services.postgresql = { + enable = true; + package = pkgs.postgresql_17; + extraPlugins = [ pkgs.postgresql_17.pkgs.pgvector ]; + dataDir = "/data/postgresql"; + }; + + # ── Forgejo (bare metal during transition, then K8s) ── + services.forgejo = { + enable = true; + database.type = "postgres"; + database.name = "forgejo"; + database.user = "forgejo"; + settings.server.ROOT_URL = "https://os.scoble.me/forgejo/"; + lfs.enable = true; + }; + + # ── OpenClaw gateway stays bare metal ── + # (managed separately via systemd user service) + + # ── Firewall ── + networking.firewall = { + enable = true; + trustedInterfaces = [ "tailscale0" ]; + allowedTCPPorts = [ 80 443 6443 ]; + }; + + # ── User ── + users.users.sscoble = { + isNormalUser = true; + extraGroups = [ "wheel" "docker" ]; + }; + + security.sudo.extraRules = [{ + users = [ "sscoble" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.settings.trusted-users = [ "sscoble" ]; +} diff --git a/nodes/n97-2.nix b/nodes/n97-2.nix new file mode 100644 index 0000000..5de8628 --- /dev/null +++ b/nodes/n97-2.nix @@ -0,0 +1,42 @@ +{ config, lib, pkgs, ... }: + +let + tailscaleIP = "100.x.x.x"; +in { + networking.hostName = "n97-2"; + + services.k3s-cluster = { + enable = true; + role = "agent"; + serverAddr = "https://100.x.x.x:6443"; # n97-1 IP + tailscaleIP = tailscaleIP; + tokenFile = "/var/lib/k3s/cluster-token"; + withOllama = true; + }; + + # Ollama models to serve + services.ollama.models = [ + "deepseek-v4-pro:cloud" # heavy reasoning + "kimi-k2.6:cloud" # backup reasoning + "glm-5.1:cloud" # fallback + ]; + + networking.firewall = { + enable = true; + trustedInterfaces = [ "tailscale0" ]; + allowedTCPPorts = [ 11434 ]; + }; + + users.users.sscoble = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + }; + + security.sudo.extraRules = [{ + users = [ "sscoble" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.settings.trusted-users = [ "sscoble" ]; +} diff --git a/nodes/n97-3.nix b/nodes/n97-3.nix new file mode 100644 index 0000000..f097c6f --- /dev/null +++ b/nodes/n97-3.nix @@ -0,0 +1,42 @@ +{ config, lib, pkgs, ... }: + +let + tailscaleIP = "100.x.x.x"; +in { + networking.hostName = "n97-3"; + + services.k3s-cluster = { + enable = true; + role = "agent"; + serverAddr = "https://100.x.x.x:6443"; # n97-1 IP + tailscaleIP = tailscaleIP; + tokenFile = "/var/lib/k3s/cluster-token"; + withOllama = true; + }; + + services.ollama.models = [ + "qwen3-coder-next:cloud" # targeted code + "deepseek-v4-flash:cloud" # fast default + "nomic-embed-text" # embeddings + "mxbai-embed-large" # embeddings + ]; + + networking.firewall = { + enable = true; + trustedInterfaces = [ "tailscale0" ]; + allowedTCPPorts = [ 11434 ]; + }; + + users.users.sscoble = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + }; + + security.sudo.extraRules = [{ + users = [ "sscoble" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.settings.trusted-users = [ "sscoble" ]; +} diff --git a/nodes/rpi4-1.nix b/nodes/rpi4-1.nix new file mode 100644 index 0000000..38d7e3f --- /dev/null +++ b/nodes/rpi4-1.nix @@ -0,0 +1,36 @@ +{ config, lib, pkgs, ... }: + +let + tailscaleIP = "100.x.x.x"; +in { + networking.hostName = "rpi4-1"; + + services.k3s-cluster = { + enable = true; + role = "agent"; + serverAddr = "https://100.x.x.x:6443"; # n97-1 IP + tailscaleIP = tailscaleIP; + tokenFile = "/var/lib/k3s/cluster-token"; + }; + + # ARM worker — light services only + # Taints/tolerations will be set via kubectl to prevent x86 pods landing here + + networking.firewall = { + enable = true; + trustedInterfaces = [ "tailscale0" ]; + }; + + users.users.sscoble = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + }; + + security.sudo.extraRules = [{ + users = [ "sscoble" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.settings.trusted-users = [ "sscoble" ]; +} diff --git a/nodes/rpi4-2.nix b/nodes/rpi4-2.nix new file mode 100644 index 0000000..c5ad02d --- /dev/null +++ b/nodes/rpi4-2.nix @@ -0,0 +1,36 @@ +{ config, lib, pkgs, ... }: + +let + tailscaleIP = "100.x.x.x"; +in { + networking.hostName = "rpi4-2"; + + services.k3s-cluster = { + enable = true; + role = "agent"; + serverAddr = "https://100.x.x.x:6443"; # n97-1 IP + tailscaleIP = tailscaleIP; + tokenFile = "/var/lib/k3s/cluster-token"; + }; + + # ARM worker — Umami and monitoring + # Taints/tolerations set via kubectl + + networking.firewall = { + enable = true; + trustedInterfaces = [ "tailscale0" ]; + }; + + users.users.sscoble = { + isNormalUser = true; + extraGroups = [ "wheel" ]; + }; + + security.sudo.extraRules = [{ + users = [ "sscoble" ]; + commands = [{ command = "ALL"; options = [ "NOPASSWD" ]; }]; + }]; + + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.settings.trusted-users = [ "sscoble" ]; +} diff --git a/scripts/bootstrap-node.sh b/scripts/bootstrap-node.sh new file mode 100755 index 0000000..3e3bf72 --- /dev/null +++ b/scripts/bootstrap-node.sh @@ -0,0 +1,58 @@ +# Bootstrap a new node for the Scobleclaw k3s cluster +# +# Usage: ./bootstrap-node.sh +# +# Prerequisites: +# - NixOS installed on target node +# - SSH access to root@ +# - This repo cloned locally +# +set -euo pipefail + +HOSTNAME="${1:?Usage: bootstrap-node.sh }" +TSIP="${2:?Need tailscale IP}" +ROLE="${3:-agent}" +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +echo "==> Bootstrapping $HOSTNAME ($TSIP) as k3s $ROLE" + +# 1. Generate cluster token if it doesn't exist +TOKEN_FILE="/var/lib/k3s/cluster-token" +if [ ! -f "$TOKEN_FILE" ]; then + echo "==> Generating k3s cluster token" + sudo mkdir -p /var/lib/k3s + head -c 32 /dev/urandom | base64 | sudo tee "$TOKEN_FILE" > /dev/null + sudo chmod 600 "$TOKEN_FILE" +fi + +# 2. Copy token to target node +echo "==> Copying cluster token to $HOSTNAME" +ssh "root@$TSIP" "mkdir -p /var/lib/k3s" +scp "$TOKEN_FILE" "root@$TSIP:$TOKEN_FILE" + +# 3. Copy NixOS config +echo "==> Copying NixOS configuration" +ssh "root@$TSIP" "mkdir -p /etc/nixos/nodes /etc/nixos/modules" +scp "$REPO_DIR/flake.nix" "root@$TSIP:/etc/nixos/flake.nix" +scp "$REPO_DIR/hardware-n97.nix" "root@$TSIP:/etc/nixos/hardware-n97.nix" 2>/dev/null || true +scp "$REPO_DIR/hardware-rpi4.nix" "root@$TSIP:/etc/nixos/hardware-rpi4.nix" 2>/dev/null || true +scp "$REPO_DIR/modules/"*.nix "root@$TSIP:/etc/nixos/modules/" +scp "$REPO_DIR/nodes/$HOSTNAME.nix" "root@$TSIP:/etc/nixos/nodes/" + +# 4. Build and switch +echo "==> Building NixOS configuration for $HOSTNAME" +ssh "root@$TSIP" "nixos-rebuild switch --flake /etc/nixos#$HOSTNAME" + +# 5. Wait for k3s to come up +echo "==> Waiting for k3s to start..." +sleep 10 + +if [ "$ROLE" = "server" ]; then + echo "==> k3s server is running. Get kubeconfig:" + echo " scp root@$TSIP:/etc/rancher/k3s/k3s.yaml ~/.kube/config-scobleclaw" + echo " sed -i 's/127.0.0.1/$TSIP/g' ~/.kube/config-scobleclaw" +else + echo "==> k3s agent joined. Verify with: kubectl get nodes" +fi + +echo "==> Done! $HOSTNAME is ready." \ No newline at end of file diff --git a/scripts/migrate-db.sh b/scripts/migrate-db.sh new file mode 100755 index 0000000..591bcfd --- /dev/null +++ b/scripts/migrate-db.sh @@ -0,0 +1,49 @@ +# Migrate PostgreSQL from bare metal (n97-1) to CloudNative PG (k3s) +# +# Prerequisites: +# - CNPG operator installed in k3s +# - postgres-scobleclaw cluster created (apps/postgres/cluster.yaml) +# - Both clusters accessible from n97-1 +# +set -euo pipefail + +SOURCE_HOST="127.0.0.1" +SOURCE_PORT="5432" +CNPG_HOST="postgres-scobleclaw-rw.database.svc.cluster.local" +CNPG_PORT="5432" + +echo "==> Starting PostgreSQL migration" +echo " Source: $SOURCE_HOST:$SOURCE_PORT (bare metal)" +echo " Target: $CNPG_HOST:$CNPG_PORT (k3s/CNPG)" + +# 1. Dump all databases +echo "==> Dumping databases from bare metal..." +sudo -u postgres pg_dumpall --clean --if-exists > /tmp/pg-migration-dump.sql + +# 2. Wait for CNPG cluster to be ready +echo "==> Waiting for CNPG cluster to be ready..." +kubectl -n database wait --for=condition=Ready cluster/postgres-scobleclaw --timeout=300s + +# 3. Get CNPG superuser password +CNPG_PASSWORD=$(kubectl -n database get secret postgres-scobleclaw-superuser -o jsonpath='{.data.password}' | base64 -d) + +# 4. Restore dump into CNPG +echo "==> Restoring dump into CNPG..." +PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres < /tmp/pg-migration-dump.sql + +# 5. Verify +echo "==> Verifying migration..." +DB_COUNT=$(PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres -t -c "SELECT count(*) FROM pg_database WHERE datname IN ('forgejo','umami');" | tr -d ' ') +echo " Databases found: $DB_COUNT (expected: 2)" + +if [ "$DB_COUNT" = "2" ]; then + echo "==> Migration successful! You can now disable bare-metal PostgreSQL:" + echo " Set services.postgresql.enable = false; in nodes/n97-1.nix" + echo " And rebuild: nixos-rebuild switch --flake .#n97-1" +else + echo "==> WARNING: Not all databases migrated. Check manually." + exit 1 +fi + +# Cleanup +rm -f /tmp/pg-migration-dump.sql \ No newline at end of file