Initial cluster config: NixOS flake + k3s + 5 node definitions

- Multi-host flake.nix (3x N97 x86_64 + 2x RPi4 aarch64)
- Shared modules: k3s-cluster, tailscale, meos-gateway
- Node configs: n97-1 (control plane), n97-2/n97-3 (Ollama workers), rpi4-1/rpi4-2 (ARM workers)
- K8s manifests: Forgejo, Postgres (CNPG), Umami, OpenBrain, MeOS Dashboard, MCP servers
- Bootstrap and DB migration scripts
- Migration plan doc (7-day phased rollout)
This commit is contained in:
cos Agent 2026-05-04 12:22:09 -07:00
parent 791a9b6721
commit 7b5ffe6b11
21 changed files with 1184 additions and 2 deletions

76
modules/k3s-cluster.nix Normal file
View file

@ -0,0 +1,76 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.k3s-cluster;
in {
options.services.k3s-cluster = {
enable = mkEnableOption "k3s cluster node";
role = mkOption {
type = types.enum [ "server" "agent" ];
default = "agent";
description = "k3s role: server (control plane) or agent (worker).";
};
serverAddr = mkOption {
type = types.str;
default = "https://127.0.0.1:6443";
description = "k3s server URL for agents to join.";
};
tailscaleIP = mkOption {
type = types.nullOr types.str;
default = null;
description = "This node's Tailscale IP (for --advertise-address).";
};
tokenFile = mkOption {
type = types.nullOr types.str;
default = "/var/lib/k3s/cluster-token";
description = "File containing the cluster shared secret.";
};
withOllama = mkOption {
type = types.bool;
default = false;
description = "Enable Ollama on this node (GPU passthrough).";
};
};
config = mkIf cfg.enable {
services.k3s = {
enable = true;
role = cfg.role;
extraFlags = with cfg; concatStringsSep " " (
[ "--flannel-iface=tailscale0" ]
++ optionals (tailscaleIP != null) [
"--advertise-address=${tailscaleIP}"
"--node-ip=${tailscaleIP}"
]
++ optionals (role == "server") [
"--cluster-init"
"--disable=traefik"
"--disable=servicelb"
"--kube-controller-manager-arg=bind-address=0.0.0.0"
]
);
inherit tokenFile;
# Allow pods on control plane (we need the resources)
inherit (cfg) serverAddr;
};
# Open k3s API port on control plane
networking.firewall.allowedTCPPorts = optionals (cfg.role == "server") [ 6443 ];
# ── Ollama (optional, on worker nodes with GPU/NPU) ──
services.ollama = mkIf cfg.withOllama {
enable = true;
acceleration = "opencl"; # Intel N97 iGPU
environmentVariables = {
OLLAMA_KEEP_ALIVE = "24h";
OLLAMA_HOST = "0.0.0.0:11434";
};
openFirewall = true;
};
};
}

94
modules/meos-gateway.nix Normal file
View file

@ -0,0 +1,94 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.meos-gateway;
certDir = config.security.acme.certs."os.scoble.me".directory;
in {
options.services.meos-gateway = {
enable = mkEnableOption "MeOS Gateway — nginx reverse proxy for os.scoble.me";
tailscaleIP = mkOption {
type = types.str;
description = "n97-1 Tailscale IP for nginx to bind to.";
};
};
config = mkIf cfg.enable {
# ── Nginx — SSL-terminating reverse proxy ──
services.nginx = {
enable = true;
virtualHosts."os.scoble.me" = {
addSSL = true;
sslCertificate = "${certDir}/fullchain.pem";
sslCertificateKey = "${certDir}/key.pem";
listen = [
{ addr = cfg.tailscaleIP; port = 443; ssl = true; }
];
locations = {
"/forgejo/" = {
proxyPass = "http://k3s-forgejo.kube-system.svc.cluster.local:3000/";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Prefix /forgejo;
'';
};
"/umami" = {
proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
"/umami/" = {
proxyPass = "http://k3s-umami.kube-system.svc.cluster.local:8069";
proxyWebsockets = true;
extraConfig = ''
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
'';
};
"/dashboard/" = {
proxyPass = "http://k3s-dashboard.kube-system.svc.cluster.local:3000/";
proxyWebsockets = true;
};
"/" = {
proxyPass = "http://127.0.0.1:8082";
proxyWebsockets = true;
};
};
};
# HTTP -> HTTPS redirect
virtualHosts."os.scoble.me-http" = {
listen = [ { addr = cfg.tailscaleIP; port = 80; } ];
serverName = "os.scoble.me";
extraConfig = "return 301 https://$server_name$request_uri;";
};
};
# ACME with Cloudflare DNS challenge
security.acme = {
acceptTerms = true;
defaults.email = "scott@scoble.me";
certs."os.scoble.me" = {
dnsProvider = "cloudflare";
environmentFile = "/var/lib/acme/cloudflare.env";
group = "nginx";
};
};
# Firewall
networking.firewall.allowedTCPPorts = [ 80 443 ];
};
}

34
modules/tailscale.nix Normal file
View file

@ -0,0 +1,34 @@
{ config, lib, pkgs, ... }:
with lib;
let
cfg = config.services.tailscale-node;
in {
options.services.tailscale-node = {
enable = mkEnableOption "Tailscale mesh node";
authKeyFile = mkOption {
type = types.nullOr types.str;
default = null;
description = "Path to file containing Tailscale pre-auth key (for unattended setup).";
};
};
config = mkIf cfg.enable {
services.tailscale = {
enable = true;
interfaceName = "tailscale0";
authKeyFile = cfg.authKeyFile;
extraUpFlags = "--accept-routes --accept-dns=false";
};
# Persist Tailscale state across reboots
systemd.services.tailscale = {
after = [ "network-online.target" ];
wants = [ "network-online.target" ];
serviceConfig.SupplementaryGroups = [ "nginx" ];
};
networking.firewall.trustedInterfaces = [ "tailscale0" ];
};
}