sscoble/devops
1
0
Fork 0
Code Issues 9 Pull requests Projects Releases Packages Wiki Activity Actions

Initial cluster config: NixOS flake + k3s + 5 node definitions

Browse source 
- Multi-host flake.nix (3x N97 x86_64 + 2x RPi4 aarch64)
- Shared modules: k3s-cluster, tailscale, meos-gateway
- Node configs: n97-1 (control plane), n97-2/n97-3 (Ollama workers), rpi4-1/rpi4-2 (ARM workers)
- K8s manifests: Forgejo, Postgres (CNPG), Umami, OpenBrain, MeOS Dashboard, MCP servers
- Bootstrap and DB migration scripts
- Migration plan doc (7-day phased rollout)
This commit is contained in:
cos Agent 2026-05-04 12:22:09 -07:00
parent 791a9b6721
commit 7b5ffe6b11
21 changed files with 1184 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

58
scripts/bootstrap-node.sh Executable file
View file

@ -0,0 +1,58 @@
# Bootstrap a new node for the Scobleclaw k3s cluster
#
# Usage: ./bootstrap-node.sh <hostname> <tailscale-ip> <role: server|agent>
#
# Prerequisites:
# - NixOS installed on target node
# - SSH access to root@<tailscale-ip>
# - This repo cloned locally
#
set -euo pipefail
HOSTNAME="${1:?Usage: bootstrap-node.sh <hostname> <tailscale-ip> <role>}"
TSIP="${2:?Need tailscale IP}"
ROLE="${3:-agent}"
REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
echo "==> Bootstrapping $HOSTNAME ($TSIP) as k3s $ROLE"
# 1. Generate cluster token if it doesn't exist
TOKEN_FILE="/var/lib/k3s/cluster-token"
if [ ! -f "$TOKEN_FILE" ]; then
echo "==> Generating k3s cluster token"
sudo mkdir -p /var/lib/k3s
head -c 32 /dev/urandom | base64 | sudo tee "$TOKEN_FILE" > /dev/null
sudo chmod 600 "$TOKEN_FILE"
fi
# 2. Copy token to target node
echo "==> Copying cluster token to $HOSTNAME"
ssh "root@$TSIP" "mkdir -p /var/lib/k3s"
scp "$TOKEN_FILE" "root@$TSIP:$TOKEN_FILE"
# 3. Copy NixOS config
echo "==> Copying NixOS configuration"
ssh "root@$TSIP" "mkdir -p /etc/nixos/nodes /etc/nixos/modules"
scp "$REPO_DIR/flake.nix" "root@$TSIP:/etc/nixos/flake.nix"
scp "$REPO_DIR/hardware-n97.nix" "root@$TSIP:/etc/nixos/hardware-n97.nix" 2>/dev/null || true
scp "$REPO_DIR/hardware-rpi4.nix" "root@$TSIP:/etc/nixos/hardware-rpi4.nix" 2>/dev/null || true
scp "$REPO_DIR/modules/"*.nix "root@$TSIP:/etc/nixos/modules/"
scp "$REPO_DIR/nodes/$HOSTNAME.nix" "root@$TSIP:/etc/nixos/nodes/"
# 4. Build and switch
echo "==> Building NixOS configuration for $HOSTNAME"
ssh "root@$TSIP" "nixos-rebuild switch --flake /etc/nixos#$HOSTNAME"
# 5. Wait for k3s to come up
echo "==> Waiting for k3s to start..."
sleep 10
if [ "$ROLE" = "server" ]; then
echo "==> k3s server is running. Get kubeconfig:"
echo " scp root@$TSIP:/etc/rancher/k3s/k3s.yaml ~/.kube/config-scobleclaw"
echo " sed -i 's/127.0.0.1/$TSIP/g' ~/.kube/config-scobleclaw"
else
echo "==> k3s agent joined. Verify with: kubectl get nodes"
fi
echo "==> Done! $HOSTNAME is ready."

49
scripts/migrate-db.sh Executable file
View file

@ -0,0 +1,49 @@
# Migrate PostgreSQL from bare metal (n97-1) to CloudNative PG (k3s)
#
# Prerequisites:
# - CNPG operator installed in k3s
# - postgres-scobleclaw cluster created (apps/postgres/cluster.yaml)
# - Both clusters accessible from n97-1
#
set -euo pipefail
SOURCE_HOST="127.0.0.1"
SOURCE_PORT="5432"
CNPG_HOST="postgres-scobleclaw-rw.database.svc.cluster.local"
CNPG_PORT="5432"
echo "==> Starting PostgreSQL migration"
echo " Source: $SOURCE_HOST:$SOURCE_PORT (bare metal)"
echo " Target: $CNPG_HOST:$CNPG_PORT (k3s/CNPG)"
# 1. Dump all databases
echo "==> Dumping databases from bare metal..."
sudo -u postgres pg_dumpall --clean --if-exists > /tmp/pg-migration-dump.sql
# 2. Wait for CNPG cluster to be ready
echo "==> Waiting for CNPG cluster to be ready..."
kubectl -n database wait --for=condition=Ready cluster/postgres-scobleclaw --timeout=300s
# 3. Get CNPG superuser password
CNPG_PASSWORD=$(kubectl -n database get secret postgres-scobleclaw-superuser -o jsonpath='{.data.password}' | base64 -d)
# 4. Restore dump into CNPG
echo "==> Restoring dump into CNPG..."
PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres < /tmp/pg-migration-dump.sql
# 5. Verify
echo "==> Verifying migration..."
DB_COUNT=$(PGPASSWORD="$CNPG_PASSWORD" psql -h "$CNPG_HOST" -p "$CNPG_PORT" -U postgres -t -c "SELECT count(*) FROM pg_database WHERE datname IN ('forgejo','umami');" | tr -d ' ')
echo " Databases found: $DB_COUNT (expected: 2)"
if [ "$DB_COUNT" = "2" ]; then
echo "==> Migration successful! You can now disable bare-metal PostgreSQL:"
echo " Set services.postgresql.enable = false; in nodes/n97-1.nix"
echo " And rebuild: nixos-rebuild switch --flake .#n97-1"
else
echo "==> WARNING: Not all databases migrated. Check manually."
exit 1
fi
# Cleanup
rm -f /tmp/pg-migration-dump.sql