ops: deploy the LiveKit SFU and TURN server to dev2

.github/workflows/deploy-livekit.yml

@@ -1,11 +1,14 @@
+# Deploy the PairUX LiveKit SFU (sfu.pairux.com), which runs on dev2 since it
+# left its DigitalOcean droplet on 2026-10-06. The compose file is in
+# apps/livekit/dev2/; secrets (livekit.yaml, egress.yaml) live only on the box.
 name: Deploy LiveKit SFU Server
 
 on:
   push:
     branches:
       - master
     paths:
-      - 'apps/livekit/**'
+      - 'apps/livekit/dev2/**'
       - '.github/workflows/deploy-livekit.yml'
   workflow_dispatch:
 
@@ -16,50 +19,44 @@ concurrency:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    name: Deploy to LiveKit Droplet
+    name: Deploy to dev2
+    timeout-minutes: 15
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Setup SSH
+      # ${{ }} values go through env, never straight into a run: body.
+      - name: Set up ssh
+        env:
+          SSH_KEY: ${{ secrets.DEV2_SSH_KEY }}
+          KNOWN_HOSTS: ${{ secrets.DEV2_KNOWN_HOSTS }}
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.DROPLET_SFU_SSH_KEY }}" > ~/.ssh/id_rsa
-          chmod 600 ~/.ssh/id_rsa
-          ssh-keyscan -p ${{ secrets.DROPLET_SFU_PORT || 22 }} ${{ secrets.DROPLET_SFU_HOST }} >> ~/.ssh/known_hosts
+          install -d -m 700 ~/.ssh
+          printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts
+          chmod 644 ~/.ssh/known_hosts
 
-      - name: Load env and deploy
+      - name: Deploy
         env:
-          SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }}
-          SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }}
-          SSH_USER: ${{ secrets.DROPLET_SFU_USER }}
-          ENV_FILE: ${{ secrets.ENV_FILE }}
+          DEV2_USER: ${{ secrets.DEV2_USER }}
+          DEV2_HOST: ${{ secrets.DEV2_HOST }}
         run: |
-          # Write ENV_FILE and source LIVEKIT vars
-          echo "$ENV_FILE" > /tmp/.env
-          source <(grep -E '^LIVEKIT_API_(KEY|SECRET)=' /tmp/.env)
-
-          # Copy setup script to droplet
-          scp -P $SSH_PORT apps/livekit/setup-livekit-server.sh $SSH_USER@$SSH_HOST:/tmp/
-
-          # Run setup script with values from ENV_FILE
-          ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo bash /tmp/setup-livekit-server.sh '$LIVEKIT_API_KEY' '$LIVEKIT_API_SECRET' '' 'sfu.pairux.com'"
+          dir=/home/anthony/www/sfu.pairux.com
+          scp -o BatchMode=yes apps/livekit/dev2/docker-compose.yml "$DEV2_USER@$DEV2_HOST:$dir/docker-compose.yml.new"
+          scp -o BatchMode=yes apps/livekit/dev2/deploy.sh "$DEV2_USER@$DEV2_HOST:$dir/deploy.sh"
+          ssh -o BatchMode=yes "$DEV2_USER@$DEV2_HOST" "chmod 755 $dir/deploy.sh && $dir/deploy.sh"
 
-          # Cleanup
-          rm -f /tmp/.env
-
-      - name: Verify LiveKit server
-        env:
-          SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }}
-          SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }}
-          SSH_USER: ${{ secrets.DROPLET_SFU_USER }}
+      - name: Verify sfu.pairux.com answers
         run: |
-          ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo docker compose -f /opt/pairux-livekit/docker-compose.yml ps"
-          ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo docker compose -f /opt/pairux-livekit/docker-compose.yml logs --tail=10"
+          for i in $(seq 1 10); do
+            code=$(curl -s -o /dev/null -w '%{http_code}' https://sfu.pairux.com/rtc/validate || true)
+            [ "$code" = 401 ] && { echo "sfu.pairux.com signalling up ($code)"; exit 0; }
+            echo "attempt $i: $code"; sleep 6
+          done
+          echo "sfu.pairux.com never answered"; exit 1
 
       - name: Cleanup
         if: always()
-        run: |
-          rm -f ~/.ssh/id_rsa
-          rm -f /tmp/.env
+        run: rm -f ~/.ssh/id_ed25519

.github/workflows/deploy-turn.yml

@@ -1,12 +1,15 @@
+# Deploy the PairUX TURN server (turn.pairux.com, coturn), which runs on dev2
+# since it left its DigitalOcean droplet on 2026-10-06. The compose file is in
+# apps/turn/dev2/; turnserver.conf (with the credential) lives only on the box.
 name: Deploy TURN Server
 
 on:
   push:
     branches:
       - master
     paths:
-      - 'apps/turn/**'
-      - '!apps/turn/package.json'
+      - 'apps/turn/dev2/**'
+      - '.github/workflows/deploy-turn.yml'
   workflow_dispatch:
 
 concurrency:
@@ -16,42 +19,45 @@ concurrency:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    name: Deploy to TURN Droplet
+    name: Deploy to dev2
+    timeout-minutes: 15
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Setup SSH
+      # ${{ }} values go through env, never straight into a run: body.
+      - name: Set up ssh
+        env:
+          SSH_KEY: ${{ secrets.DEV2_SSH_KEY }}
+          KNOWN_HOSTS: ${{ secrets.DEV2_KNOWN_HOSTS }}
         run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.DROPLET_SSH_KEY }}" > ~/.ssh/id_rsa
-          chmod 600 ~/.ssh/id_rsa
-          ssh-keyscan -p ${{ secrets.DROPLET_PORT || 22 }} ${{ secrets.DROPLET_HOST }} >> ~/.ssh/known_hosts
+          install -d -m 700 ~/.ssh
+          printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts
+          chmod 644 ~/.ssh/known_hosts
 
-      - name: Deploy setup script
+      - name: Deploy
         env:
-          SSH_HOST: ${{ secrets.DROPLET_HOST }}
-          SSH_PORT: ${{ secrets.DROPLET_PORT || 22 }}
-          SSH_USER: ${{ secrets.DROPLET_USER }}
-          TURN_PASSWORD: ${{ secrets.TURN_SERVER_CREDENTIAL }}
-          TURN_REALM: turn.pairux.com
+          DEV2_USER: ${{ secrets.DEV2_USER }}
+          DEV2_HOST: ${{ secrets.DEV2_HOST }}
         run: |
-          # Copy setup script to droplet
-          scp -P $SSH_PORT apps/turn/setup-turn-server.sh $SSH_USER@$SSH_HOST:/tmp/
-
-          # Run setup script
-          ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo bash /tmp/setup-turn-server.sh '$TURN_PASSWORD' '' '$TURN_REALM'"
+          dir=/home/anthony/www/turn.pairux.com
+          scp -o BatchMode=yes apps/turn/dev2/docker-compose.yml "$DEV2_USER@$DEV2_HOST:$dir/docker-compose.yml.new"
+          scp -o BatchMode=yes apps/turn/dev2/deploy.sh "$DEV2_USER@$DEV2_HOST:$dir/deploy.sh"
+          ssh -o BatchMode=yes "$DEV2_USER@$DEV2_HOST" "chmod 755 $dir/deploy.sh && $dir/deploy.sh"
 
-      - name: Verify TURN server
-        env:
-          SSH_HOST: ${{ secrets.DROPLET_HOST }}
-          SSH_PORT: ${{ secrets.DROPLET_PORT || 22 }}
-          SSH_USER: ${{ secrets.DROPLET_USER }}
+      - name: Verify turn.pairux.com TLS answers
         run: |
-          ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo systemctl status coturn --no-pager"
+          for i in $(seq 1 10); do
+            if echo | timeout 10 openssl s_client -connect turn.pairux.com:5349 -servername turn.pairux.com 2>/dev/null | grep -q 'Verify return code: 0'; then
+              echo "turn.pairux.com:5349 TLS ok"; exit 0
+            fi
+            echo "attempt $i"; sleep 6
+          done
+          echo "turn.pairux.com:5349 never answered"; exit 1
 
       - name: Cleanup
         if: always()
-        run: |
-          rm -f ~/.ssh/id_rsa
+        run: rm -f ~/.ssh/id_ed25519

.github/workflows/sfu-diag.yml

@@ -10,15 +10,15 @@ jobs:
       - name: Setup SSH
         run: |
           mkdir -p ~/.ssh
-          echo "${{ secrets.DROPLET_SFU_SSH_KEY }}" > ~/.ssh/id_rsa
+          echo "${{ secrets.DEV2_SSH_KEY }}" > ~/.ssh/id_rsa
           chmod 600 ~/.ssh/id_rsa
-          ssh-keyscan -p ${{ secrets.DROPLET_SFU_PORT || 22 }} ${{ secrets.DROPLET_SFU_HOST }} >> ~/.ssh/known_hosts
+          ssh-keyscan -p 22 ${{ secrets.DEV2_HOST }} >> ~/.ssh/known_hosts
 
       - name: Gather network + LiveKit ICE diagnostics
         env:
-          SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }}
-          SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }}
-          SSH_USER: ${{ secrets.DROPLET_SFU_USER }}
+          SSH_HOST: ${{ secrets.DEV2_HOST }}
+          SSH_PORT: 22
+          SSH_USER: ${{ secrets.DEV2_USER }}
         run: |
           run() { ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "$1"; }
           echo "===== ip route get 8.8.8.8 (default src IP) ====="
@@ -28,8 +28,8 @@ jobs:
           echo "===== default route ====="
           run "ip route show default"
           echo "===== livekit advertised IPs ====="
-          run "cd /opt/pairux-livekit && sudo docker compose logs livekit 2>/dev/null | grep -i 'using external IP' | tail -2"
+          run "cd /home/anthony/www/sfu.pairux.com && docker compose logs livekit 2>/dev/null | grep -i 'using external IP' | tail -2"
           echo "===== recent ICE / dtls / candidate events (last 30m) ====="
-          run "cd /opt/pairux-livekit && sudo docker compose logs --since 30m livekit 2>/dev/null | grep -iE 'ice|dtls|candidate|disconnect|connection failed|reconnect|selected pair' | tail -60"
+          run "cd /home/anthony/www/sfu.pairux.com && docker compose logs --since 30m livekit 2>/dev/null | grep -iE 'ice|dtls|candidate|disconnect|connection failed|reconnect|selected pair' | tail -60"
           echo "===== egress recent ====="
-          run "cd /opt/pairux-livekit && sudo docker compose logs --since 30m egress 2>/dev/null | tail -20"
+          run "cd /home/anthony/www/sfu.pairux.com && docker compose logs --since 30m egress 2>/dev/null | tail -20"

apps/livekit/dev2/deploy.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Box-side deploy for the PairUX LiveKit SFU on dev2 (sfu.pairux.com).
+#
+# deploy-livekit.yml copies docker-compose.yml here as docker-compose.yml.new
+# and runs this script. Secrets live only on the box in conf/ (livekit.yaml,
+# egress.yaml), never in the repo. An unchanged compose file is a no-op, so a
+# push that touches nothing here does not restart the SFU or drop live calls.
+set -euo pipefail
+
+ROOT=/home/anthony/www/sfu.pairux.com
+cd "$ROOT"
+
+if [ -f docker-compose.yml.new ]; then
+  docker compose -f docker-compose.yml.new config -q
+  if cmp -s docker-compose.yml.new docker-compose.yml; then
+    rm -f docker-compose.yml.new
+    echo "compose file unchanged"
+  else
+    cp docker-compose.yml docker-compose.yml.prev
+    mv docker-compose.yml.new docker-compose.yml
+    echo "compose file updated (previous kept as docker-compose.yml.prev)"
+  fi
+fi
+
+docker compose pull -q
+docker compose up -d
+
+for _ in $(seq 1 30); do
+  if curl -fsS -o /dev/null http://127.0.0.1:7880/; then
+    docker compose ps
+    exit 0
+  fi
+  sleep 2
+done
+
+echo "livekit did not answer on 127.0.0.1:7880" >&2
+if [ -f docker-compose.yml.prev ]; then
+  echo "rolling back to docker-compose.yml.prev" >&2
+  mv docker-compose.yml.prev docker-compose.yml
+  docker compose up -d
+fi
+exit 1

apps/livekit/dev2/docker-compose.yml

@@ -0,0 +1,35 @@
+# PairUX LiveKit SFU on dev2 (moved off DO droplet sfu.pairux.com 2026-10-06).
+# Host networking: LiveKit needs real UDP. Public ports: 7881/tcp, 7882/udp,
+# 61100-61300/udp (rtc range), 3480/udp (LiveKit's own TURN). 7880 (signalling)
+# is local only, fronted by nginx wss://sfu.pairux.com. Redis is a private bus on 127.0.0.1:6390.
+name: pairux-sfu
+services:
+  redis:
+    image: redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499
+    container_name: pairux-sfu-redis
+    restart: unless-stopped
+    network_mode: host
+    command: redis-server --bind 127.0.0.1 --port 6390 --save "" --appendonly no
+
+  livekit:
+    image: livekit/livekit-server@sha256:6fd3b7088874c4d119160dd688798dfec852bc014786d392caad15f6f63912a3
+    container_name: pairux-livekit
+    restart: unless-stopped
+    network_mode: host
+    depends_on: [redis]
+    volumes:
+      - ./conf/livekit.yaml:/etc/livekit.yaml:ro
+    command: --config /etc/livekit.yaml --bind 127.0.0.1
+
+  # Server-side restreamer / recorder (room composite -> RTMP / file).
+  egress:
+    image: livekit/egress@sha256:bf2b648b947349c3e9ff7aa8c718f00378d5c06af7624652a3653318e00333ce
+    container_name: pairux-egress
+    restart: unless-stopped
+    network_mode: host
+    depends_on: [redis, livekit]
+    environment:
+      - EGRESS_CONFIG_FILE=/etc/egress.yaml
+    volumes:
+      - ./conf/egress.yaml:/etc/egress.yaml:ro
+    cap_add: [SYS_ADMIN]

apps/turn/dev2/deploy.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Box-side deploy for the PairUX coturn on dev2 (turn.pairux.com).
+#
+# deploy-turn.yml copies docker-compose.yml here as docker-compose.yml.new and
+# runs this script. turnserver.conf (with the TURN credential) lives only on the
+# box in conf/; the cert is renewed by root's acme.sh into certs/. An unchanged
+# compose file is a no-op, so live relays are not cut.
+set -euo pipefail
+
+ROOT=/home/anthony/www/turn.pairux.com
+cd "$ROOT"
+
+if [ -f docker-compose.yml.new ]; then
+  docker compose -f docker-compose.yml.new config -q
+  if cmp -s docker-compose.yml.new docker-compose.yml; then
+    rm -f docker-compose.yml.new
+    echo "compose file unchanged"
+  else
+    cp docker-compose.yml docker-compose.yml.prev
+    mv docker-compose.yml.new docker-compose.yml
+    echo "compose file updated (previous kept as docker-compose.yml.prev)"
+  fi
+fi
+
+docker compose pull -q
+docker compose up -d
+
+sleep 5
+if [ "$(docker inspect -f '{{.State.Running}} {{.RestartCount}}' pairux-coturn)" = "true 0" ]; then
+  docker compose ps
+  exit 0
+fi
+
+echo "coturn is not running cleanly" >&2
+docker logs --tail 20 pairux-coturn >&2 || true
+if [ -f docker-compose.yml.prev ]; then
+  echo "rolling back to docker-compose.yml.prev" >&2
+  mv docker-compose.yml.prev docker-compose.yml
+  docker compose up -d
+fi
+exit 1

apps/turn/dev2/docker-compose.yml

@@ -0,0 +1,15 @@
+# PairUX coturn on dev2 (moved off DO droplet turn.pairux.com 2026-10-06).
+# Runs as uid 3478 (pairux-turn) so ufw before.rules can confine relays to
+# dev2's own IP to LiveKit's ports. Public: 3478 tcp/udp, 5349 tcp/udp, relay 61400-61500.
+name: pairux-turn
+services:
+  coturn:
+    image: coturn/coturn@sha256:6a1d1a281b8f64ca1a343429bb0232fa70c5f0eae3c8424ba0859b696e880974
+    container_name: pairux-coturn
+    network_mode: host
+    restart: unless-stopped
+    user: "3478:3478"
+    volumes:
+      - ./conf/turnserver.conf:/etc/coturn/turnserver.conf:ro
+      - ./certs:/certs:ro
+    command: ["-c", "/etc/coturn/turnserver.conf"]