diff --git a/.github/workflows/deploy-livekit.yml b/.github/workflows/deploy-livekit.yml index bd2ba52..e3ef587 100644 --- a/.github/workflows/deploy-livekit.yml +++ b/.github/workflows/deploy-livekit.yml @@ -1,3 +1,6 @@ +# Deploy the PairUX LiveKit SFU (sfu.pairux.com), which runs on dev2 since it +# left its DigitalOcean droplet on 2026-10-06. The compose file is in +# apps/livekit/dev2/; secrets (livekit.yaml, egress.yaml) live only on the box. name: Deploy LiveKit SFU Server on: @@ -5,7 +8,7 @@ on: branches: - master paths: - - 'apps/livekit/**' + - 'apps/livekit/dev2/**' - '.github/workflows/deploy-livekit.yml' workflow_dispatch: @@ -16,50 +19,44 @@ concurrency: jobs: deploy: runs-on: ubuntu-latest - name: Deploy to LiveKit Droplet + name: Deploy to dev2 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v6 - - name: Setup SSH + # ${{ }} values go through env, never straight into a run: body. + - name: Set up ssh + env: + SSH_KEY: ${{ secrets.DEV2_SSH_KEY }} + KNOWN_HOSTS: ${{ secrets.DEV2_KNOWN_HOSTS }} run: | - mkdir -p ~/.ssh - echo "${{ secrets.DROPLET_SFU_SSH_KEY }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p ${{ secrets.DROPLET_SFU_PORT || 22 }} ${{ secrets.DROPLET_SFU_HOST }} >> ~/.ssh/known_hosts + install -d -m 700 ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 644 ~/.ssh/known_hosts - - name: Load env and deploy + - name: Deploy env: - SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }} - SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }} - SSH_USER: ${{ secrets.DROPLET_SFU_USER }} - ENV_FILE: ${{ secrets.ENV_FILE }} + DEV2_USER: ${{ secrets.DEV2_USER }} + DEV2_HOST: ${{ secrets.DEV2_HOST }} run: | - # Write ENV_FILE and source LIVEKIT vars - echo "$ENV_FILE" > /tmp/.env - source <(grep -E '^LIVEKIT_API_(KEY|SECRET)=' /tmp/.env) - - # Copy setup script to droplet - scp -P $SSH_PORT apps/livekit/setup-livekit-server.sh $SSH_USER@$SSH_HOST:/tmp/ - - # Run setup script with values from ENV_FILE - ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo bash /tmp/setup-livekit-server.sh '$LIVEKIT_API_KEY' '$LIVEKIT_API_SECRET' '' 'sfu.pairux.com'" + dir=/home/anthony/www/sfu.pairux.com + scp -o BatchMode=yes apps/livekit/dev2/docker-compose.yml "$DEV2_USER@$DEV2_HOST:$dir/docker-compose.yml.new" + scp -o BatchMode=yes apps/livekit/dev2/deploy.sh "$DEV2_USER@$DEV2_HOST:$dir/deploy.sh" + ssh -o BatchMode=yes "$DEV2_USER@$DEV2_HOST" "chmod 755 $dir/deploy.sh && $dir/deploy.sh" - # Cleanup - rm -f /tmp/.env - - - name: Verify LiveKit server - env: - SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }} - SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }} - SSH_USER: ${{ secrets.DROPLET_SFU_USER }} + - name: Verify sfu.pairux.com answers run: | - ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo docker compose -f /opt/pairux-livekit/docker-compose.yml ps" - ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo docker compose -f /opt/pairux-livekit/docker-compose.yml logs --tail=10" + for i in $(seq 1 10); do + code=$(curl -s -o /dev/null -w '%{http_code}' https://sfu.pairux.com/rtc/validate || true) + [ "$code" = 401 ] && { echo "sfu.pairux.com signalling up ($code)"; exit 0; } + echo "attempt $i: $code"; sleep 6 + done + echo "sfu.pairux.com never answered"; exit 1 - name: Cleanup if: always() - run: | - rm -f ~/.ssh/id_rsa - rm -f /tmp/.env + run: rm -f ~/.ssh/id_ed25519 diff --git a/.github/workflows/deploy-turn.yml b/.github/workflows/deploy-turn.yml index ac5a4d6..a3540ff 100644 --- a/.github/workflows/deploy-turn.yml +++ b/.github/workflows/deploy-turn.yml @@ -1,3 +1,6 @@ +# Deploy the PairUX TURN server (turn.pairux.com, coturn), which runs on dev2 +# since it left its DigitalOcean droplet on 2026-10-06. The compose file is in +# apps/turn/dev2/; turnserver.conf (with the credential) lives only on the box. name: Deploy TURN Server on: @@ -5,8 +8,8 @@ on: branches: - master paths: - - 'apps/turn/**' - - '!apps/turn/package.json' + - 'apps/turn/dev2/**' + - '.github/workflows/deploy-turn.yml' workflow_dispatch: concurrency: @@ -16,42 +19,45 @@ concurrency: jobs: deploy: runs-on: ubuntu-latest - name: Deploy to TURN Droplet + name: Deploy to dev2 + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v6 - - name: Setup SSH + # ${{ }} values go through env, never straight into a run: body. + - name: Set up ssh + env: + SSH_KEY: ${{ secrets.DEV2_SSH_KEY }} + KNOWN_HOSTS: ${{ secrets.DEV2_KNOWN_HOSTS }} run: | - mkdir -p ~/.ssh - echo "${{ secrets.DROPLET_SSH_KEY }}" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p ${{ secrets.DROPLET_PORT || 22 }} ${{ secrets.DROPLET_HOST }} >> ~/.ssh/known_hosts + install -d -m 700 ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 644 ~/.ssh/known_hosts - - name: Deploy setup script + - name: Deploy env: - SSH_HOST: ${{ secrets.DROPLET_HOST }} - SSH_PORT: ${{ secrets.DROPLET_PORT || 22 }} - SSH_USER: ${{ secrets.DROPLET_USER }} - TURN_PASSWORD: ${{ secrets.TURN_SERVER_CREDENTIAL }} - TURN_REALM: turn.pairux.com + DEV2_USER: ${{ secrets.DEV2_USER }} + DEV2_HOST: ${{ secrets.DEV2_HOST }} run: | - # Copy setup script to droplet - scp -P $SSH_PORT apps/turn/setup-turn-server.sh $SSH_USER@$SSH_HOST:/tmp/ - - # Run setup script - ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo bash /tmp/setup-turn-server.sh '$TURN_PASSWORD' '' '$TURN_REALM'" + dir=/home/anthony/www/turn.pairux.com + scp -o BatchMode=yes apps/turn/dev2/docker-compose.yml "$DEV2_USER@$DEV2_HOST:$dir/docker-compose.yml.new" + scp -o BatchMode=yes apps/turn/dev2/deploy.sh "$DEV2_USER@$DEV2_HOST:$dir/deploy.sh" + ssh -o BatchMode=yes "$DEV2_USER@$DEV2_HOST" "chmod 755 $dir/deploy.sh && $dir/deploy.sh" - - name: Verify TURN server - env: - SSH_HOST: ${{ secrets.DROPLET_HOST }} - SSH_PORT: ${{ secrets.DROPLET_PORT || 22 }} - SSH_USER: ${{ secrets.DROPLET_USER }} + - name: Verify turn.pairux.com TLS answers run: | - ssh -p $SSH_PORT $SSH_USER@$SSH_HOST "sudo systemctl status coturn --no-pager" + for i in $(seq 1 10); do + if echo | timeout 10 openssl s_client -connect turn.pairux.com:5349 -servername turn.pairux.com 2>/dev/null | grep -q 'Verify return code: 0'; then + echo "turn.pairux.com:5349 TLS ok"; exit 0 + fi + echo "attempt $i"; sleep 6 + done + echo "turn.pairux.com:5349 never answered"; exit 1 - name: Cleanup if: always() - run: | - rm -f ~/.ssh/id_rsa + run: rm -f ~/.ssh/id_ed25519 diff --git a/.github/workflows/sfu-diag.yml b/.github/workflows/sfu-diag.yml index ce252f5..4057ffe 100644 --- a/.github/workflows/sfu-diag.yml +++ b/.github/workflows/sfu-diag.yml @@ -10,15 +10,15 @@ jobs: - name: Setup SSH run: | mkdir -p ~/.ssh - echo "${{ secrets.DROPLET_SFU_SSH_KEY }}" > ~/.ssh/id_rsa + echo "${{ secrets.DEV2_SSH_KEY }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa - ssh-keyscan -p ${{ secrets.DROPLET_SFU_PORT || 22 }} ${{ secrets.DROPLET_SFU_HOST }} >> ~/.ssh/known_hosts + ssh-keyscan -p 22 ${{ secrets.DEV2_HOST }} >> ~/.ssh/known_hosts - name: Gather network + LiveKit ICE diagnostics env: - SSH_HOST: ${{ secrets.DROPLET_SFU_HOST }} - SSH_PORT: ${{ secrets.DROPLET_SFU_PORT || 22 }} - SSH_USER: ${{ secrets.DROPLET_SFU_USER }} + SSH_HOST: ${{ secrets.DEV2_HOST }} + SSH_PORT: 22 + SSH_USER: ${{ secrets.DEV2_USER }} run: | run() { ssh -p "$SSH_PORT" "$SSH_USER@$SSH_HOST" "$1"; } echo "===== ip route get 8.8.8.8 (default src IP) =====" @@ -28,8 +28,8 @@ jobs: echo "===== default route =====" run "ip route show default" echo "===== livekit advertised IPs =====" - run "cd /opt/pairux-livekit && sudo docker compose logs livekit 2>/dev/null | grep -i 'using external IP' | tail -2" + run "cd /home/anthony/www/sfu.pairux.com && docker compose logs livekit 2>/dev/null | grep -i 'using external IP' | tail -2" echo "===== recent ICE / dtls / candidate events (last 30m) =====" - run "cd /opt/pairux-livekit && sudo docker compose logs --since 30m livekit 2>/dev/null | grep -iE 'ice|dtls|candidate|disconnect|connection failed|reconnect|selected pair' | tail -60" + run "cd /home/anthony/www/sfu.pairux.com && docker compose logs --since 30m livekit 2>/dev/null | grep -iE 'ice|dtls|candidate|disconnect|connection failed|reconnect|selected pair' | tail -60" echo "===== egress recent =====" - run "cd /opt/pairux-livekit && sudo docker compose logs --since 30m egress 2>/dev/null | tail -20" + run "cd /home/anthony/www/sfu.pairux.com && docker compose logs --since 30m egress 2>/dev/null | tail -20" diff --git a/apps/livekit/dev2/deploy.sh b/apps/livekit/dev2/deploy.sh new file mode 100755 index 0000000..e63c30d --- /dev/null +++ b/apps/livekit/dev2/deploy.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Box-side deploy for the PairUX LiveKit SFU on dev2 (sfu.pairux.com). +# +# deploy-livekit.yml copies docker-compose.yml here as docker-compose.yml.new +# and runs this script. Secrets live only on the box in conf/ (livekit.yaml, +# egress.yaml), never in the repo. An unchanged compose file is a no-op, so a +# push that touches nothing here does not restart the SFU or drop live calls. +set -euo pipefail + +ROOT=/home/anthony/www/sfu.pairux.com +cd "$ROOT" + +if [ -f docker-compose.yml.new ]; then + docker compose -f docker-compose.yml.new config -q + if cmp -s docker-compose.yml.new docker-compose.yml; then + rm -f docker-compose.yml.new + echo "compose file unchanged" + else + cp docker-compose.yml docker-compose.yml.prev + mv docker-compose.yml.new docker-compose.yml + echo "compose file updated (previous kept as docker-compose.yml.prev)" + fi +fi + +docker compose pull -q +docker compose up -d + +for _ in $(seq 1 30); do + if curl -fsS -o /dev/null http://127.0.0.1:7880/; then + docker compose ps + exit 0 + fi + sleep 2 +done + +echo "livekit did not answer on 127.0.0.1:7880" >&2 +if [ -f docker-compose.yml.prev ]; then + echo "rolling back to docker-compose.yml.prev" >&2 + mv docker-compose.yml.prev docker-compose.yml + docker compose up -d +fi +exit 1 diff --git a/apps/livekit/dev2/docker-compose.yml b/apps/livekit/dev2/docker-compose.yml new file mode 100644 index 0000000..643208f --- /dev/null +++ b/apps/livekit/dev2/docker-compose.yml @@ -0,0 +1,35 @@ +# PairUX LiveKit SFU on dev2 (moved off DO droplet sfu.pairux.com 2026-10-06). +# Host networking: LiveKit needs real UDP. Public ports: 7881/tcp, 7882/udp, +# 61100-61300/udp (rtc range), 3480/udp (LiveKit's own TURN). 7880 (signalling) +# is local only, fronted by nginx wss://sfu.pairux.com. Redis is a private bus on 127.0.0.1:6390. +name: pairux-sfu +services: + redis: + image: redis@sha256:858f009f9709ce576febc734aa78b8f6d624b82571f9ddb6bda4377c833b3499 + container_name: pairux-sfu-redis + restart: unless-stopped + network_mode: host + command: redis-server --bind 127.0.0.1 --port 6390 --save "" --appendonly no + + livekit: + image: livekit/livekit-server@sha256:6fd3b7088874c4d119160dd688798dfec852bc014786d392caad15f6f63912a3 + container_name: pairux-livekit + restart: unless-stopped + network_mode: host + depends_on: [redis] + volumes: + - ./conf/livekit.yaml:/etc/livekit.yaml:ro + command: --config /etc/livekit.yaml --bind 127.0.0.1 + + # Server-side restreamer / recorder (room composite -> RTMP / file). + egress: + image: livekit/egress@sha256:bf2b648b947349c3e9ff7aa8c718f00378d5c06af7624652a3653318e00333ce + container_name: pairux-egress + restart: unless-stopped + network_mode: host + depends_on: [redis, livekit] + environment: + - EGRESS_CONFIG_FILE=/etc/egress.yaml + volumes: + - ./conf/egress.yaml:/etc/egress.yaml:ro + cap_add: [SYS_ADMIN] diff --git a/apps/turn/dev2/deploy.sh b/apps/turn/dev2/deploy.sh new file mode 100755 index 0000000..c1ad2dd --- /dev/null +++ b/apps/turn/dev2/deploy.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Box-side deploy for the PairUX coturn on dev2 (turn.pairux.com). +# +# deploy-turn.yml copies docker-compose.yml here as docker-compose.yml.new and +# runs this script. turnserver.conf (with the TURN credential) lives only on the +# box in conf/; the cert is renewed by root's acme.sh into certs/. An unchanged +# compose file is a no-op, so live relays are not cut. +set -euo pipefail + +ROOT=/home/anthony/www/turn.pairux.com +cd "$ROOT" + +if [ -f docker-compose.yml.new ]; then + docker compose -f docker-compose.yml.new config -q + if cmp -s docker-compose.yml.new docker-compose.yml; then + rm -f docker-compose.yml.new + echo "compose file unchanged" + else + cp docker-compose.yml docker-compose.yml.prev + mv docker-compose.yml.new docker-compose.yml + echo "compose file updated (previous kept as docker-compose.yml.prev)" + fi +fi + +docker compose pull -q +docker compose up -d + +sleep 5 +if [ "$(docker inspect -f '{{.State.Running}} {{.RestartCount}}' pairux-coturn)" = "true 0" ]; then + docker compose ps + exit 0 +fi + +echo "coturn is not running cleanly" >&2 +docker logs --tail 20 pairux-coturn >&2 || true +if [ -f docker-compose.yml.prev ]; then + echo "rolling back to docker-compose.yml.prev" >&2 + mv docker-compose.yml.prev docker-compose.yml + docker compose up -d +fi +exit 1 diff --git a/apps/turn/dev2/docker-compose.yml b/apps/turn/dev2/docker-compose.yml new file mode 100644 index 0000000..92eaddb --- /dev/null +++ b/apps/turn/dev2/docker-compose.yml @@ -0,0 +1,15 @@ +# PairUX coturn on dev2 (moved off DO droplet turn.pairux.com 2026-10-06). +# Runs as uid 3478 (pairux-turn) so ufw before.rules can confine relays to +# dev2's own IP to LiveKit's ports. Public: 3478 tcp/udp, 5349 tcp/udp, relay 61400-61500. +name: pairux-turn +services: + coturn: + image: coturn/coturn@sha256:6a1d1a281b8f64ca1a343429bb0232fa70c5f0eae3c8424ba0859b696e880974 + container_name: pairux-coturn + network_mode: host + restart: unless-stopped + user: "3478:3478" + volumes: + - ./conf/turnserver.conf:/etc/coturn/turnserver.conf:ro + - ./certs:/certs:ro + command: ["-c", "/etc/coturn/turnserver.conf"]