AR2 Closeout tasks & pipeline updates

.github/workflows/ci.yml

@@ -13,8 +13,8 @@ jobs:
         python-version: '3.12'
     - name: Lint
       run: |
-        pip install ruff
-        ruff check app
+        pip install ruff==0.9.6
+        ruff check app --ignore E402
 
   test:
     runs-on: ubuntu-latest
@@ -53,4 +53,196 @@ jobs:
       run: |
         PYTHONPATH=. pytest app/tests/
 
+  migration:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Checkout Hub (for the schema drift guard)
+      uses: actions/checkout@v4
+      continue-on-error: true
+      with:
+        repository: agstack/ar2-hub
+        ref: main
+        path: _hub
+        token: ${{ secrets.AGSTACK_PAT }}
+
+    - name: Checkout Pancake (for the schema drift guard)
+      uses: actions/checkout@v4
+      continue-on-error: true
+      with:
+        repository: agstack/pancake
+        ref: main
+        path: _pancake
+        token: ${{ secrets.AGSTACK_PAT }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r migration/requirements.txt
+        pip install pytest ruff==0.9.6
+
+    - name: Lint
+      run: ruff check migration --ignore E402
+
+    - name: Run migration tests
+      run: |
+        if [ ! -d _hub ] || [ ! -d _pancake ]; then
+          echo "::warning::hub and/or pancake not checked out; "\
+               "the schema drift guard will SKIP rather than compare."
+        fi
+        pytest migration/tests -q -rs
+
+    - name: Import rehearsal against fixtures
+      run: |
+        python -m migration.run --source fixture --sample 3000 --create-all \
+          --ar2-url sqlite:///ci_ar2.db \
+          --hub-url sqlite:///ci_hub.db \
+          --pancake-url sqlite:///ci_pancake.db || true
+
+
+
+  e2e-test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test_db
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+    - name: Checkout AR2
+      uses: actions/checkout@v4
+      with:
+        path: ar2
+
+    - name: Checkout Pancake
+      uses: actions/checkout@v4
+      with:
+        repository: agstack/pancake
+        ref: main
+        path: pancake
+        token: ${{ secrets.AGSTACK_PAT }}
+
+    - name: Checkout Hub
+      uses: actions/checkout@v4
+      with:
+        repository: agstack/ar2-hub
+        ref: main
+        path: hub
+        token: ${{ secrets.AGSTACK_PAT }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Provision Databases
+      run: |
+        PGPASSWORD=password psql -h localhost -U postgres -c "CREATE DATABASE ar2_test;"
+        PGPASSWORD=password psql -h localhost -U postgres -c "CREATE DATABASE hub_test;"
 
+    - name: Install AR2 Dependencies
+      run: |
+        cd ar2
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Install Hub Dependencies
+      run: |
+        cd hub
+        pip install -r requirements.txt
+
+    - name: Install Pancake Dependencies
+      run: |
+        cd pancake/services
+        pip install -r requirements.txt
+
+    - name: Start Services
+      run: |
+        # Start Hub
+        cd hub
+        export DATABASE_URL="postgresql://postgres:password@localhost:5432/hub_test"
+        export REGISTRY_SERVERS='{"USA":"http://127.0.0.1:8001","India":"http://127.0.0.1:8001","Common":"http://127.0.0.1:8001"}'
+        export SERVER_BASE_URL="http://127.0.0.1:8000"
+        export WORLD_SHP_FILE="./shapefiles/99bfd9e7-bb42-4728-87b5-07f8c8ac631c2020328-1-1vef4ev.lu5nk.shp"
+        uvicorn hub_main:app --port 8000 > hub.log 2>&1 &
+        cd ..
+
+        # Start AR2 Node
+        cd ar2
+        export DATABASE_URL="postgresql://postgres:password@localhost:5432/ar2_test"
+        export HUB_URL="http://127.0.0.1:8000"
+        export JWKS_URL="http://127.0.0.1:8000/.well-known/jwks.json"
+        export AR_TRUSTED_AUTHORITY_PUBKEY="$PWD/app/tests/testkit/dev_keys/authority_issuer_public.pem"
+        export AR_TRUSTED_ISSUER_PUBKEY="$PWD/app/tests/testkit/dev_keys/dev_issuer_public.pem"
+        export AR2_INTERNAL_SHARED_SECRET="supersecret"
+        uvicorn app.main:app --port 8001 > node.log 2>&1 &
+        cd ..
+
+        # Start Pancake
+        cd pancake/services
+        export HUB_URL=""
+        export HUB_JWKS_URL="http://127.0.0.1:8000/.well-known/jwks.json"
+        export AR2_NODE_URL="http://127.0.0.1:8001"
+        export AR2_INTERNAL_SHARED_SECRET="supersecret"
+        export PANCAKE_ENV="test"
+        export PANCAKE_ISSUER_KEY=$(cat ../../ar2/app/tests/testkit/dev_keys/dev_issuer_private.pem)
+        export PANCAKE_TRUSTED_AUTHORITY_PUBKEY="../../ar2/app/tests/testkit/dev_keys/authority_issuer_public.pem"
+        uvicorn --factory pancake_services.grants.app:create_app --port 8100 > pancake.log 2>&1 &
+        cd ../..
+
+    - name: Poll Health and Dump Logs on Failure
+      run: |
+        TIMEOUT=30
+
+        check_health() {
+          local name=$1
+          local url=$2
+          local log_file=$3
+
+          local t=$TIMEOUT
+          echo "Waiting for $name..."
+          while ! curl -s $url > /dev/null; do
+            sleep 1
+            t=$((t-1))
+            if [ $t -eq 0 ]; then
+              echo "$name failed to start!"
+              echo "--- $name Log ---"
+              cat $log_file
+              exit 1
+            fi
+          done
+          echo "$name is up!"
+        }
+
+        check_health "Hub" "http://127.0.0.1:8000/docs" "hub/hub.log"
+        check_health "Node" "http://127.0.0.1:8001/docs" "ar2/node.log"
+        check_health "Pancake" "http://127.0.0.1:8100/docs" "pancake/services/pancake.log"
+
+    - name: Run E2E Trace-Forward Test
+      run: |
+        cd ar2
+        if ! bash scripts/e2e_traceforward.sh; then
+          echo "=== HUB LOGS ==="
+          cat ../hub/hub.log
+          echo "=== NODE LOGS ==="
+          cat node.log
+          echo "=== PANCAKE LOGS ==="
+          cat ../pancake/services/pancake.log
+          exit 1
+        fi

.github/workflows/stomata.yml

@@ -0,0 +1,126 @@
+# Copy to .github/workflows/stomata.yml in the repository being governed.
+#
+# Pushing a file under .github/workflows/ requires a token with `workflow` scope.
+# If yours lacks it the push is rejected with a message that does not say so
+# clearly; someone whose token has the scope needs to add this file.
+#
+# The important line is the last one. CI does NOT inherit the local skip waivers:
+# a waiver that names its environment ("waived locally, where the hub is not
+# checked out") is only honest if the environment that has no excuse removes it.
+
+name: stomata
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: ar2_test
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # The harness is a pinned submodule. Without this it is absent and the
+          # job fails on a missing file rather than on anything meaningful.
+          submodules: true
+          token: ${{ secrets.AGSTACK_PAT }}
+
+      # Sibling repositories that cross-layer tests compare against. Without
+      # these, those tests SKIP -- and a skipped cross-layer test reports the
+      # same green as one that ran. That has happened three times.
+      - name: Check out ar2-hub
+        uses: actions/checkout@v4
+        with:
+          repository: agstack/ar2-hub
+          path: .sibling/ar2-hub
+          token: ${{ secrets.AGSTACK_PAT }}
+          ref: main
+
+      - name: Check out pancake
+        uses: actions/checkout@v4
+        with:
+          repository: agstack/pancake
+          path: .sibling/pancake
+          token: ${{ secrets.AGSTACK_PAT }}
+          ref: main
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f migration/requirements.txt ]; then pip install -r migration/requirements.txt; fi
+          pip install pytest ruff
+
+      - name: Apply schema upgrades
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/ar2_test
+        run: |
+          # create_all adds missing tables but never missing columns, so a
+          # database that already exists needs these explicitly. Harmless here,
+          # where the database is empty, and it keeps the script exercised: a
+          # migration nobody runs is a migration nobody knows is broken.
+          for f in scripts/schema_upgrade_*.sql; do
+            [ -e "$f" ] || continue
+            psql "$DATABASE_URL" -f "$f"
+          done
+
+      - name: Run the gate, including mutations
+        env:
+          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/ar2_test
+          HUB_CHECKOUT: ${{ github.workspace }}/.sibling/ar2-hub
+          PANCAKE_CHECKOUT: ${{ github.workspace }}/.sibling/pancake
+          MIGRATION_HUB_MODELS: ${{ github.workspace }}/.sibling/ar2-hub/user_models.py
+          MIGRATION_PANCAKE_MODELS: ${{ github.workspace }}/.sibling/pancake/services/pancake_services/grants/models.py
+          STOMATA_PYTHON: python
+          NO_COLOR: '1'
+        run: harness/bin/stomata run --full
+
+      - name: Publish the run record
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: stomata-state
+          path: .stomata/state.json
+
+      - name: Refuse locally-waived skips here
+        if: always()
+        env:
+          NO_COLOR: '1'
+        run: |
+          # CI has the sibling checkouts, so nothing may skip. This is the line
+          # that makes an environment-scoped waiver honest rather than permanent.
+          python - <<'PY'
+          import json, pathlib, sys
+          state = json.loads(pathlib.Path(".stomata/state.json").read_text())
+          skipped = [
+              (s["name"], reason)
+              for s in state["suites"]
+              for reason in s.get("skip_reasons", [])
+          ]
+          if skipped:
+              print("Skips are not permitted in CI, where every dependency is present:")
+              for suite, reason in skipped:
+                  print(f"  {suite}: {reason['location']}: {reason['reason']}")
+              sys.exit(1)
+          print("no skips")
+          PY

.gitignore

@@ -60,4 +60,9 @@ bulk_test_20800.geojson
 generate_synthetic_geojson.py
 demo_guide_local.md
 test_curls_local.txt
-manual_e2e_testing_guide.md
+manual_e2e_testing_guide.md
+# The last stomata run. Local, and regenerated on every run, which is why a
+# review packet carries a COPY of it under packets/ rather than pointing here --
+# a pointer to this path resolves only on the machine that produced it.
+# .stomata/baseline.json IS tracked: it is the shared ratchet.
+.stomata/state.json

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "harness"]
+	path = harness
+	url = https://github.com/agstack/stomata.git