datacommonsorg · vish-cs · Sep 11, 2026
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,5 @@ import-automation/executor/config_override.json
 # Ignore generated troubleshooting post-mortem documents
 agents/troubleshooting/
 
-
+# Ignore compiled Airflow import catalog (generated at build time)
+import-automation/workflow/imports_catalog.json
diff --git a/import-automation/cloudbuild/cloudbuild.workflow.yaml b/import-automation/cloudbuild/cloudbuild.workflow.yaml
@@ -0,0 +1,113 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Cloud Build configuration to compile manifest.json files, sync Airflow DAGs to Cloud Composer,
+# and run an End-to-End (E2E) DAG verification test.
+#
+# Usage:
+#   gcloud builds submit . \
+#     --config=import-automation/cloudbuild/cloudbuild.workflow.yaml \
+#     --project=datcom-ci \
+#     --substitutions=_PROJECT_ID=datcom-import-automation-prod,_COMPOSER_ENV_NAME=import-automation-airflow,_LOCATION=us-central1
+
+substitutions:
+  _PROJECT_ID: 'datcom-import-automation-prod'
+  _LOCATION: 'us-central1'
+  _COMPOSER_ENV_NAME: 'import-automation-airflow'
+  _DAG_SUBDIR: 'datacommons_airflow'
+  _DAG_BUCKET: ''
+  _ALLOWLIST: ''
+  _RUN_E2E_TEST: 'true'
+  _E2E_DAG_ID: 'USFed_ConstantMaturityRates_Test'
+  _E2E_SKIP_IMPORT_JOB: 'false'
+
+steps:
+  # 1. Compile manifest.json files into imports_catalog.json
+  - id: 'build-catalog'
+    name: 'python:3.11-slim'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        ARGS="--data-dir=. --output=import-automation/workflow/imports_catalog.json"
+        if [ -n "${_ALLOWLIST}" ]; then
+          ARGS="$${ARGS} --allowlist ${_ALLOWLIST}"
+        fi
+        python import-automation/workflow/build_manifest_catalog.py $${ARGS}
+
+  # 2. Validate DAG, factory, and E2E test syntax
+  - id: 'validate-dags'
+    name: 'python:3.11-slim'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        python -m py_compile import-automation/workflow/import_automation_workflow.py
+        python -m py_compile import-automation/workflow/golden_verification.py
+        python -m py_compile import-automation/workflow/import_dags_factory.py
+        python -m py_compile import-automation/workflow/e2e_dag_test.py
+        echo "DAG syntax validation passed."
+
+  # 3. Deploy/Sync DAGs, factory, and catalog to Cloud Composer GCS bucket
+  - id: 'deploy-dags'
+    name: 'gcr.io/cloud-builders/gcloud'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [ -n "${_DAG_BUCKET}" ]; then
+          TARGET_GCS="gs://${_DAG_BUCKET}/dags/${_DAG_SUBDIR}"
+        else
+          DAG_PREFIX=$(gcloud composer environments describe ${_COMPOSER_ENV_NAME} \
+            --location=${_LOCATION} \
+            --project=${_PROJECT_ID} \
+            --format="value(config.dagGcsPrefix)")
+          TARGET_GCS="$${DAG_PREFIX}/${_DAG_SUBDIR}"
+        fi
+
+        gcloud composer environments describe ${_COMPOSER_ENV_NAME} \
+          --location=${_LOCATION} \
+          --project=${_PROJECT_ID} \
+          --format="value(config.airflowUri)" > /workspace/composer_webserver_url.txt || true
+
+        echo "Deploying Airflow DAGs to: $${TARGET_GCS}"
+        gcloud storage cp import-automation/workflow/import_automation_workflow.py "$${TARGET_GCS}/import_automation_workflow.py"
+        gcloud storage cp import-automation/workflow/golden_verification.py "$${TARGET_GCS}/golden_verification.py"
+        gcloud storage cp import-automation/workflow/import_dags_factory.py "$${TARGET_GCS}/import_dags_factory.py"
+        gcloud storage cp import-automation/workflow/imports_catalog.json "$${TARGET_GCS}/imports_catalog.json"
+        echo "Successfully deployed Airflow DAGs."
+
+  # 4. Run End-to-End DAG Verification Test in Cloud Composer
+  - id: 'e2e-test-dag'
+    name: 'python:3.11-slim'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [ "${_RUN_E2E_TEST}" != "true" ]; then
+          echo "Skipping E2E DAG test (_RUN_E2E_TEST=${_RUN_E2E_TEST})."
+          exit 0
+        fi
+        pip install --quiet google-auth requests
+        E2E_ARGS="--project-id=${_PROJECT_ID} --location=${_LOCATION} --composer-env=${_COMPOSER_ENV_NAME} --dag-id=${_E2E_DAG_ID}"
+        if [ -s /workspace/composer_webserver_url.txt ]; then
+          E2E_ARGS="$${E2E_ARGS} --webserver-url=$(cat /workspace/composer_webserver_url.txt)"
+        fi
+        if [ "${_E2E_SKIP_IMPORT_JOB}" = "true" ]; then
+          E2E_ARGS="$${E2E_ARGS} --skip-import-job"
+        fi
+        python3 import-automation/workflow/e2e_dag_test.py $${E2E_ARGS}
+
+options:
+  logging: CLOUD_LOGGING_ONLY
diff --git a/import-automation/cloudbuild/ingestion-golden-verification.cloudbuild.yaml b/import-automation/cloudbuild/ingestion-golden-verification.cloudbuild.yaml
@@ -0,0 +1,127 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Cloud Build configuration for staging schema verification gate.
+# Clears Staging Redis cache, runs golden tests in update mode, generates diffs,
+# creates a GitHub PR if changes are detected, and outputs summary for human approval.
+
+substitutions:
+  _GITHUB_ORG: 'datacommonsorg'
+  _GITHUB_REPO: 'website'
+  _GITHUB_BRANCH: 'master'
+  _GITHUB_AUTHOR: 'datacommons-robot-author'
+  _PR_REVIEWER: 'datacommonsorg/website-reviewers'
+  _DIFF_BUCKET: 'datcom-ci-test'
+
+availableSecrets:
+  secretManager:
+  - versionName: projects/879489846695/secrets/GH_PAT/versions/latest
+    env: 'ghsecret'
+
+steps:
+  # 1. Initialize mixer submodule so helm configs for mixer are available
+  - id: 'init-mixer-submodule'
+    name: 'gcr.io/cloud-builders/git'
+    args: ['submodule', 'update', '--init', '--depth', '1', 'mixer']
+
+  # 2. Clear Redis cache on Staging Mixer
+  - id: 'clear-staging-mixer-cache'
+    name: 'gcr.io/datcom-ci/datacommons-script-runner:latest'
+    entrypoint: 'bash'
+    args:
+      - 'tools/clearcache/run.sh'
+      - 'mixer'
+      - 'staging'
+
+  # 3. Run Explore & NL golden tests against staging backend in update mode (-g)
+  - id: 'run-golden-tests'
+    name: 'python:3.11.3'
+    entrypoint: '/bin/sh'
+    args:
+      - '-c'
+      - |
+        pip install uv
+        ./run_test.sh --explore -g || true
+        ./run_test.sh --nl -g || true
+
+  # 4. Detect diffs, create PR if needed, and write diff_summary.json to GCS
+  - id: 'create-pr-and-summary'
+    name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim'
+    entrypoint: 'bash'
+    secretEnv: ['ghsecret']
+    args:
+      - '-c'
+      - |
+        apt-get update && apt-get install -y git gh jq
+
+        DIFF_TARGETS="server/integration_tests/test_data/ server/config/nl_page/"
+        BUILD_ID_SAFE=$(echo "${BUILD_ID}" | tr -cd '[:alnum:]-')
+
+        # Check for diffs in golden directories
+        if git diff --exit-code --quiet -- $${DIFF_TARGETS}; then
+          echo "NO_DIFF_DETECTED"
+          cat <<EOF > /workspace/diff_summary.json
+        {
+          "has_diff": false,
+          "pr_url": "",
+          "build_id": "${BUILD_ID}",
+          "message": "No golden diffs detected"
+        }
+        EOF
+        else
+          echo "DIFF_DETECTED: Generating branch and opening PR..."
+          BRANCH_NAME="schema-golden-diff-$${BUILD_ID_SAFE}"
+          PR_URL=""
+
+          if [ -n "$$ghsecret" ]; then
+            git config user.name "${_GITHUB_AUTHOR}"
+            git config user.email "${_GITHUB_AUTHOR}@users.noreply.github.com"
+            git checkout -b "$${BRANCH_NAME}"
+            git add $${DIFF_TARGETS}
+            git commit -m "test(goldens): automated schema staging diff update (build $${BUILD_ID_SAFE})"
+
+            export GH_TOKEN="$$ghsecret"
+            git push "https://${_GITHUB_AUTHOR}:$${ghsecret}@github.com/${_GITHUB_ORG}/${_GITHUB_REPO}.git" "HEAD:$${BRANCH_NAME}" || true
+
+            PR_URL=$(gh pr create \
+              --repo "${_GITHUB_ORG}/${_GITHUB_REPO}" \
+              --title "Staging Schema Golden Diff: Build $${BUILD_ID_SAFE}" \
+              --body "Automated golden diff from staging schema import. Please review and merge. [Build Log](https://console.cloud.google.com/cloud-build/builds/${BUILD_ID}?project=${PROJECT_ID})" \
+              --base "${_GITHUB_BRANCH}" \
+              --reviewer "${_PR_REVIEWER}" \
+              --head "$${BRANCH_NAME}" \
+              --label "golden-diff-gate" 2>/dev/null || echo "")
+            echo "AUTOMATED_PR_URL:$${PR_URL}"
+          else
+            echo "GH_PAT secret not provided; skipped git push/PR creation."
+          fi
+
+          cat <<EOF > /workspace/diff_summary.json
+        {
+          "has_diff": true,
+          "pr_url": "$${PR_URL}",
+          "build_id": "${BUILD_ID}",
+          "branch_name": "$${BRANCH_NAME}",
+          "message": "Golden diff detected"
+        }
+        EOF
+        fi
+
+        echo "Uploading diff_summary.json to gs://${_DIFF_BUCKET}/golden_diffs/${BUILD_ID}/diff_summary.json..."
+        gcloud storage cp /workspace/diff_summary.json "gs://${_DIFF_BUCKET}/golden_diffs/${BUILD_ID}/diff_summary.json" || true
+        cat /workspace/diff_summary.json
+
+options:
+  machineType: 'E2_HIGHCPU_32'
+timeout: '2400s'
diff --git a/import-automation/workflow/README.md b/import-automation/workflow/README.md
@@ -0,0 +1,32 @@
+# Import Automation Workflow (Airflow / Cloud Composer)
+
+This directory contains the Apache Airflow DAG definitions and dynamic factory for automating Data Commons data imports.
+
+## Architecture
+
+1. **`build_manifest_catalog.py`**:
+   Scans all `manifest.json` files in the repository (excluding `scripts/entities`) and compiles them into `imports_catalog.json`.
+   To run manually:
+   ```bash
+   python3 import-automation/workflow/build_manifest_catalog.py
+   ```
+
+2. **`imports_catalog.json`**:
+   The compiled catalog of all import configurations, cron schedules, curator emails, and resource allocations.
+
+3. **`import_dags_factory.py`**:
+   Airflow dynamic DAG factory that reads `imports_catalog.json` and registers an independent DAG for each import specification.
+   All DAGs are created paused by default (`is_paused_upon_creation=True`, `catchup=False`).
+
+4. **`import_automation_workflow.py`**:
+   Core Airflow DAG definition that defines `build_dag` and executes the 4-stage pipeline:
+   - **Cloud Batch Job**: Runs `dc-import-executor` container.
+   - **Staging Ingestion**: Updates staging version via `import-helper-service-staging`, triggers ingestion via Spanner Cloud Workflow, and polls until completion.
+   - **Production Ingestion**: Triggers fire-and-forget production Spanner ingestion upon staging success.
+   - **Workflow Summary**: Aggregates execution status across stages and reports errors.
+
+## Continuous Deployment
+
+Cloud Build automatically updates the catalog and syncs DAGs to Cloud Composer:
+- Config: `import-automation/cloudbuild/cloudbuild.workflow.yaml`
+- Target: `gs://<dag_bucket>/dags/datacommons_airflow/`