diff --git a/.env.example b/.env.example index f147629..8e4f793 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Copy to .env (git-ignored). Never commit real values. -DBRICKS_HOST=adb-xxxx.azuredatabricks.net -DBRICKS_HTTP_PATH=/sql/1.0/warehouses/xxxxxxxx -DBRICKS_TOKEN=dapi...your-token-here +DATABRICKS_HOST=adb-xxxx.azuredatabricks.net +DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/xxxxxxxx +DATABRICKS_TOKEN=dapi...your-token-here DBT_SCHEMA=dev_yourname diff --git a/.github/workflows/pr-body-check.yml b/.github/workflows/pr-body-check.yml new file mode 100644 index 0000000..f6109aa --- /dev/null +++ b/.github/workflows/pr-body-check.yml @@ -0,0 +1,57 @@ +name: PR body check + +# Fails the check when a pull request description is missing the sections from +# .github/PULL_REQUEST_TEMPLATE.md. GitHub only auto-fills that template in the +# web "compose" form and in `gh pr create` with no --body; a PR opened through +# the REST API or `gh pr create --body "..."` (the path most AI tools take) +# silently skips it. This check is the only thing that actually enforces it. +# +# Uses pull_request_target (in addition to pull_request) so fork PRs always run +# the workflow from main. The job only reads github.event.pull_request.body — +# it does not check out PR head code. +# +# Recovery is automatic: editing the PR description fires the `edited` event and +# re-runs this check with the new body. No new commit or manual re-run needed. + +on: + pull_request: + types: [opened, edited, reopened, synchronize] + branches: [main] + pull_request_target: + types: [opened, edited, reopened, synchronize] + branches: [main] + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Check required sections are present + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + required=( + "## Summary" + "## How to review" + "## Secrets hygiene checklist" + ) + missing=() + for section in "${required[@]}"; do + if ! printf '%s' "$PR_BODY" | grep -qiF "$section"; then + missing+=("$section") + fi + done + if [ ${#missing[@]} -ne 0 ]; then + echo "::error::Your PR description is missing required sections. Start from the template (.github/PULL_REQUEST_TEMPLATE.md) and keep these headings:" + for m in "${missing[@]}"; do echo " - $m"; done + echo "" + echo "If you (or an AI tool) opened this PR without the template, click 'Edit' on the" + echo "PR description, paste the template, and fill it in. Editing the description" + echo "re-runs this check automatically. A complete PR is easy to review and" + echo "reproducible: that is part of the assignment." + exit 1 + fi + echo "All required PR sections present." diff --git a/README.md b/README.md index c6ed11b..cefafb5 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ data-assignment-week-13/ ├── task-3/ # Git-backed Job scheduling │ ├── SCHEDULING.md # Jobs vs Airflow write-up + Job Run URL │ └── screenshots/ # Job config, green run, paused trigger +├── task-4/ # optional bonuses only (create if needed) ├── .env.example ├── AI_ASSIST.md # LLM interaction log └── README.md diff --git a/task-1/README.md b/task-1/README.md index 9a1b2af..973f050 100644 --- a/task-1/README.md +++ b/task-1/README.md @@ -11,4 +11,4 @@ Add your Databricks notebook here. - Results displayed with `show()` on small aggregated DataFrames (not `collect()` on the raw table). - Two or three sentences on when you would choose PySpark versus dbt SQL. -See the [Week 13 assignment](https://github.com/HackYourFuture/datatrack/blob/main/Data%20Track/Week%2013/week_13__6_assignment.md) for full requirements. +See the [Week 13 assignment](https://github.com/HackYourFuture/datatrack/blob/main/Data%20Track/Week%2013/week_13__7_assignment.md) for full requirements. diff --git a/task-2/README.md b/task-2/README.md index e5951e5..5706176 100644 --- a/task-2/README.md +++ b/task-2/README.md @@ -7,6 +7,6 @@ Copy your **Week 10 dbt project** into this folder (or start from the `week-13-d 1. Install `dbt-databricks` locally (`pip install dbt-databricks` or `uv tool install dbt-databricks`). 2. Copy `profiles.yml.example` to `profiles.yml` (git-ignored) and export the env vars from `.env.example`. 3. Configure `fct_trips` as `materialized='incremental'` with `incremental_strategy='merge'` and a real `unique_key`. -4. Run `dbt build --select fct_trips` twice and document both timings plus your explanation in `WRITEUP.md`. +4. Run `dbt build --select fct_trips --full-refresh` for the baseline, then `dbt build --select fct_trips` for the incremental rerun. Document both timings plus your explanation in `WRITEUP.md`. **Do not commit:** `profiles.yml`, `.env`, or any Databricks token. diff --git a/task-2/profiles.yml.example b/task-2/profiles.yml.example index 557e82e..ab26809 100644 --- a/task-2/profiles.yml.example +++ b/task-2/profiles.yml.example @@ -6,7 +6,7 @@ nyc_taxi: type: databricks catalog: hyf schema: "{{ env_var('DBT_SCHEMA') }}" - host: "{{ env_var('DBRICKS_HOST') }}" - http_path: "{{ env_var('DBRICKS_HTTP_PATH') }}" - token: "{{ env_var('DBRICKS_TOKEN') }}" + host: "{{ env_var('DATABRICKS_HOST') }}" + http_path: "{{ env_var('DATABRICKS_HTTP_PATH') }}" + token: "{{ env_var('DATABRICKS_TOKEN') }}" threads: 4