diff --git a/.hyf/grader_lib.sh b/.hyf/grader_lib.sh index d383a4a..3142cfe 100644 --- a/.hyf/grader_lib.sh +++ b/.hyf/grader_lib.sh @@ -3,15 +3,23 @@ # Source this at the top of test.sh: # source "$(dirname "$0")/grader_lib.sh" # -# Provides: pass(), fail(), warn(), print_results(), write_score(), +# Provides: pass(), fail(), warn(), blocker(), print_results(), write_score(), # and a set of common static-analysis checks derived from recurring # PR review patterns across cohort c55. +# +# blocker(): use for leaked-secret findings (a committed profiles.yml/.env, +# a hardcoded password/connection string). It behaves like fail() for the +# printed report, but also flips a flag that forces write_score() to report +# pass=false regardless of the earned point total -- a leaked secret must +# be fixed before the PR can pass, it cannot be "pointed around." _grader_details=() +_grader_blocker=false pass() { _grader_details+=("✓ PASS $1"); } fail() { _grader_details+=("✗ FAIL $1"); } warn() { _grader_details+=("⚠ WARN $1"); } +blocker() { _grader_details+=("🚫 BLOCKER $1"); _grader_blocker=true; } print_results() { local header="${1:-Autograder Results}" @@ -28,6 +36,10 @@ write_score() { local outfile="${3:-$(dirname "${BASH_SOURCE[0]}")/score.json}" local pass_flag="false" [[ "$score" -ge "$passing" ]] && pass_flag="true" + if [[ "$_grader_blocker" == true ]]; then + pass_flag="false" + echo "🚫 A blocker was found (leaked secret) -- forcing pass=false regardless of score." >&2 + fi cat > "$outfile" << JSON { "score": $score, diff --git a/.hyf/test.sh b/.hyf/test.sh index 5f27fe7..c4142d5 100644 --- a/.hyf/test.sh +++ b/.hyf/test.sh @@ -63,25 +63,25 @@ for f in "${required_files[@]}"; do pass "found $f" else fail "missing required file: $f" - ((missing += 1)) + missing=$((missing + 1)) fi done if ls "$REPO_ROOT/tests/"*.sql &>/dev/null 2>&1; then pass "found singular test in tests/" else fail "no .sql file found in tests/ -- add at least one singular test" - ((missing += 1)) + missing=$((missing + 1)) fi if [[ -s "$REPO_ROOT/docs/lineage.png" ]]; then pass "found docs/lineage.png" else fail "docs/lineage.png missing -- run dbt docs generate + serve and screenshot the lineage graph" - ((missing += 1)) + missing=$((missing + 1)) fi if [[ "$missing" -eq 0 ]]; then l1=10 fi -((score += l1)) +score=$((score + l1)) pass "Level 1: required files ($l1/10 pts)" # ── Level 2 (15 pts): secrets hygiene ─────────────────────────────────────── @@ -90,23 +90,23 @@ gi="$REPO_ROOT/.gitignore" ex="$REPO_ROOT/profiles.yml.example" if [[ -f "$gi" ]] && grep -qE "^profiles\.yml$" "$gi"; then - ((l2 += 5)); pass ".gitignore: profiles.yml excluded" + l2=$((l2 + 5)); pass ".gitignore: profiles.yml excluded" else fail ".gitignore must contain 'profiles.yml' on its own line" fi if [[ -f "$ex" ]] && grep -qE "env_var\(" "$ex"; then - ((l2 += 5)); pass "profiles.yml.example: uses env_var() -- no hardcoded password" + l2=$((l2 + 5)); pass "profiles.yml.example: uses env_var() -- no hardcoded password" else fail "profiles.yml.example must use env_var('PG_PASSWORD') instead of a hardcoded password" fi if [[ -f "$REPO_ROOT/profiles.yml" ]]; then - fail "profiles.yml is committed -- BLOCKER: run: git rm --cached profiles.yml" + blocker "profiles.yml is committed -- run: git rm --cached profiles.yml, then rotate the Postgres password since it was pushed" else - ((l2 += 5)); pass "profiles.yml not committed (correctly git-ignored)" + l2=$((l2 + 5)); pass "profiles.yml not committed (correctly git-ignored)" fi -((score += l2)) +score=$((score + l2)) pass "Level 2: secrets hygiene ($l2/15 pts)" # ── Level 3 (20 pts): staging models ──────────────────────────────────────── @@ -115,35 +115,35 @@ st="$REPO_ROOT/models/staging/stg_trips.sql" sz="$REPO_ROOT/models/staging/stg_zones.sql" if file_is_filled "$st" && sqlgrep "\{\{[[:space:]]*source\(" "$st"; then - ((l3 += 4)); pass "stg_trips.sql: uses {{ source() }} reference" + l3=$((l3 + 4)); pass "stg_trips.sql: uses {{ source() }} reference" else fail "stg_trips.sql: must use {{ source('nyc_taxi', 'raw_trips') }}" fi if file_is_filled "$st" && sqlgrep "pickup_location_id" "$st" && sqlgrep "IS NOT NULL|NOT NULL" "$st"; then - ((l3 += 4)); pass "stg_trips.sql: filters NULL pickup_location_id" + l3=$((l3 + 4)); pass "stg_trips.sql: filters NULL pickup_location_id" else fail "stg_trips.sql: must filter WHERE pickup_location_id IS NOT NULL" fi if file_is_filled "$st" && sqlgrep "fare_amount" "$st" && sqlgrep ">=.*0|> -1" "$st"; then - ((l3 += 4)); pass "stg_trips.sql: filters negative fares (fare_amount >= 0)" + l3=$((l3 + 4)); pass "stg_trips.sql: filters negative fares (fare_amount >= 0)" else fail "stg_trips.sql: must filter WHERE fare_amount >= 0" fi if file_is_filled "$st" && sqlgrep "tip_pct|safe_divide" "$st"; then - ((l3 += 4)); pass "stg_trips.sql: tip_pct column present" + l3=$((l3 + 4)); pass "stg_trips.sql: tip_pct column present" else fail "stg_trips.sql: tip_pct column missing -- add {{ safe_divide('tip_amount', 'fare_amount') }} AS tip_pct" fi if file_is_filled "$sz" && sqlgrep "\{\{[[:space:]]*source\(" "$sz"; then - ((l3 += 4)); pass "stg_zones.sql: uses {{ source() }} and is filled" + l3=$((l3 + 4)); pass "stg_zones.sql: uses {{ source() }} and is filled" else fail "stg_zones.sql: must use {{ source('nyc_taxi', 'raw_zones') }}" fi -((score += l3)) +score=$((score + l3)) pass "Level 3: staging models ($l3/20 pts)" # ── Level 4 (20 pts): mart model ───────────────────────────────────────────── @@ -151,22 +151,22 @@ l4=0 mart="$REPO_ROOT/models/marts/fct_daily_borough_stats.sql" if file_is_filled "$mart"; then - ((l4 += 2)); pass "fct_daily_borough_stats.sql: file filled" + l4=$((l4 + 2)); pass "fct_daily_borough_stats.sql: file filled" if sqlgrep "\{\{[[:space:]]*ref\('stg_trips'\)" "$mart" && sqlgrep "\{\{[[:space:]]*ref\('stg_zones'\)" "$mart"; then - ((l4 += 4)); pass "mart: ref() to both stg_trips and stg_zones" + l4=$((l4 + 4)); pass "mart: ref() to both stg_trips and stg_zones" else fail "mart: must reference both {{ ref('stg_trips') }} and {{ ref('stg_zones') }}" fi if sqlgrep "(INNER|LEFT)?[[:space:]]*JOIN" "$mart"; then - ((l4 += 4)); pass "mart: JOIN to zones present" + l4=$((l4 + 4)); pass "mart: JOIN to zones present" else fail "mart: no JOIN -- must join stg_trips to stg_zones on pickup_location_id = location_id" fi if sqlgrep "GROUP[[:space:]]+BY" "$mart"; then - ((l4 += 4)); pass "mart: GROUP BY present" + l4=$((l4 + 4)); pass "mart: GROUP BY present" else fail "mart: no GROUP BY -- must aggregate to (pickup_borough, pickup_date) grain" fi @@ -174,20 +174,20 @@ if file_is_filled "$mart"; then cols_ok=0 for col in pickup_borough pickup_date trip_count total_fare avg_tip_pct avg_trip_distance; do if sqlgrep "$col" "$mart"; then - ((cols_ok += 1)) + cols_ok=$((cols_ok + 1)) else fail "mart: required output column '$col' not found" fi done if [[ "$cols_ok" -eq 6 ]]; then - ((l4 += 6)); pass "mart: all 6 required output columns present" + l4=$((l4 + 6)); pass "mart: all 6 required output columns present" elif [[ "$cols_ok" -ge 4 ]]; then - ((l4 += 3)); warn "mart: $cols_ok/6 required columns present" + l4=$((l4 + 3)); warn "mart: $cols_ok/6 required columns present" fi else fail "fct_daily_borough_stats.sql: still contains TODO stubs" fi -((score += l4)) +score=$((score + l4)) pass "Level 4: mart model ($l4/20 pts)" # ── Level 5 (15 pts): tests ───────────────────────────────────────────────── @@ -196,28 +196,28 @@ mart_yml="$REPO_ROOT/models/marts/_fct_daily_borough_stats.yml" singular_test=$(ls "$REPO_ROOT/tests/"*.sql 2>/dev/null | head -1 || true) if [[ -f "$mart_yml" ]] && grep -qiE "unique_combination_of_columns" "$mart_yml"; then - ((l5 += 5)); pass "mart YAML: dbt_utils.unique_combination_of_columns test present" + l5=$((l5 + 5)); pass "mart YAML: dbt_utils.unique_combination_of_columns test present" else fail "mart YAML: missing dbt_utils.unique_combination_of_columns on (pickup_borough, pickup_date)" fi stg_yml="$REPO_ROOT/models/staging/_stg_trips.yml" if [[ -f "$stg_yml" ]] && grep -qiE "not_null" "$stg_yml"; then - ((l5 += 5)); pass "staging YAML: not_null tests present" + l5=$((l5 + 5)); pass "staging YAML: not_null tests present" else fail "staging YAML (_stg_trips.yml): not_null tests missing on key columns" fi if [[ -n "$singular_test" ]] && file_is_filled "$singular_test"; then if sqlgrep "\{\{[[:space:]]*ref\('fct_daily_borough_stats'\)" "$singular_test"; then - ((l5 += 5)); pass "singular test: filled and references fct_daily_borough_stats" + l5=$((l5 + 5)); pass "singular test: filled and references fct_daily_borough_stats" else - ((l5 += 3)); pass "singular test: filled (does not reference fct_daily_borough_stats -- check)" + l5=$((l5 + 3)); pass "singular test: filled (does not reference fct_daily_borough_stats -- check)" fi else fail "singular test: empty or still a TODO stub" fi -((score += l5)) +score=$((score + l5)) pass "Level 5: tests ($l5/15 pts)" # ── Level 6 (10 pts): documentation ───────────────────────────────────────── @@ -226,7 +226,7 @@ mart_yml="$REPO_ROOT/models/marts/_fct_daily_borough_stats.yml" if [[ -f "$mart_yml" ]] && grep -qiE "grain|one row per" "$mart_yml"; then if ! grep -B5 -A5 "grain\|one row per" "$mart_yml" | grep -qiE "TODO"; then - ((l6 += 5)); pass "mart YAML: grain stated in model description" + l6=$((l6 + 5)); pass "mart YAML: grain stated in model description" else fail "mart YAML: grain mentioned but description still has TODO -- fill it in" fi @@ -237,14 +237,14 @@ fi if [[ -f "$mart_yml" ]]; then filled_descs=$(grep "description:" "$mart_yml" | grep -v "TODO" | grep -cvE '^\s*description:\s*("")?\s*$' || true) if [[ "$filled_descs" -ge 4 ]]; then - ((l6 += 5)); pass "mart YAML: $filled_descs column descriptions filled" + l6=$((l6 + 5)); pass "mart YAML: $filled_descs column descriptions filled" elif [[ "$filled_descs" -ge 2 ]]; then - ((l6 += 3)); warn "mart YAML: only $filled_descs column descriptions filled (target: all 6)" + l6=$((l6 + 3)); warn "mart YAML: only $filled_descs column descriptions filled (target: all 6)" else fail "mart YAML: column descriptions are empty -- explain meaning and units for each column" fi fi -((score += l6)) +score=$((score + l6)) pass "Level 6: documentation ($l6/10 pts)" # ── Level 7 (10 pts): business answers + AI log ───────────────────────────── @@ -255,11 +255,11 @@ ai="$REPO_ROOT/AI_ASSIST.md" if file_is_filled "$ans"; then sql_blocks=$(grep -c "^\`\`\`sql" "$ans" 2>/dev/null || echo 0) if [[ "$sql_blocks" -ge 4 ]]; then - ((l7 += 5)); pass "reports/answers.md: 4 SQL blocks present and filled" + l7=$((l7 + 5)); pass "reports/answers.md: 4 SQL blocks present and filled" elif [[ "$sql_blocks" -ge 2 ]]; then - ((l7 += 3)); warn "reports/answers.md: $sql_blocks SQL block(s) present (need 4)" + l7=$((l7 + 3)); warn "reports/answers.md: $sql_blocks SQL block(s) present (need 4)" else - ((l7 += 2)); warn "reports/answers.md: filled but SQL blocks missing" + l7=$((l7 + 2)); warn "reports/answers.md: filled but SQL blocks missing" fi else fail "reports/answers.md: empty or still contains TODO stubs" @@ -268,14 +268,14 @@ fi if file_is_filled "$ai"; then chars=$(wc -c < "$ai" | tr -d ' ') if [[ "$chars" -ge 800 ]]; then - ((l7 += 5)); pass "AI_ASSIST.md: filled (${chars} chars)" + l7=$((l7 + 5)); pass "AI_ASSIST.md: filled (${chars} chars)" else - ((l7 += 2)); warn "AI_ASSIST.md: present but brief (${chars} chars -- target 800+)" + l7=$((l7 + 2)); warn "AI_ASSIST.md: present but brief (${chars} chars -- target 800+)" fi else fail "AI_ASSIST.md: empty or still contains TODO stubs" fi -((score += l7)) +score=$((score + l7)) pass "Level 7: business answers + AI log ($l7/10 pts)" # ── Final ───────────────────────────────────────────────────────────────────── diff --git a/.user.yml b/.user.yml new file mode 100644 index 0000000..b59c4ca --- /dev/null +++ b/.user.yml @@ -0,0 +1 @@ +id: 65fed7dc-cfb1-40ec-af82-fb56180cafce diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 76f6a37..ea7fe3d 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -4,23 +4,29 @@ Document one place you used an LLM during this assignment. ## The problem - - -TODO - +While building the stg_trips staging model, i had an SQL syntax issue because i placed the "WHERE" clause before the "FROM" clause. ## The prompt - -TODO - +Im building a dbt staging model and my SQL query is failing. what is the problem here? +my query was : +SELECT + pickup_datetime, + pickup_location_id, + fare_amount, + tip_amount, + trip_distance, + + {{ safe_divide('tip_amount', 'fare_amount') }} as tip_pct + +where pickup_location_id is not null +FROM {{ source('nyc_taxi', 'raw_trips') }} + and fare_amount >= 0 ## The response -TODO +the llm explained that SQL clauses follow a fixed order. The FROM clause must come before the WHERE clause because SQL needs to know the source table before applying filters. ## Reflection @@ -29,8 +35,7 @@ TODO 'ratio' to 'tip_pct' to match the assignment schema." --> TODO - ---- +i changed the query order so that "FROM" comes immediately after the SELECT columns, followed by the WHERE filters. > Remember: never paste real connection strings, passwords, or PII into an LLM. > The NYC TLC dataset is public so sample rows are safe here, but practise the habit. diff --git a/docs/Lineage.png b/docs/Lineage.png new file mode 100644 index 0000000..8ca3cc6 Binary files /dev/null and b/docs/Lineage.png differ diff --git a/macros/safe_divide.sql b/macros/safe_divide.sql index e30c8fa..edbbe46 100644 --- a/macros/safe_divide.sql +++ b/macros/safe_divide.sql @@ -3,8 +3,9 @@ -- Use for tip_pct = tip_amount / fare_amount and similar ratio columns. {% macro safe_divide(numerator, denominator) %} - -- TODO: implement the macro body. - -- Use NULLIF(denominator, 0) to avoid division-by-zero errors. - -- Return only the SQL expression (no SELECT, no semicolon). - NULL + + {{ numerator }} / nullif({{ denominator }}, 0) + {% endmacro %} + + diff --git a/models/marts/_fct_daily_borough_stats.yml b/models/marts/_fct_daily_borough_stats.yml index 15bba90..d02fcdf 100644 --- a/models/marts/_fct_daily_borough_stats.yml +++ b/models/marts/_fct_daily_borough_stats.yml @@ -3,22 +3,36 @@ version: 2 models: - name: fct_daily_borough_stats description: > - TODO: state the grain (one row per ___), the source lineage - (built from ___ and ___), and at least one known caveat - (e.g. rows dropped in staging, any WARN-severity tests). + Daily borough-level taxi statistics with a grain of one row per + (pickup_borough, pickup_date). + Built from stg_trips and stg_zones. + Trips with missing or invalid pickup locations are removed during staging + or excluded by the INNER JOIN. + The avg_tip_pct warning test may return some rows where + tips are higher than the fare amount. + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - pickup_borough + - pickup_date # TODO: Task 5 -- add the compound uniqueness test on the mart's primary # key (pickup_borough, pickup_date). You need the dbt_utils package for # this: declare it in packages.yml and run `dbt deps` first. columns: - name: pickup_borough - description: "TODO: explain what this column contains and where it comes from" + description: "The TLC borough where trips were picked up, sourced from stg_zones.borough" + tests: + - not_null - name: pickup_date - description: "TODO: explain units and how it is derived" + description: "The calendar date of the trip pickup, derived from pickup_datetime." + tests: + - not_null - name: trip_count - description: "TODO: explain what is counted (unit: number of trips)" + description: "The number of taxi trips picked up in the borough on that dat." - name: total_fare - description: "TODO: explain units (USD) and what fare_amount represents" + description: "The total fare amount collected in US dollars, calculated as the sum of fare_amount" - name: avg_tip_pct - description: "TODO: explain the ratio (tip_amount / fare_amount)" + description: "The average tipping ratio calculated as tip_amount divided by fare_amount." - name: avg_trip_distance - description: "TODO: explain units (miles, from TLC source data)" + description: "The average trip distance in miles based on the TLC trip_distance field" diff --git a/models/marts/fct_daily_borough_stats.sql b/models/marts/fct_daily_borough_stats.sql index bf47070..7aa8977 100644 --- a/models/marts/fct_daily_borough_stats.sql +++ b/models/marts/fct_daily_borough_stats.sql @@ -2,6 +2,7 @@ -- Grain: one row per (pickup_borough, pickup_date). -- Used to answer: trip volume, revenue, tipping behaviour, and distance profile -- per borough per day for January 2024. +{{ config(materialized='table') }} WITH trips AS ( SELECT * @@ -26,13 +27,27 @@ SELECT -- total_fare NUMERIC - sum(fare_amount) -- avg_tip_pct NUMERIC - avg(tip_pct) -- avg_trip_distance NUMERIC - avg(trip_distance) - NULL AS pickup_borough, - NULL AS pickup_date, - NULL AS trip_count, - NULL AS total_fare, - NULL AS avg_tip_pct, - NULL AS avg_trip_distance + z.borough AS pickup_borough, + + t.pickup_datetime::date AS pickup_date, + + count(*) AS trip_count, + + sum(t.fare_amount) AS total_fare, + + avg(t.tip_pct) AS avg_tip_pct, + + avg(t.trip_distance) AS avg_trip_distance + FROM trips t -- TODO: add JOIN to zones here +INNER JOIN zones z + + ON t.pickup_location_id = z.location_id -- TODO: add GROUP BY here +GROUP BY + + z.borough, + + t.pickup_datetime::date diff --git a/models/staging/_sources.yml b/models/staging/_sources.yml index 4bbee9c..288cf98 100644 --- a/models/staging/_sources.yml +++ b/models/staging/_sources.yml @@ -5,6 +5,6 @@ sources: schema: nyc_taxi # TODO: confirm this matches the schema where raw_trips and raw_zones live tables: - name: raw_trips - description: "TODO: one sentence on what this table contains and its grain" + description: "Raw NYC green Taxi trip records, with one row representing one individual taxi trip." - name: raw_zones - description: "TODO: one sentence on what this table contains" + description: "Raw TLC taxi zone lookup table containing location IDs and borough/zone information." diff --git a/models/staging/_stg_trips.yml b/models/staging/_stg_trips.yml index 14829a8..31ad815 100644 --- a/models/staging/_stg_trips.yml +++ b/models/staging/_stg_trips.yml @@ -2,19 +2,23 @@ version: 2 models: - name: stg_trips - description: "TODO: state the grain (one row per ___) and what source this reads from" + description: "One row per taxi trip from the raw_trips source after basic cleaning and filtering." columns: - name: pickup_datetime - description: "TODO" + description: "Timestamp when the trip started" + tests: + - not_null # TODO: Task 5 -- add not_null tests on every column used as a join or # group-by key. See the chapter's dbt Tests section for the syntax. - name: pickup_location_id - description: "TODO" + description: "TLC location ID where the trip was picked up." + tests: + - not_null - name: fare_amount - description: "TODO" + description: "Fare charged for the trip in US dollars" - name: tip_amount - description: "TODO" + description: "Tip paid by the passenger in US dollars" - name: trip_distance - description: "TODO" + description: "Distance travelled during the trip in miles" - name: tip_pct - description: "TODO" + description: "Tip as a percentage of the fare amount" diff --git a/models/staging/_stg_zones.yml b/models/staging/_stg_zones.yml index c4b785d..8e625cf 100644 --- a/models/staging/_stg_zones.yml +++ b/models/staging/_stg_zones.yml @@ -2,11 +2,15 @@ version: 2 models: - name: stg_zones - description: "TODO: state the grain and what source this reads from" + description: "One row per TLC taxi zone from the raw_zones source" columns: - name: location_id - description: "TODO" + description: "Unique identifier for each TLC taxi zone." + tests: + - not_null + - unique + # TODO: Task 5 -- this column is the join key. Which two generic tests # guarantee a clean one-to-many join from stg_trips? - name: borough - description: "TODO" + description: "Borough that the taxi zone belongs to." diff --git a/models/staging/stg_trips.sql b/models/staging/stg_trips.sql index b5f7b81..200da18 100644 --- a/models/staging/stg_trips.sql +++ b/models/staging/stg_trips.sql @@ -1,13 +1,18 @@ -- Staging model: one row per NYC green taxi trip (January 2024). -- Renames source columns, adds derived columns, and filters bad rows. -- Downstream: fct_daily_borough_stats joins this to stg_zones. +{{ config(materialized='view') }} SELECT - -- TODO: select the columns you need for the mart: - -- pickup_datetime, pickup_location_id, fare_amount, tip_amount, trip_distance - -- - -- TODO: add tip_pct using {{ safe_divide('tip_amount', 'fare_amount') }} - -- - -- TODO: filter out rows where pickup_location_id IS NULL or fare_amount < 0 + pickup_datetime, + pickup_location_id, + fare_amount, + tip_amount, + trip_distance, + + {{ safe_divide('tip_amount', 'fare_amount') }} as tip_pct FROM {{ source('nyc_taxi', 'raw_trips') }} + +where pickup_location_id is not null + and fare_amount >= 0 diff --git a/models/staging/stg_zones.sql b/models/staging/stg_zones.sql index cc34d23..9d59a0f 100644 --- a/models/staging/stg_zones.sql +++ b/models/staging/stg_zones.sql @@ -1,7 +1,8 @@ -- Staging model: one row per TLC zone (265 zones). -- Exposes location_id and borough for use as a lookup in the mart. +{{ config(materialized='view') }} SELECT - -- TODO: select location_id and borough from {{ source('nyc_taxi', 'raw_zones') }} - + location_id, + borough FROM {{ source('nyc_taxi', 'raw_zones') }} diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..e397fd7 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,5 @@ +packages: + - name: dbt_utils + package: dbt-labs/dbt_utils + version: 1.3.0 +sha1_hash: 226ae69cdfbc9367e2aa2c472b01f99dbce11de0 diff --git a/packages.yml b/packages.yml index bbb2357..ed35da3 100644 --- a/packages.yml +++ b/packages.yml @@ -2,4 +2,6 @@ # `dbt deps` to install it. You need it for the compound uniqueness test # on the mart. See https://hub.getdbt.com/dbt-labs/dbt_utils/latest/ # for the package block syntax. -packages: [] +packages: + - package: dbt-labs/dbt_utils + version: 1.3.0 diff --git a/reports/answers.md b/reports/answers.md index 8fb8cf2..22e2710 100644 --- a/reports/answers.md +++ b/reports/answers.md @@ -1,6 +1,6 @@ # Business Question Answers -Queries run against `dev_.fct_daily_borough_stats`. +Queries run against `dev_baraah.fct_daily_borough_stats`. ## Q1: Highest total `total_fare` across the whole loaded dataset @@ -9,10 +9,20 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats grouped by pickup_borough, sum total_fare, order DESC ``` - -**Result:** TODO - -**Interpretation:** TODO (one sentence) +SELECT + pickup_borough, + SUM(total_fare) AS total_revenue +FROM dev_baraah.fct_daily_borough_stats +GROUP BY pickup_borough +ORDER BY total_revenue DESC +LIMIT 1; +**Result:** + pickup_borough | total_revenue +----------------+--------------- + Manhattan | 493955.62 +(1 row) + +**Interpretation:** The borough with the highest total fare revenue across the loaded dataset was Manhattan $493955.62. --- @@ -23,10 +33,20 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats grouped by pickup_date, sum trip_count, order DESC LIMIT 1 ``` - -**Result:** TODO - -**Interpretation:** TODO (one sentence) +SELECT + pickup_date, + SUM(trip_count) AS total_trips +FROM dev_baraah.fct_daily_borough_stats +GROUP BY pickup_date +ORDER BY total_trips DESC +LIMIT 1; +**Result:** + pickup_date | total_trips +-------------+------------- + 2024-01-17 | 2221 +(1 row) + +**Interpretation:** The busiest day in the dataset was 2024-01-17 with 2221 trips across all boroughs. --- @@ -37,10 +57,25 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: query fct_daily_borough_stats order by avg_tip_pct DESC LIMIT 5 ``` - -**Result:** TODO - -**Interpretation:** TODO — note whether any avg_tip_pct > 1 rows appear and what causes them +SELECT + pickup_borough, + pickup_date, + avg_tip_pct +FROM dev_baraah.fct_daily_borough_stats +ORDER BY avg_tip_pct DESC +LIMIT 5; + +**Result:** + pickup_borough | pickup_date | avg_tip_pct +----------------+-------------+-------------------- + Unknown | 2024-01-30 | 2.500125 + Unknown | 2024-01-07 | 1.3396296296296295 + Unknown | 2024-01-11 | 1.1044956140350877 + Unknown | 2024-01-16 | 1 + Unknown | 2024-01-18 | 0.611111111111111 +(5 rows) + +**Interpretation:** The highest average tip percentage occurred in Unknown on 2024-01-30 . --- @@ -51,7 +86,18 @@ Queries run against `dev_.fct_daily_borough_stats`. ```sql -- TODO: use percentile_cont(0.5) WITHIN GROUP (ORDER BY trip_count) filtered by borough ``` +SELECT + pickup_borough, + percentile_cont(0.5) WITHIN GROUP (ORDER BY trip_count) AS median_trip_count +FROM dev_baraah.fct_daily_borough_stats +WHERE pickup_borough IN ('Manhattan', 'Brooklyn') +GROUP BY pickup_borough; **Result:** TODO + pickup_borough | median_trip_count +----------------+------------------- + Brooklyn | 248 + Manhattan | 1169.5 +(2 rows) -**Interpretation:** TODO (one sentence on the ratio) +**Interpretation:** Manhattan had a higher median daily trip count than Brooklyn. diff --git a/test-output.txt b/test-output.txt new file mode 100644 index 0000000..a5e039c --- /dev/null +++ b/test-output.txt @@ -0,0 +1 @@ +bash: test.sh: No such file or directory diff --git a/tests/assert_avg_tip_pct_within_bounds.sql b/tests/assert_avg_tip_pct_within_bounds.sql index b0fab32..9e91833 100644 --- a/tests/assert_avg_tip_pct_within_bounds.sql +++ b/tests/assert_avg_tip_pct_within_bounds.sql @@ -12,8 +12,15 @@ -- unique_combination primary-key tests, which you want to stay at ERROR. -- -- The test passes (no WARN) when zero rows are returned; any returned rows are flagged. +{{ config(severity='warn') }} -- TODO: write the SELECT here. -- Query {{ ref('fct_daily_borough_stats') }} and return rows where avg_tip_pct > 1. -SELECT NULL AS pickup_borough, NULL AS pickup_date, NULL AS avg_tip_pct -WHERE FALSE -- TODO: replace with the real query +SELECT + pickup_borough, + pickup_date, + avg_tip_pct + +FROM {{ ref('fct_daily_borough_stats') }} + +WHERE avg_tip_pct > 1