diff --git a/.devcontainer/076-GitHubCopilotCostOptimization/devcontainer.json b/.devcontainer/076-GitHubCopilotCostOptimization/devcontainer.json new file mode 100644 index 0000000000..84e10376c4 --- /dev/null +++ b/.devcontainer/076-GitHubCopilotCostOptimization/devcontainer.json @@ -0,0 +1,36 @@ +{ + "name": "076-GitHubCopilotCostOptimization", + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + "workspaceFolder": "/workspace/076-GitHubCopilotCostOptimization/Student/Resources", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached", + "hostRequirements": { + "cpus": 2 + }, + "features": { + "ghcr.io/devcontainers/features/python:1": { "version": "3.12" } + }, + "customizations": { + "vscode": { + "extensions": [ + "github.copilot", + "github.copilot-chat", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode" + ] + }, + "codespaces": { + "openFiles": [ + "076-GitHubCopilotCostOptimization/Student/Resources/README.md" + ] + } + }, + "postCreateCommand": "curl -LsSf https://astral.sh/uv/install.sh | sh && ~/.local/bin/uv tool install specify-cli --from git+https://github.com/github/spec-kit.git", + + "forwardPorts": [3000], + "portsAttributes": { + "3000": { + "label": "CityScout NYC App", + "onAutoForward": "notify" + } + } +} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..5e6740a18d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/dotnet +{ + "name" : "DevOps with GitHub WTH in Codespaces", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/dotnet:0-6.0", + "workspaceFolder": "/workspace/031-DevOpsWithGitHub", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached", + "features": { + "ghcr.io/devcontainers/features/azure-cli:1": { + "installBicep": true, + "version": "latest" + } + } + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [5000, 5001], + // "portsAttributes": { + // "5001": { + // "protocol": "https" + // } + // } + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "dotnet restore", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/xxx-GitHubCopilotCostOptimization/devcontainer.json b/.devcontainer/xxx-GitHubCopilotCostOptimization/devcontainer.json new file mode 100644 index 0000000000..84e10376c4 --- /dev/null +++ b/.devcontainer/xxx-GitHubCopilotCostOptimization/devcontainer.json @@ -0,0 +1,36 @@ +{ + "name": "076-GitHubCopilotCostOptimization", + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + "workspaceFolder": "/workspace/076-GitHubCopilotCostOptimization/Student/Resources", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached", + "hostRequirements": { + "cpus": 2 + }, + "features": { + "ghcr.io/devcontainers/features/python:1": { "version": "3.12" } + }, + "customizations": { + "vscode": { + "extensions": [ + "github.copilot", + "github.copilot-chat", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode" + ] + }, + "codespaces": { + "openFiles": [ + "076-GitHubCopilotCostOptimization/Student/Resources/README.md" + ] + } + }, + "postCreateCommand": "curl -LsSf https://astral.sh/uv/install.sh | sh && ~/.local/bin/uv tool install specify-cli --from git+https://github.com/github/spec-kit.git", + + "forwardPorts": [3000], + "portsAttributes": { + "3000": { + "label": "CityScout NYC App", + "onAutoForward": "notify" + } + } +} diff --git a/076-GitHubCopilotCostOptimization/.wordlist.txt b/076-GitHubCopilotCostOptimization/.wordlist.txt new file mode 100644 index 0000000000..1512ace165 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/.wordlist.txt @@ -0,0 +1,32 @@ +UBB +SDD +completable +Unoptimized +mathrm +tokenization +backoff +invalidations +subtask +Linters +applyTo +CityScout +runnable +Monotonicity +Andressa +Siqueira +Nefedova +discoverable +checkable +pickleball +breakpoint +differentiator +selectable +NYCApp +geolocation +unmanaged +useEffect +useState +YYYY +priceLevel +bestFor +walkability \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Lectures.pptx b/076-GitHubCopilotCostOptimization/Coach/Lectures.pptx new file mode 100644 index 0000000000..34cfe52f45 Binary files /dev/null and b/076-GitHubCopilotCostOptimization/Coach/Lectures.pptx differ diff --git a/076-GitHubCopilotCostOptimization/Coach/README.md b/076-GitHubCopilotCostOptimization/Coach/README.md new file mode 100644 index 0000000000..2c259afe92 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/README.md @@ -0,0 +1,131 @@ +# What The Hack - GitHub Copilot Cost Optimization - Coach Guide + +## Introduction + +Welcome to the coach's guide for the GitHub Copilot Cost Optimization What The Hack. Here you will find links to specific guidance for coaches for each of the challenges. + +This hack teaches teams to maximize code quality while minimizing GitHub Copilot credit spend under usage-based billing (UBB). The focus is on practical engineering techniques: context engineering, model selection, session hygiene, and spec-driven development. + +**NOTE:** If you are a Hackathon participant, this is the answer guide. Don't cheat yourself by looking at these during the hack! Go learn something. :) + +## Coach's Guides + +- Challenge 00: **[Prerequisites & Baseline](./Solution-00.md)** + - Set up the hack environment and establish baseline credit spend measurements +- Challenge 01: **[Spec-Driven Development](./Solution-01.md)** + - Leverage deterministic controls as token controls +- Challenge 02: **[Context Engineering](./Solution-02.md)** + - Optimize instruction architecture and attachment precision +- Challenge 03: **[Model Selection & Output Constraints](./Solution-03.md)** + - Compare models and apply output constraints to reduce costs +- Challenge 04: **[Session Configuration (Tools + Cache)](./Solution-04.md)** + - Manage MCP tool sprawl and understand cache invalidation +- Challenge 05: **[Context Window Management](./Solution-05.md)** + - Use /compact proactively to maintain quality and control costs +- Challenge 06: **[Token Golf Competition](./Solution-06.md)** + - Competitive coding challenge optimizing for lowest credit spend + +## Coach Prerequisites + +This hack has pre-reqs that a coach is responsible for understanding and/or setting up BEFORE hosting an event. Please review the [What The Hack Hosting Guide](https://aka.ms/wthhost) for information on how to host a hack event. + +The guide covers the common preparation steps a coach needs to do before any What The Hack event, including how to properly configure Microsoft Teams. + +### Student Resources + +Before the hack, it is the Coach's responsibility to download and package up the contents of the `/Student/Resources` folder of this hack into a "Resources.zip" file. The coach should then provide a copy of the Resources.zip file to all students at the start of the hack. + +The Student/Resources folder should contain: + +- **Starter codebase** with intentionally sub-optimal GitHub Copilot configuration: + - Bloated `.github/copilot-instructions.md` (~1500 tokens) for Challenge 02 + - Multiple MCP tools enabled (some unnecessary) for Challenge 04 + - Sample code with opportunities for spec-first vs. trial-and-error comparison +- **Baseline coding task** (baseline-task.md) for Challenge 00 +- **Model selection prompts + measurement template** (`Challenge-03-ModelSelection/`) for Challenge 03 +- **Token Golf task specification** for Challenge 06 with acceptance criteria + +Always refer students to the [What The Hack website](https://aka.ms/wth) for the student guide: [https://aka.ms/wth](https://aka.ms/wth) + +**NOTE:** Students should **not** be given a link to the What The Hack repo before or during a hack. The student guide does **NOT** have any links to the Coach's guide or the What The Hack repo on GitHub. + +### Additional Coach Prerequisites + +**Coach must complete Challenge 06 (Token Golf) before the hack:** + +- Complete the Token Golf coding task yourself to establish "par" score +- Document your credit spend and techniques used +- This "par" becomes the benchmark teams compete against + +**Prepare measurement tools:** + +- Verify token visibility in VS Code Output panel (GitHub Copilot Chat) +- Test access to GitHub Copilot usage dashboard + +## Azure Requirements + +This hack does not require Azure resources. All work is done locally with GitHub Copilot in Visual Studio Code. + +## Suggested Hack Agenda + +This hack can be delivered as a half-day or full-day event: + +### Half-Day Format (4 hours) +- Challenge 00: Prerequisites & Baseline (30 minutes) +- Challenge 01: Spec-Driven Development (45 minutes) +- Challenge 02: Context Engineering (45 minutes) +- Challenge 03: Model Selection & Output Constraints (45 minutes) +- Challenge 04: Session Configuration (30 minutes) +- Challenge 06: Token Golf Competition (45 minutes) + +### Full-Day Format (6-7 hours) +- Challenge 00: Prerequisites & Baseline (45 minutes) +- Challenge 01: Spec-Driven Development (60 minutes) +- Challenge 02: Context Engineering (60 minutes) +- Break (15 minutes) +- Challenge 03: Model Selection & Output Constraints (60 minutes) +- Challenge 04: Session Configuration (45 minutes) +- Challenge 05: Context Window Management (45 minutes) +- Break (15 minutes) +- Challenge 06: Token Golf Competition (60 minutes) +- Debrief & Winning Strategies (30 minutes) + +## Repository Contents + +- `./Coach` + - Coach's Guide and related files +- `./Coach/Solutions` + - Solution files with example measurements and techniques for each challenge +- `./Student` + - Student's Challenge Guide +- `./Student/Resources` + - Starter codebase, baseline task, Token Golf specification (Must be packaged up by the coach and provided to students at start of event) + +- Azure resources that will be consumed by a student implementing the hack's challenges +- Azure permissions required by a student to complete the hack's challenges. + +## Suggested Hack Agenda (Optional) + +_This section is optional. You may wish to provide an estimate of how long each challenge should take for an average squad of students to complete and/or a proposal of how many challenges a coach should structure each session for a multi-session hack event. For example:_ + +- Sample Day 1 + - Challenge 1 (1 hour) + - Challenge 2 (30 mins) + - Challenge 3 (2 hours) +- Sample Day 2 + - Challenge 4 (45 mins) + - Challenge 5 (1 hour) + - Challenge 6 (45 mins) + +## Repository Contents + +_The default files & folders are listed below. You may add to this if you want to specify what is in additional sub-folders you may add._ + +- `./Coach` + - Coach's Guide and related files +- `./Coach/Solutions` + - Solution files with completed example answers to a challenge +- `./Student` + - Student's Challenge Guide +- `./Student/Resources` + - Resource files, sample code, scripts, etc meant to be provided to students. (Must be packaged up by the coach and provided to students at start of event) diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-00.md b/076-GitHubCopilotCostOptimization/Coach/Solution-00.md new file mode 100644 index 0000000000..d14906e4fd --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-00.md @@ -0,0 +1,76 @@ +# Challenge 00 - Prerequisites & Baseline - Coach's Guide + +**[Home](./README.md)** - [Next Solution >](./Solution-01.md) + +## Notes & Guidance + +This challenge establishes the measurement baseline that all subsequent challenges compare against. Students must be able to track credit consumption accurately before they can optimize. + +### Key Concepts to Explain + +**Usage-Based Billing (UBB) Fundamentals:** +- Credit cost = (input tokens × input rate) + (output tokens × output rate) +- Different models have different rates (Sonnet/GPT-4 class vs. Haiku class can be 10x difference) +- VS Code Output panel shows token breakdowns per interaction +- GitHub Copilot usage dashboard shows cumulative spend + +**Baseline Purpose:** +- Establishes "before" measurement for all optimization techniques +- Students should use their natural workflow—no optimization yet +- Document approach (inline completions, chat frequency, workspace references used) +- This baseline will likely be inefficient—that's intentional + +### Common Blockers + +**UBB Not Enabled:** +- Some organizations may still be on seat-based billing +- Students need admin access or must coordinate with billing admin +- Workaround: Coach can provide sample baseline data if UBB isn't available + +**Token Visibility Issues:** +- VS Code Output panel must be configured to show GitHub Copilot Chat logs +- Some students may not see token counts if using older Copilot extension version +- Solution: Update GitHub Copilot extension to latest version + +**Baseline Task Selection:** +- Task should be non-trivial (10-20 minutes of development) +- Task should be completable in multiple ways (leaves room for optimization) +- Suggested baseline tasks: + - Implement a REST API endpoint with validation and error handling + - Add authentication middleware to an existing service + - Refactor a module to use dependency injection + +### Expected Time + +45 minutes total: +- 15 minutes: Environment setup and UBB verification +- 20 minutes: Complete baseline coding task +- 10 minutes: Document measurements and approach + +### Success Criteria Validation + +Students should demonstrate: +- Screenshot or screen share of VS Code Output panel showing token counts +- Screenshot of GitHub Copilot usage dashboard showing current credit balance +- Completed baseline task code (doesn't need to be perfect) +- Written documentation: total credits consumed, features used, approach taken + +### Hints to Share + +- Don't optimize yet! Use your natural workflow +- Document everything—this is your "before" picture +- If credits seem surprisingly high, that's normal and expected +- Track not just total credits, but which interactions cost the most +- Note: inline completions are typically much cheaper per-interaction than chat/edits + +### Reference Measurements + +Typical baseline measurements for a 15-20 minute coding task: +- Unoptimized: 50-80 credits (using GPT-4 class model, verbose chat, broad references) +- Expected after optimization: 10-20 credits (targeted instructions, efficient model, pinned attachments) + +### Additional Resources for Coach + +- Ensure Resources.zip contains `baseline-task.md` with clear requirements +- Consider providing a reference completion for teams that get stuck +- Have sample token breakdowns ready to share if students can't see theirs diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-01.md b/076-GitHubCopilotCostOptimization/Coach/Solution-01.md new file mode 100644 index 0000000000..8d74e6bae4 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-01.md @@ -0,0 +1,238 @@ +# Challenge 01 - Spec-Driven Development with Spec Kit - Coach's Guide + +[< Previous Solution](./Solution-00.md) - **[Home](./README.md)** - [Next Solution >](./Solution-02.md) + +# Coach Guide: Direct Prompt vs. Spec-Driven Development + +## Purpose + +This guide helps coaches facilitate an experiment comparing two ways of building the same React and TypeScript application with GitHub Copilot: + +1. A direct implementation prompt. +2. A structured Markdown specification using Spec-Driven Development (SDD). + +The goal is not to prove that one approach is universally better. Participants should observe how context influences implementation time, credit consumption, interruptions, interventions, rework, and the delivered application. + +## Learning Outcomes + +By the end of the activity, participants should be able to: + +- Explain the role of a specification as implementation context. +- Compare direct prompting and SDD under equivalent conditions. +- Distinguish functional effectiveness from implementation efficiency. +- Include specification preparation when evaluating full-cycle efficiency. +- Discuss experimental results without generalizing from a single run. + +## Recommended Duration + +| Activity | Suggested time | +|---|---:| +| Introduction and environment check | 10 minutes | +| Direct prompt implementation | 20–30 minutes | +| SDD implementation | 20–30 minutes | +| Comparison and discussion | 20 minutes | +| Total | 70–90 minutes | + +Do not impose an implementation time limit if it would prevent participants from completing and manually verifying the application. + +## Coach Preparation + +Before the session: + +- Confirm that VS Code, Node.js, npm, and GitHub Copilot are available. +- Select one GitHub Copilot model to use in both runs. +- Keep the provided `sdd.md` content available. +- Prepare a timer and measurement worksheet. +- Ensure participants can inspect Copilot credit consumption. +- Ask participants to close unrelated conversations and projects. +- Explain that generated files cannot be reused between runs. + +The folders must begin empty. Existing scaffolds, templates, or files may affect the comparison and should be recorded if used. + +## Fairness Rules + +The coach must enforce the following controls: + +- Same computer and environment. +- Same VS Code and GitHub Copilot versions. +- Same Copilot model. +- Same application scope and manual verification checklist. +- Separate empty folders and new conversations. +- No reuse of generated code between runs. +- No additional implementation instructions unless required to unblock progress. +- Every clarification, correction, or intervention must be recorded. + +Do not steer Copilot toward a specific framework choice, package, UI design, or solution unless the participant is blocked. Any such assistance counts as an intervention. + +## Facilitation Flow + +### 1. Introduce the Experiment + +Explain that the experiment evaluates three dimensions: + +- **Effectiveness:** whether the application satisfies the requirements. +- **Implementation efficiency:** effort after the prompt or specification is ready. +- **Full-cycle efficiency:** implementation effort plus specification preparation. + +Remind participants that a functional tie can still reveal differences in time, cost, ambiguity, and rework. + +### 2. Run the Direct Prompt Approach + +Before the participant submits the prompt: + +- Confirm the folder is empty. +- Confirm a new Copilot conversation is open. +- Record the selected model. +- Prepare the timer. + +Start timing when the implementation prompt is sent. During execution, observe without directing the participant. + +Record: + +- Start and end time. +- Credits consumed. +- Questions asked by Copilot. +- Dependency or tooling decisions requested. +- Participant interventions. +- Errors, failed commands, and rework. +- Any functionality added beyond the requested scope. + +Stop timing only after the application is running and the full manual checklist has been completed. + +### 3. Run the SDD Approach + +Before the second run: + +- Confirm a different empty folder is being used. +- Create the provided `sdd.md` in the project root. +- Open a new Copilot conversation. +- Select the same model. +- Verify that no files from the first implementation were reused. + +Start timing when the participant sends the instruction to read and implement `sdd.md`. + +Apply the same observation and stopping rules used in the direct prompt run. Do not give the SDD implementation credit for additional features that were not required by the specification. + +### 4. Perform Manual Verification + +Use exactly the same checklist for both applications: + +- [ ] Create a note with a title and content. +- [ ] Create a note without content. +- [ ] Reject creation when the title is empty or whitespace-only. +- [ ] Display all notes created during the session. +- [ ] Update an existing note’s title and content. +- [ ] Reject an update with an empty title. +- [ ] Update without creating a duplicate note. +- [ ] Delete an existing note. +- [ ] Confirm that the deleted note disappears. +- [ ] Reload the page and confirm that notes may be lost. +- [ ] Confirm no backend, database, or persistent storage is used. +- [ ] Confirm no automated tests were added. + +Record failures and corrections instead of silently fixing them. + +## Measurement Definitions + +Use consistent definitions across both runs: + +- **Question:** Copilot explicitly requests information or a decision. +- **Interruption:** progress pauses because participant input is required. +- **Intervention:** the participant supplies a correction, additional instruction, command, or code change. +- **Error:** a build, runtime, dependency, or functional failure. +- **Rework:** work required to correct a missing or incorrect result. +- **Implementation time:** time from sending the implementation instruction until the application runs and passes manual verification. +- **Specification preparation cost:** time and credits used to author or refine the specification, excluding implementation. + +If the specification was supplied ready-made, record its known preparation cost separately. If that cost is unavailable, mark it as “not measured” rather than treating it as zero. + +## Results Worksheet + +| Aspect | Direct prompt | SDD with ready specification | +|---|---:|---:| +| Model used | | | +| Implementation time | | | +| Implementation credits | | | +| Questions asked | | | +| Interruptions | | | +| User interventions | | | +| Errors | | | +| Rework actions | | | +| Requirements passed | /12 | /12 | +| Out-of-scope features | | | +| Qualitative UX score | /5 | /5 | + +Record SDD preparation separately: + +| Specification metric | Result | +|---|---:| +| Preparation time | | +| Preparation credits | | +| Number of revisions | | + +Calculate full-cycle SDD values as: + +$$ +T_{\mathrm{SDD\ total}} += +T_{\mathrm{specification}} ++ +T_{\mathrm{implementation}} +$$ + +$$ +C_{\mathrm{SDD\ total}} += +C_{\mathrm{specification}} ++ +C_{\mathrm{implementation}} +$$ + +## Qualitative UX Guidance + +To reduce subjectivity, score both applications from 1 to 5 using the same criteria: + +- Form and action clarity. +- Validation feedback. +- Note readability. +- Discoverability of edit and delete actions. +- Overall consistency and usability. + +Visual polish is secondary. It must not compensate for missing functional requirements. + +## Debrief Questions + +Ask participants: + +1. Did both applications satisfy the same requirements? +2. Which approach required more decisions during implementation? +3. Which ambiguities appeared in the direct prompt run? +4. Did SDD eliminate ambiguity or move decisions to specification preparation? +5. Which approach used less implementation time and fewer credits? +6. Does the conclusion change when specification preparation is included? +7. What specification content had the greatest implementation impact? +8. Which parts of the specification could be reused? +9. How might a larger application change the outcome? +10. How might later requirement changes affect each approach? + +## Coaching Notes + +Avoid announcing an expected winner. Common valid outcomes include: + +- Both approaches deliver equivalent applications. +- SDD reduces interruptions but costs more overall for a small application. +- The direct prompt is faster but requires more decisions or corrections. +- The structured specification improves consistency without reducing credits. +- Natural model variation outweighs the difference between approaches. + +Treat every result as an observation, not proof. Encourage participants to describe what happened, identify contributing factors, and state the experiment’s limitations. + +## Completion Criteria + +The challenge is complete when: + +- Both applications have been built independently. +- Both have been evaluated with the same checklist. +- All measurements and interventions have been recorded. +- Implementation and specification preparation costs remain separate. +- Participants can explain the observed tradeoffs and limitations. diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-02.md b/076-GitHubCopilotCostOptimization/Coach/Solution-02.md new file mode 100644 index 0000000000..2609bcbc2b --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-02.md @@ -0,0 +1,145 @@ +# Challenge 02 - Context Engineering - Coach's Guide + +[< Previous Solution](./Solution-01.md) - **[Home](./README.md)** - [Next Solution >](./Solution-03.md) + +## Notes & Guidance + +This challenge teaches students that persistent context has recurring costs. The goal is to minimize always-on overhead while maintaining quality through scoped instructions and precise attachments. + +### Key Concepts to Explain + +**The Three Types of Context Costs:** + +1. **Persistent Instructions (recurring cost):** + - `.github/copilot-instructions.md` loads on EVERY interaction + - Example: 1500-token global file = 1500 tokens × every chat/edit call + - Target: keep global instructions under 500-1000 tokens + +2. **GitHub Copilot Skills (conditional cost):** + - Only load when explicitly invoked + - Example: `@workspace /new` only loads creation templates when called + - Best for: task-specific patterns that don't apply to all work + +3. **Attachments (per-call cost):** + - You control exactly what's included each time + - Example: `#file:src/api/routes.ts` only loads when you attach it + - `#Codebase` can pull in thousands of unnecessary tokens + +### Part A: Instruction Scoping Strategy + +**Audit Process:** + +Help students categorize their bloated instructions: + +- **Keep Global (under 500 tokens):** + - Code style preferences that apply universally (e.g., "prefer functional patterns") + - Quality standards (e.g., "always handle errors explicitly") + - Organization-wide conventions + +- **Move to Language-Specific:** + - `.github/copilot-instructions-python.md` for Python-only rules + - `.github/copilot-instructions-typescript.md` for TypeScript-only rules + - These only load when working in that language + +- **Move to Path-Specific:** + - `src/api/.github/copilot-instructions.md` for API-specific patterns + - `tests/.github/copilot-instructions.md` for testing conventions + - These only load when working in that directory + +- **Convert to Skills:** + - Task-specific templates (e.g., "how to write a migration") + - Conditional patterns (e.g., "when adding a new model, do X") + +**Expected Savings:** + +- Before: ~1500 tokens per interaction (global instructions) +- After: ~400 tokens global + ~200 tokens scoped (only when in scope) +- Result: 60-70% reduction in instruction overhead for most interactions + +### Part B: Attachment Precision + +**Common Anti-Pattern:** + +``` +Prompt: "Refactor the authentication logic" +Attachments: #Codebase +``` +This pulls in everything—potentially 50,000+ tokens for a large repo. + +**Optimized Pattern:** + +``` +Prompt: "Refactor the authentication logic" +Attachments: + - #file:src/auth/middleware.ts:1-50 + - #file:src/auth/types.ts + - #file:tests/auth.test.ts +``` +This pulls in exactly what's needed—perhaps 500 tokens. + +**Demonstration Exercise:** + +Have students complete the same refactor twice: +1. With `#Codebase` (record token count from Output panel) +2. With pinned files (record token count) + +Typical results: 10,000+ tokens → 800 tokens (90%+ reduction) + +### Common Blockers + +**Students Don't Know What to Keep Global:** + +Rule of thumb: If a rule only applies to 20% of your codebase, it shouldn't be global. + +**Path-Specific Instructions Not Loading:** + +GitHub Copilot walks up the directory tree looking for `.github/copilot-instructions.md` files. Ensure: +- Directory structure is correct +- File is named exactly `.github/copilot-instructions.md` (not `.github/copilot-instructions-api.md` in the api folder) + +**Can't Measure Token Reduction:** + +- VS Code Output panel (View → Output → "GitHub Copilot Chat") shows token counts +- If not visible, update GitHub Copilot extension to latest version + +### Expected Time + +60 minutes total: +- 20 minutes: Audit and categorize global instructions +- 20 minutes: Restructure into scoped files +- 20 minutes: Test attachment precision and measure savings + +### Success Criteria Validation + +Students should demonstrate: +- Global `.github/copilot-instructions.md` reduced to under 500 tokens +- At least one path-specific or language-specific instruction file created +- Token comparison showing savings from pinned attachments vs. `#Codebase` +- Same baseline task completed with reduced credit cost + +### Hints to Share + +- Start by asking: "Does this rule apply to ALL code I write?" If no, scope it +- Use language-specific files for linting/formatting rules that vary by language +- Use path-specific files for architectural patterns (e.g., API conventions in `src/api/`) +- Pin files when you know exactly what you need; use `#Codebase` only for discovery +- The goal isn't zero instructions—it's right-sized instructions + +### Reference Solutions + +Typical restructuring: + +**Before (1500 tokens global):** +``` +.github/copilot-instructions.md (all rules) +``` + +**After (400 tokens global + scoped):** +``` +.github/copilot-instructions.md (universal rules, ~400 tokens) +.github/copilot-instructions-python.md (Python-specific, ~300 tokens) +.github/copilot-instructions-typescript.md (TypeScript-specific, ~250 tokens) +src/api/.github/copilot-instructions.md (API patterns, ~200 tokens) +``` + +Students working in `src/api/*.ts` now load: 400 + 250 + 200 = 850 tokens (vs. 1500 before) diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-03.md b/076-GitHubCopilotCostOptimization/Coach/Solution-03.md new file mode 100644 index 0000000000..455fad835b --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-03.md @@ -0,0 +1,137 @@ +# Challenge 03 - Model Selection & Output Constraints - Coach's Guide + +[< Previous Solution](./Solution-02.md) - **[Home](./README.md)** - [Next Solution >](./Solution-04.md) + +## Notes & Guidance + +This challenge teaches one durable idea: **total cost = token rate × tokens × attempts**, so +*lower cost per request is not lower cost per result.* Students prove it by running the same +prompt on a **base model** and a **premium model** across an **easy** task and a **hard** task, +measuring outcome and tokens. The magic is the contrast: opposite winners on the two tasks. + +> **Terminology:** we use **base model** (included, no premium-request multiplier — e.g. +> GPT-4.1, GPT-4o) and **premium model** (frontier/reasoning model billed as a premium request +> with a multiplier — e.g. GPT-5, Claude Sonnet, o-series). Avoid "cheap/expensive" with +> students — it's imprecise and the whole point is that "cheap" can be the costliest choice. + +### The core mechanic + +| | Easy task (version sort) | Hard task (the car wash) | +|---|---|---| +| Base model | ✅ correct, **fewer tokens** | ❌ confidently wrong ("walk"), looks low-cost (a trap) | +| Premium model | ✅ correct, more tokens | ✅ correct ("drive"), more tokens | +| **Lesson** | Use the base model → save credits | Base model's wrong answer is the costliest option → pay for capability | + +### Reference answer key (measured) + +Measured via the GitHub Models inference API, single-shot, **n=4 per cell** (see +`Solutions/Challenge-03-ModelSelection-Solutions/`). Exact numbers vary by model/version; the +**pattern** is what matters. + +| Model | Easy: pass rate | Easy: avg tokens | Hard: pass rate | Hard: avg tokens | +|---|---|---|---|---| +| `gpt-4.1` (base) | **4/4 (100%)** | ~106 | **0/4 (0%)** — answers "walk" | ~166 | +| `gpt-5` (premium) | **4/4 (100%)** | ~293 | **4/4 (100%)** — answers "drive" | ~1000 | + +Takeaways to draw out: +- Easy task: identical correct answer, but the base model used **~2.8× fewer tokens.** +- Hard task: the base model was wrong **every** time (recommends walking, leaving the car at + home). It was not even lower-cost to be wrong here — and no number of retries reliably fixes a + reasoning-ceiling failure. The premium model was the only correct option. + +### Why these specific tasks + +- **Version sort (easy):** real, practical, and both tiers get it right — so the only + differentiator is token cost. Clean "base model wins" demonstration. +- **The car wash (hard):** a pragmatic-reasoning trap. The "40 meters" bait makes base models + fixate on distance ("walk, it's greener") and miss that *the car must be present to be + washed.* The right answer (**drive**) is obvious to any human in a second — so students can + *see* the base model is wrong without trusting us. Crucially this tracks **capability**, so + results are monotonic with model strength. +- **Rejected — letter-counting / "gotcha" tasks** (e.g. "count the l's"): these are + **tokenization artifacts, not reasoning tests.** We measured them and results do **not** track + price — a base model can pass while a premium model fails (we observed Claude Haiku correct + and Claude Opus wrong on the same letter-count). That would teach the *opposite* of the + lesson. Do not use letter-counts, decimal-comparison, or "strawberry"-style traps. +- **Rejected — coin-change "greedy trap":** modern small models pass it; no longer discriminates. + +### The "no retry" rule, and the cost-of-retries reasoning (Part 2) + +Tell students explicitly: when the base model fails Part 2, **do not coach or retry it** for the +measurement. Then have them reason about retries: +- *How many base-model attempts would cost more than one premium call?* (Plug in token counts or + your organizations multipliers.) +- *Would any number of attempts reliably get the right answer here?* (No — it's a capability + ceiling. This is the key insight: retrying the lower-cost model is throwing good money after + bad on tasks it can't reason through.) + +### Part 4 — classify your own tasks + +This is where the skill becomes transferable. Students write two real tasks from their own work, +predict the tier, run both, and check. Push them to articulate *why* a task is base- vs +premium-tier (well-defined/low-ambiguity vs multi-step reasoning/judgment) — that predictive +rule of thumb is the real deliverable. + +## How students run it + +No codebase. Students use the **VS Code Copilot Chat model picker** to switch between a +base-tier and premium-tier model, paste the two prompts from +`Student/Resources/Challenge-03-ModelSelection/prompts.md`, and read tokens from the Output +panel / usage view. They fill in `measurements-template.md`. + +> Organizations expose different models. Coach students to pick *any* base-tier vs premium-tier pair +> their picker offers — the pattern holds. If your organization only exposes one tier, use the coach +> answer key above as the demonstration, or run the harness live. + +> **Token visibility caveat:** depending on Copilot version/organization, the Output panel may not show a +> clean per-request token number. If so, students can still observe the *pattern* (premium model +> emits much more text). For exact counts, the coach harness (or the GitHub Models playground) +> reports `usage.total_tokens` directly. See `TODO.md` — aligning the token-measurement +> instructions across the whole hack (Challenge 00 sets this up) is tracked there. + +## Expected Time + +~45 minutes: +- 10 min: Part 1 (easy) on both models +- 10 min: Part 2 (hard) on both models + cost-of-retries reasoning +- 10 min: Part 3 compare + discussion +- 10 min: Part 4 classify-your-own-tasks +- 5 min: optional output-constraint run + +## Success Criteria Validation + +Students should produce a completed Part 3 table showing **opposite winners**: base model wins +the easy task on tokens; premium model is the only correct option on the hard task. They should +articulate the retry math (attempts-to-break-even) and the capability-ceiling point, and +classify two of their own tasks by tier. Bonus: a measurable token drop from an output +constraint on a verbose task. + +## Common Blockers + +- **Student retries the base model until it's right.** Stop them — that defeats the lesson. + Redirect to the cost-of-retries reasoning instead. +- **Base model happens to say "drive" once.** It's stochastic; have them run it 2–3×. It + recommends walking the large majority of the time. (If a student's base model reliably gets it + right, their "base" model may actually be fairly strong — have them drop to the smallest model + the picker offers.) +- **Organization exposes only one model tier.** Fall back to the answer key, or use the coach harness. +- **Token counts not visible.** Premium model still visibly produces much more text — record the + pattern. For exact numbers use the harness. (See token-visibility caveat above.) +- **Premium model token counts vary a lot.** Reasoning models have variable hidden token use; + emphasize ratio/pattern over absolute numbers. + +## Hints to Share + +- "Base" vs "premium" is about *total* cost, not per-request price. A wrong answer you have to + redo is the costliest outcome. +- Match the model to the task: base for well-defined/low-ambiguity work, premium for anything + needing real reasoning or judgment. +- Output constraints cut cost on any model — but they don't make a wrong model right. +- The car-wash distance is bait. The task is testing whether the model reasons about the *goal*. + +## Reference Data & Reproduction + +See `Solutions/Challenge-03-ModelSelection-Solutions/`: +- `harness.py` — measurement harness (counts pass/fail + tokens, with rate-limit backoff) +- `results.md` — the measured tables above +- `TODO.md` — author follow-ups (token-measurement alignment, advanced/Auto-mode scope) diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-04.md b/076-GitHubCopilotCostOptimization/Coach/Solution-04.md new file mode 100644 index 0000000000..8166ef126b --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-04.md @@ -0,0 +1,202 @@ +# Challenge 04 - Session Configuration (Tools + Cache) - Coach's Guide + +[< Previous Solution](./Solution-03.md) - **[Home](./README.md)** - [Next Solution >](./Solution-05.md) + +## Notes & Guidance + +This challenge teaches students that MCP tools add token overhead on every request, and that configuration changes can invalidate cached context. The TypeScript repository scan scenario provides a simple, measurable task to observe these effects. + +### Key Concepts to Explain + +**MCP Tool Overhead:** +- Each MCP tool's definition (name, description, parameter schema) is included in the system prompt +- This overhead applies to every LLM request, even if the tool is never invoked +- The GitHub MCP server typically enables 6-8 tools by default +- For the repository scan task, students likely only need `search_code` and `get_file_contents` +- Unused tools like `create_issue`, `search_users`, `list_pull_requests` add unnecessary overhead + +**Cache Invalidation:** +- GitHub Copilot CLI uses Anthropic Claude models with prompt caching +- Cache invalidation occurs when the system prompt changes (e.g., tool definitions, instructions) +- Common triggers: toggling MCP tools, editing `.github/copilot-instructions.md`, switching models +- Clean sessions maintain stable configuration = higher cache reuse = lower costs +- Messy sessions with mid-task changes = cache invalidation = reprocessing overhead + +**Important Note on Documentation:** +GitHub's official documentation doesn't explicitly list cache invalidation triggers. Students discover these through experimentation by observing credit spikes after configuration changes. The underlying Anthropic prompt caching technology invalidates cache when the prompt prefix changes. + +### The Scenario + +Students use GitHub Copilot CLI to scan `microsoft/TypeScript` repository: +- Find all `package.json` files +- Find all `tsconfig.json` files +- Read main package.json for dependencies +- Generate a formatted report + +This is a multi-turn task (5-10 agent steps) that provides multiple measurement opportunities. + +### Expected Time + +60 minutes: +- 15 minutes: Complete initial scan with all tools, record baseline +- 10 minutes: Identify unused tools via VS Code Output panel +- 15 minutes: Disable unused tools, repeat scan, measure savings +- 15 minutes: Clean vs. messy session comparison +- 5 minutes: Document findings + +### Expected Measurements + +**Tool Overhead Savings:** +- Baseline: 6-8 GitHub MCP tools enabled +- Optimized: 2 tools enabled (search_code, get_file_contents) +- Removed: 4-6 unused tools +- Each unused tool: ~100-300 tokens per agent step +- Scan task: ~5-10 agent steps +- **Expected savings: 2,000-15,000 tokens (20-150 credits)** + +Actual savings vary based on: +- Model selected (Sonnet vs Haiku have different base costs) +- Number of tools initially enabled +- Complexity of tool schemas +- Number of agent steps taken + +**Cache Invalidation Impact:** +- Clean session: Configure once, complete scan = stable cache +- Messy session: Toggle tool mid-scan + edit instructions = cache invalidations +- **Expected difference: 10-30% higher cost for messy session** + +The difference may be subtle since the scan task is relatively short. Longer tasks show more dramatic cache invalidation penalties. + +### Common Blockers + +**Students Can't Find MCP Tool Configuration:** +- MCP servers are configured in Copilot CLI settings (platform-specific) +- On macOS/Linux: `~/.copilot/mcp.json` or via CLI settings +- On Windows: `%USERPROFILE%\.copilot\mcp.json` +- Students can also use `copilot mcp` command to manage servers + +**Students Don't See Tool Invocations:** +- VS Code Output panel: View → Output → select "GitHub Copilot" from dropdown +- Tool calls appear as JSON-RPC messages showing tool name and parameters +- If no tools appear, they may not have GitHub MCP server configured +- Provide example output so they know what to look for + +**Students Can't Measure Credits Accurately:** +- Use `/usage` command after each task to see credit breakdown +- GitHub Copilot usage dashboard shows cumulative spend (may be delayed) +- Encourage students to record: task name, timestamp, credits used +- Focus on relative differences (before/after) rather than absolute numbers + +**Cache Invalidation Effects Are Subtle:** +- Short tasks may not show dramatic differences +- Encourage students to make obvious changes: toggle multiple tools, edit instructions significantly +- Credit spikes may be small (10-30%) but still measurable +- The learning goal is awareness of the pattern, not hitting specific numbers + +### Hints to Share + +**If students struggle to identify unused tools:** +- "Look for tools that never appear in the Output panel during your scan" +- "Ask yourself: did I create any issues? Did I search for users? If no, you don't need those tools" +- "For a read-only repository scan, you only need search and read tools" + +**If students can't see cache invalidation effects:** +- "Try making more dramatic changes mid-session—edit instructions significantly, toggle 2-3 tools" +- "The effect is more visible on longer tasks; your short scan may show smaller differences" +- "Focus on observing the pattern, not hitting a specific percentage" + +**If students question why this matters:** +- "In production, developers run hundreds of Copilot interactions per day" +- "Small per-request overhead × hundreds of requests = significant monthly cost" +- "Clean session hygiene is like good Git hygiene—prevents accumulated waste" + +### Technical Deep Dive (If Asked) + +**How MCP Tools Add Overhead:** + +From the Model Context Protocol specification, each tool includes: +```json +{ + "name": "get_weather", + "description": "Get current weather information for a location", + "inputSchema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name or zip code" + } + }, + "required": ["location"] + } +} +``` + +This full schema is sent to the LLM on every request so the model knows: +- What tools are available +- What each tool does +- How to call them correctly + +A simple tool uses ~100-150 tokens; complex tools with many parameters use 300-500+ tokens. + +**Why Cache Invalidation Happens:** + +GitHub Copilot CLI uses Anthropic Claude models. Anthropic's prompt caching works by caching the "prompt prefix" (system instructions + tool definitions + conversation history up to a cache marker). + +If anything in that prefix changes—even one token—the cache is invalidated and must be rebuilt. This means: +- Toggling a tool → system prompt changes → cache invalidated +- Editing instructions → system prompt changes → cache invalidated +- Switching models → different model has its own cache space + +Reference: [Anthropic Prompt Caching](https://claude.com/blog/prompt-caching) + +### Success Criteria Validation + +Students successfully complete when they can show: + +1. ✅ **Tool audit documentation:** + - Screenshot or text output showing enabled tools before optimization + - Evidence of which tools were actually invoked (VS Code Output panel) + - Screenshot showing reduced tool list after optimization + +2. ✅ **Measurable savings:** + - Credit spend for baseline scan (all tools enabled) + - Credit spend for optimized scan (unused tools disabled) + - Calculated savings (even if small) + +3. ✅ **Cache invalidation observation:** + - Credit spend for clean session + - Credit spend for messy session + - Description of which changes they made mid-session + - Observation that messy session cost more + +4. ✅ **Learning articulation:** + - Can explain why unused tools cost tokens + - Can name at least 2-3 configuration changes that invalidate cache + - Understands that stable configuration = better cache reuse + +### Extension Challenges + +For teams that finish early: + +**Advanced Tool Audit:** +- Configure 3-4 different MCP servers (GitHub, filesystem, HTTP, database) +- Complete a task using only one server +- Measure overhead from all the extra unused servers + +**Model Comparison:** +- Complete the scan with Sonnet (all tools) vs Haiku (minimal tools) +- Observe that cheaper model + clean config can beat expensive model + bloated config + +**Long Session Test:** +- Use Copilot CLI for 30+ minutes of varied tasks +- Observe compaction behavior when context window fills (~80% capacity) +- Measure whether frequent configuration changes trigger more compaction + +### Additional Resources for Students + +Share after they complete the challenge: + +- [Managing Context in GitHub Copilot CLI](https://docs.github.com/en/copilot/concepts/agents/copilot-cli/context-management) - Official docs on context window and compaction +- [MCP Tools Specification](https://modelcontextprotocol.io/specification/2025-06-18/server/tools.md) - Details on what's included in tool schemas +- Research report saved during hack preparation with detailed citations and technical validation diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-05.md b/076-GitHubCopilotCostOptimization/Coach/Solution-05.md new file mode 100644 index 0000000000..27d2a4afae --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-05.md @@ -0,0 +1,44 @@ +# Challenge 05 - Context Window Management - Coach's Guide + +[< Previous Solution](./Solution-04.md) - **[Home](./README.md)** - [Next Solution >](./Solution-06.md) + +## Notes & Guidance + +This challenge teaches proactive context management vs. reactive recovery. The biggest avoidable cost is not running `/compact` until context rot has already degraded quality. + +### Key Concepts to Explain + +**Context Rot Timeline:** +- 0-60% capacity: Quality remains high +- 60-70% capacity: First signs of degradation (inconsistencies) +- 70-85% capacity: Noticeable quality loss, forgotten requirements +- 85-100% capacity: Severe degradation, expensive recovery loops + +**Two Commands, Different Purposes:** +- `/compact`: Summarizes previous work, preserves state, preserves cache (steering wheel) +- `/clear`: Wipes everything, cold cache restart (reset button) + +**Natural Breakpoints for Compaction:** +- Feature complete and tests passing +- Architecture decision made +- Switching to new topic/module +- Before context hits 60% utilization + +### Expected Time + +45 minutes: +- 15 minutes: Observe context rot (Part A) +- 20 minutes: Demonstrate proactive compaction (Part B) +- 10 minutes: Document findings + +### Success Criteria Validation + +Students should demonstrate measurable quality degradation after 70% context utilization and show credit savings from proactive `/compact` vs. letting context rot. + +### Hints to Share + +- Context window utilization visible in VS Code status bar or Output panel +- Run `/compact` before you feel the pain, not after +- Natural breakpoints: "I just finished X, now starting Y"—compact at the transition +- `/compact` preserves cache; `/clear` forces cold restart +- The cost isn't running `/compact` too often—it's running it too late diff --git a/076-GitHubCopilotCostOptimization/Coach/Solution-06.md b/076-GitHubCopilotCostOptimization/Coach/Solution-06.md new file mode 100644 index 0000000000..3ee6c24ff6 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solution-06.md @@ -0,0 +1,189 @@ +# Challenge 06 - Token Golf Competition - Coach's Guide + +[< Previous Solution](./Solution-05.md) - **[Home](./README.md)** + +## Notes & Guidance + +Token Golf is the capstone challenge where teams apply all optimization techniques competitively. This is both a validation of learning and a motivating team activity. + +### Coach Pre-Work (CRITICAL) + +**You must complete the Token Golf task yourself before the hack:** + +1. Implement the coding task specified in `/Student/Resources/Challenge06/token-golf-task.md` +2. Use optimization techniques from Challenges 1-5 +3. Record your total credit consumption—this becomes "par" +4. Document your approach: prompt sequence, model choices, techniques used +5. Prepare to share your "par score" at the start of Challenge 06 + +**Setting Par:** + +Your "par score" should represent a well-optimized solution—not perfect, but competent. Aim for: +- Global instructions scoped appropriately (Challenge 01) +- Efficient model selection (Challenge 02) +- Clean session hygiene (Challenge 03) +- Proactive compaction (Challenge 04) +- Spec-first approach (Challenge 05) + +Typical par scores for a moderate complexity task: 15-25 credits (vs. 60-100 unoptimized). + +### Key Concepts to Explain + +**Scorecard Requirements:** + +Teams must document their complete approach, not just final credits. This prevents "got lucky" wins and ensures learning transfer. Required scorecard elements: + +- Total credits consumed (from VS Code Output panel) +- Prompt sequence (what prompts, in what order) +- Models selected for each subtask (e.g., Haiku for boilerplate, Sonnet for complex logic) +- Techniques from previous challenges applied: + - Which instruction scoping strategy (Challenge 01) + - Output constraints used (Challenge 02) + - Tools pruned (Challenge 03) + - Compaction points (Challenge 04) + - Spec-first approach (Challenge 05) +- Any custom techniques or strategies + +**What Counts Toward Score:** + +- All GitHub Copilot interactions: inline completions, chat, edits, workspace queries +- Token costs visible in VS Code Output panel +- From task start to acceptance criteria met (tests pass, docs exist, quality standards met) + +**What Doesn't Count:** + +- Manual typing (zero cost) +- Linters/type checkers/tests running locally (zero cost) +- Time spent (not measured, only credits) + +### Task Design Guidance + +The Token Golf task should: +- Be completable in 45-60 minutes by an experienced developer +- Have clear, measurable acceptance criteria (tests pass, docs exist, etc.) +- Offer multiple solution approaches (so there's no single "right" path) +- Require enough complexity that optimization techniques matter + +**Example Task Structure:** + +```markdown +# Token Golf Task: Build a Rate-Limiting Middleware + +## Functional Requirements +- Implement rate limiting middleware that tracks requests per IP address +- Support configurable limits (requests per minute) +- Return HTTP 429 when limit exceeded +- Clean up expired entries periodically + +## Acceptance Criteria +- All provided unit tests pass +- Middleware handles concurrent requests correctly +- Documentation includes usage examples +- Code passes linting and type checking +- No security vulnerabilities in rate tracking logic + +## Provided Files +- tests/rate-limiter.test.ts (failing tests you must make pass) +- types/middleware.ts (type definitions) +- example-usage.md (what the docs should look like) +``` + +### Competition Format Options + +**Option 1: All Teams Same Task (Standard)** +- Everyone implements the same feature +- Direct comparison of credit costs +- Simplest to judge + +**Option 2: Teams Choose Task (Advanced)** +- Provide 2-3 tasks of similar complexity +- Teams choose which to implement +- Normalizes for skill/interest differences + +**Option 3: Live Leaderboard (High Energy)** +- Teams submit scores as they complete +- Display running leaderboard on screen +- Creates competitive energy + +### Common Blockers + +**Teams Don't Know Where to Start:** + +Hint: Apply the research → plan → implement loop: +1. Research: Read requirements, understand acceptance criteria +2. Plan: Write failing tests, define types, establish spec +3. Implement: Use GitHub Copilot to code against the spec + +**Teams Optimize for Speed, Not Cost:** + +Remind: This is token golf, not speed golf. Taking an extra 2 minutes to write tests first can save 20 credits in trial-and-error. + +**Teams Hit Dead-Ends and Burn Credits Recovering:** + +Encourage `/clear` when stuck—start fresh rather than spending credits digging out of a wrong approach. + +### Expected Time + +60 minutes total: +- 5 minutes: Explain task and rules, announce "par score" +- 45 minutes: Teams implement and measure +- 10 minutes: Debrief and winning strategies + +### Judging Criteria + +**Primary:** Lowest credit cost meeting all acceptance criteria + +**Tiebreakers (in order):** +1. Completion time (earliest submission) +2. Code quality (subjective coach judgment) + +**Disqualifications:** +- Missing acceptance criteria (tests fail, docs missing, linting errors) +- Scorecard not documented (can't verify techniques used) + +### Debrief Discussion Points + +After announcing the winner, discuss: + +**Winning Strategies:** +- What techniques from Challenges 1-5 did the winner apply? +- Where did other teams spend unnecessary credits? +- What would you do differently next time? + +**Key Lessons:** +- "Write the test first" is both good TDD and good token economics +- Cheaper models with tight constraints often beat expensive models with verbose output +- The biggest credit burns are recovery loops—avoid them with upfront specs +- Cache hygiene matters—context rot forces expensive recovery + +### Variations for Future Events + +**Handicap System:** +- Give less experienced teams a credit bonus (e.g., +10 credits) +- Levels the playing field while still rewarding optimization + +**Relay Format:** +- Teams implement different features sequentially +- Each team inherits the codebase from the previous team +- Tests cache management and context handoff strategies + +**Real-World Constraints:** +- Limit model selection (e.g., Haiku only—forces maximum efficiency) +- Introduce mid-task requirement changes (tests cache invalidation handling) + +### Reference Par Scores + +Typical par scores for well-designed tasks: +- Simple task (15-20 min): 10-15 credits +- Moderate task (30-45 min): 15-25 credits +- Complex task (60 min): 25-40 credits + +If student scores are all within 10% of par, the task is well-calibrated. If spread is wide (some 2x par, some under par), the task may have trap paths that burn credits unnecessarily. + +### Post-Event Follow-Up + +Encourage teams to: +- Document their organization's "token golf guidelines" based on learnings +- Share winning strategies internally +- Apply these techniques to real work (measure before/after on production tasks) +- Track team-wide credit spend reduction over next 30 days \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/.gitkeep b/076-GitHubCopilotCostOptimization/Coach/Solutions/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/copilot-instructions.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/copilot-instructions.md new file mode 100644 index 0000000000..ec736a1d70 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/copilot-instructions.md @@ -0,0 +1,5 @@ +# Copilot Instructions (GLOBAL) + +- Write clean, readable, maintainable code +- Prefer clarity over complexity +- Return structured JSON responses \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/api.instructions.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/api.instructions.md new file mode 100644 index 0000000000..812179cc0f --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/api.instructions.md @@ -0,0 +1,11 @@ +--- +applyTo: "src/routes/**/*.ts" +--- + +# API Rules + +- Use RESTful conventions +- Return structured JSON responses +- Include a `count` field for list responses +- Validate request inputs +- Use appropriate HTTP status codes \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/events.instructions.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/events.instructions.md new file mode 100644 index 0000000000..ac754d838a --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/instructions/events.instructions.md @@ -0,0 +1,11 @@ +--- +applyTo: "src/routes/events.ts" +--- + +# Events Rules + +- Only return events where `price === 0` +- Only include events happening within the next 7 days +- Exclude past events +- Sort results by date ascending +- Ensure each event has name, date, and location \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/skills/free-events-endpoint/SKILL.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/skills/free-events-endpoint/SKILL.md new file mode 100644 index 0000000000..6a274076cf --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/.github/skills/free-events-endpoint/SKILL.md @@ -0,0 +1,12 @@ +--- +name: free-events-endpoint +description: Use when creating or updating a free events endpoint +--- + +# Free Events Endpoint + +1. Filter events where `price === 0` +2. Filter events within the next 7 days +3. Exclude invalid or incomplete entries +4. Sort results by date ascending +5. Return `{ count, events }` \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/README.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/README.md new file mode 100644 index 0000000000..53366e4c00 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/README.md @@ -0,0 +1,36 @@ + +# CityScout NYC - ✅ Challenge 2 Complete Solution (v2) + +This is the fully optimized version showing: +- Reduced global instructions ✅ +- Scoped instructions ✅ +- Copilot Skill ✅ +- Complete endpoint ✅ +- Multi-domain repo (events + restaurants + neighborhoods) ✅ + +--- + +## ✅ Key Changes + +### Global → Reduced +Only universal rules kept + +### Scoped Instructions Added +- API rules +- Events rules + +### Skill Added +Reusable pattern for endpoint implementation + +### Endpoint Implemented +- price === 0 +- within 7 days +- sorted + +--- + +## ✅ What to Demonstrate + +❌ #Codebase → sees EVERYTHING (events + restaurants + neighborhoods) +✅ Pinned files → sees ONLY events + diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/events.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/events.ts new file mode 100644 index 0000000000..cbccf0bfab --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/events.ts @@ -0,0 +1,9 @@ +const dateFromToday = (offsetDays: number) => + new Date(Date.now() + offsetDays * 86_400_000).toISOString().slice(0, 10); + +export const events = [ + { name: 'Central Park Yoga', date: dateFromToday(1), price: 0 }, + { name: 'Brooklyn Flea Market Walk', date: dateFromToday(2), price: 0 }, + { name: 'Rooftop Movie Night', date: dateFromToday(3), price: 15 }, + { name: 'Jazz in Washington Square Park', date: dateFromToday(10), price: 0 } +]; diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/neighborhoods.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/neighborhoods.ts new file mode 100644 index 0000000000..a1b5a81feb --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/neighborhoods.ts @@ -0,0 +1,5 @@ + +export const neighborhoods = [ + { name: 'West Village', borough: 'Manhattan' }, + { name: 'Williamsburg', borough: 'Brooklyn' } +]; diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/restaurants.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/restaurants.ts new file mode 100644 index 0000000000..65c0434049 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/data/restaurants.ts @@ -0,0 +1,5 @@ + +export const restaurants = [ + { name: 'West Village Pasta', borough: 'Manhattan', cuisine: 'Italian' }, + { name: 'Williamsburg Tacos', borough: 'Brooklyn', cuisine: 'Mexican' } +]; diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/events.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/events.ts new file mode 100644 index 0000000000..0e4269b584 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/events.ts @@ -0,0 +1,25 @@ + +import { Router } from 'express'; +import { events } from '../data/events.js'; + +export const eventsRouter = Router(); + +const isWithin7Days = (dateStr) => { + const today = new Date(); + const eventDate = new Date(dateStr); + const diff = (eventDate - today) / (1000 * 60 * 60 * 24); + return diff >= 0 && diff <= 7; +}; + +// ✅ FINAL IMPLEMENTATION +eventsRouter.get('/free-this-week', (req, res) => { + const result = events + .filter(e => e.price === 0) + .filter(e => isWithin7Days(e.date)) + .sort((a, b) => new Date(a.date) - new Date(b.date)); + + res.json({ + count: result.length, + events: result + }); +}); diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/neighborhoods.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/neighborhoods.ts new file mode 100644 index 0000000000..7accf971dc --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/neighborhoods.ts @@ -0,0 +1,9 @@ + +import { Router } from 'express'; +import { neighborhoods } from '../data/neighborhoods.js'; + +export const neighborhoodsRouter = Router(); + +neighborhoodsRouter.get('/', (_req, res) => { + res.json(neighborhoods); +}); diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/restaurants.ts b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/restaurants.ts new file mode 100644 index 0000000000..ce7d338504 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-02-NYCApp-Solutions/src/routes/restaurants.ts @@ -0,0 +1,9 @@ + +import { Router } from 'express'; +import { restaurants } from '../data/restaurants.js'; + +export const restaurantsRouter = Router(); + +restaurantsRouter.get('/', (_req, res) => { + res.json(restaurants); +}); diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/README.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/README.md new file mode 100644 index 0000000000..3ea11f9b54 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/README.md @@ -0,0 +1,41 @@ +# Challenge 03 — Coach Solution Assets + +Reference data and tooling for the coach. **Not** distributed to students — students use the +VS Code Copilot Chat model picker + Output panel. + +## Contents + +| File | Purpose | +|---|---| +| `harness.py` | Measurement harness: runs N trials of each (model, task), counts pass/fail, averages tokens, with rate-limit backoff | +| `results.md` | The measured answer-key tables referenced in `Solution-03.md` | +| `TODO.md` | Author follow-ups / deferred enhancements | + +## Running the harness + +The harness calls the **GitHub Models inference API** and authenticates with your +`gh auth token`. + +```bash +# default: gpt-4.1 (base) vs gpt-5 (premium), both tasks, n=4 +python harness.py --models "openai/gpt-4.1" "openai/gpt-5" --tasks sort carwash -n 4 --sleep 2 +``` + +Output is a summary table of pass rate + average tokens per (model, task), and a +`results.json` dump. + +### Tasks + +- `sort` — easy (version sort). Both tiers pass; the differentiator is tokens. +- `carwash` — hard (the "walk or drive to the car wash" trap). Base models recommend *walking* + (wrong); premium models recommend *driving* (correct). The `carwash` check marks an answer + correct when the model's leading recommendation is to **drive**. + +### Notes + +- The free GitHub Models tier is rate-limited; the harness backs off on HTTP 429. For large N, + expect it to run slowly. n=4 is sufficient to demonstrate the pattern. +- Premium models (e.g. `gpt-5`) throttle the most. Increase `--sleep` if you see many retries. +- Premium reasoning models also produce highly variable token counts — focus on the pattern. +- **Do not use temperature with `gpt-5`** via this API — it rejects non-default temperature + (HTTP 400). The harness omits the temperature parameter for this reason. diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/TODO.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/TODO.md new file mode 100644 index 0000000000..40187906f9 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/TODO.md @@ -0,0 +1,48 @@ +# Challenge 03 — Author TODO / Deferred + +Coach/author notes. The challenge is complete and runnable. This tracks follow-ups. + +## Done (this revision) +- [x] **Replaced the hard task** from letter-counting → the **car wash** ("walk or drive 40 m + to wash my car"). Letter-counting was a tokenization artifact whose results did **not** + track price (base model could beat premium — we observed Haiku right / Opus wrong). The + car wash tracks capability and is monotonic. See `results.md`. +- [x] **Switched the base/premium pair** to Copilot-selectable models. The old `gpt-4.1-nano` + reliably failed the hard task but is **not selectable in the Copilot model picker**, so + students couldn't use it. New pair: base `gpt-4.1`, premium `gpt-5`. +- [x] **Terminology:** removed "cheap/expensive"; now **base model** / **premium model** + (matches Copilot's included-vs-premium-request billing language). +- [x] **Added depth:** Part 2 cost-of-retries reasoning ("how many base attempts to exceed one + premium call?" + capability-ceiling point); Part 4 "classify your own tasks"; a + "why this matters for coding" paragraph; a token-variance note. +- [x] **Output-constraint section** rebuilt around a verbose task (palindrome function, + 190→66 tokens on gpt-4.1) — the old version-sort example showed no reduction. + +## Open +- [ ] **Token-measurement alignment (hack-wide).** The student docs point to + `View → Output → "GitHub Copilot Chat"` / usage view for token counts, per Challenge 00's + setup. Verify that Copilot Chat actually surfaces a clean per-request token number in the + target VS Code/Copilot version. If it does **not**, this is a **Challenge 00** problem + affecting every challenge — fix it there (or standardize on a token source) rather than + only in Challenge 03. Coach harness + GitHub Models playground are reliable fallback. +- [ ] **Advanced section (optional).** Add an "Advanced Challenges" block: try a mid-tier / + reasoning model on the hard task (where's the price/capability knee?); build a two-model + "base drafts → premium reviews" workflow and measure total cost. +- [ ] **Scope decision — reasoning levels + Auto mode.** The *original* Challenge-03 included + comparing reasoning-effort levels and explaining when **Auto mode** beats manual + selection. Decide whether to re-add as optional/Advanced. Needs owner sign-off. + +## Validated data (basis for the answer key) +Measured via GitHub Models API (`harness.py`), single-shot, no retries, n=4: + +| Model | Easy (version sort) | Hard (car wash) | +|---|---|---| +| gpt-4.1 (base) | ✅ 4/4, ~106 tokens | ❌ 0/4 — says "walk", ~166 tokens | +| gpt-5 (premium) | ✅ 4/4, ~293 tokens | ✅ 4/4 — says "drive", ~1000 tokens | + +Output constraint (gpt-4.1, palindrome function): ~190 → ~66 tokens = **−65%**, identical code. +Constraint showed **no** effect on the terse sort task and on gpt-5 (reasoning tokens dominate). + +Rejected: letter-count/decimal/"strawberry" (tokenization artifacts — don't track price); +`ministral-3b` (fails easy task too); `gpt-4.1-nano` (not in Copilot picker); coin-change +greedy trap (modern small models pass it). diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/harness.py b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/harness.py new file mode 100644 index 0000000000..4e08f491f1 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/harness.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +Statistical model-comparison harness for WTH Challenge 03. + +Runs N trials of each (model, task) pair, counts pass/fail, averages tokens. +Handles 429 rate limits with exponential backoff (a rate-limited call is +retried, NOT counted as a task failure). +""" +import argparse +import json +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request + +ENDPOINT = "https://models.github.ai/inference/chat/completions" + +# ---- Tasks ----------------------------------------------------------------- +SORT_PROMPT = ("Sort these software version numbers from oldest to newest and " + "list them in order separated by commas: 1.9.0, 1.10.0, 1.2.0, " + "1.11.0, 1.9.5") +CARWASH_PROMPT = ("The car wash is 40 meters from my home. I want to wash my " + "car. Should I walk or drive there?") + + +def check_sort(ans: str) -> bool: + versions = re.findall(r"\d+\.\d+\.\d+", ans) + return versions == ["1.2.0", "1.9.0", "1.9.5", "1.10.0", "1.11.0"] + + +def check_carwash(ans: str) -> bool: + """Correct iff the model recommends DRIVING (the car must be at the wash). + + Heuristic: look at the opening recommendation. Premium models lead with + 'Drive...'; base models lead with 'Walk...' / 'Walking...'. We treat the + first mention of walk/drive as the recommendation. + """ + low = ans.lower() + di = low.find("driv") + wi = low.find("walk") + if di == -1: + return False + if wi == -1: + return True + return di < wi + + +TASKS = { + "sort": {"prompt": SORT_PROMPT, "check": check_sort}, + "carwash": {"prompt": CARWASH_PROMPT, "check": check_carwash}, +} + + +def get_token() -> str: + return subprocess.check_output(["gh", "auth", "token"], text=True).strip() + + +def call(token: str, model: str, prompt: str, max_retries: int = 6): + """Returns (answer, total_tokens) or ('ERROR:...', 0). Retries on 429.""" + body = json.dumps( + {"model": model, "messages": [{"role": "user", "content": prompt}]} + ).encode() + for attempt in range(max_retries): + req = urllib.request.Request( + ENDPOINT, data=body, + headers={"Authorization": f"Bearer {token}", + "Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req) as resp: + d = json.load(resp) + return (d["choices"][0]["message"]["content"].strip(), + d["usage"]["total_tokens"]) + except urllib.error.HTTPError as e: + if e.code == 429: + wait = min(60, 2 ** attempt * 5) + time.sleep(wait) + continue + return (f"ERROR:{e.code}", 0) + except Exception as e: # noqa: BLE001 + return (f"ERROR:{e}", 0) + return ("ERROR:429-exhausted", 0) + + +def run(token: str, model: str, task_key: str, n: int, sleep: float): + task = TASKS[task_key] + passes, toks, errors = 0, [], 0 + for i in range(n): + ans, t = call(token, model, task["prompt"]) + if ans.startswith("ERROR:"): + errors += 1 + else: + if task["check"](ans): + passes += 1 + toks.append(t) + done = i + 1 + if done % 10 == 0 or done == n: + print(f" {model} / {task_key}: {done}/{n} " + f"(pass={passes} err={errors})", flush=True) + time.sleep(sleep) + avg = sum(toks) / len(toks) if toks else 0 + valid = n - errors + rate = (passes / valid * 100) if valid else 0 + return {"model": model, "task": task_key, "n": n, "valid": valid, + "passes": passes, "errors": errors, "pass_rate": rate, + "avg_tokens": avg} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--models", nargs="+", required=True) + ap.add_argument("--tasks", nargs="+", default=["sort", "carwash"]) + ap.add_argument("-n", type=int, default=50) + ap.add_argument("--sleep", type=float, default=1.0) + args = ap.parse_args() + + token = get_token() + results = [] + for model in args.models: + for task_key in args.tasks: + print(f" -> {model} / {task_key} (n={args.n})", flush=True) + results.append(run(token, model, task_key, args.n, args.sleep)) + + print("\n" + "=" * 84) + print(f"{'MODEL':<34}{'TASK':<8}{'PASS':>8}{'/N':>6}{'RATE':>8}{'AVG TOK':>10}{'ERR':>6}") + print("-" * 84) + for r in results: + print(f"{r['model']:<34}{r['task']:<8}{r['passes']:>8}{r['valid']:>6}" + f"{r['pass_rate']:>7.0f}%{r['avg_tokens']:>10.0f}{r['errors']:>6}") + print("=" * 84) + with open("results.json", "w") as f: + json.dump(results, f, indent=2) + + +if __name__ == "__main__": + main() diff --git a/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/results.md b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/results.md new file mode 100644 index 0000000000..86d6454322 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Coach/Solutions/Challenge-03-ModelSelection-Solutions/results.md @@ -0,0 +1,74 @@ +# Challenge 03 — Measured Results (Answer Key) + +Source: GitHub Models inference API. Method: `harness.py`, **n=4 trials per (model, task)**, +single-shot (no retries). Exact token counts vary by model version; the **pattern is stable.** + +Models used as the demonstration pair: +- **Base:** `openai/gpt-4.1` (included tier, no premium multiplier — selectable in Copilot Chat) +- **Premium:** `openai/gpt-5` (frontier/reasoning tier) + +## Summary + +| Model | Task | Pass rate | Avg total tokens | +|---|---|---|---| +| `openai/gpt-4.1` (base) | sort (easy) | **4/4 — 100%** | ~106 | +| `openai/gpt-4.1` (base) | car wash (hard) | **0/4 — 0%** (answers "walk") | ~166 | +| `openai/gpt-5` (premium) | sort (easy) | **4/4 — 100%** | ~293 | +| `openai/gpt-5` (premium) | car wash (hard) | **4/4 — 100%** (answers "drive") | ~1000 | + +## Interpretation + +- **Easy task (version sort):** both models 100% correct. The base model produced the identical + answer using **~2.8× fewer tokens** (106 vs 293). Paying for the premium model here buys + nothing → **use the base model.** +- **Hard task (the car wash):** the base model recommended **walking** every single time — + fixating on "40 meters" and missing that the car has to be *at* the wash. The premium model + reasoned about the goal and answered **drive** every time. Retrying the base model can't fix + this — it's a reasoning ceiling → **pay for capability.** + +## Monotonicity check (why the car wash, not letter-counting) + +We require a task where success tracks **capability** (and therefore price). The car wash does; +tokenization "gotchas" do not. Measured examples: + +| Task | gpt-4o-mini (base) | gpt-4.1 (base) | gpt-5 (premium) | Tracks capability? | +|---|---|---|---|---| +| Car wash (walk/drive) | ❌ walk | ❌ walk | ✅ drive | ✅ yes — monotonic | +| Letter count ("l" in "parallel lullaby") | mixed | ✅ 6 | ✅ 6 | ❌ no — base passes | +| Letter count (Claude, observed in Copilot) | Haiku ✅ | — | Opus ❌ | ❌ no — **inverts** | + +The letter-count row is why we dropped it: a base model (Haiku) was correct while a premium +model (Opus) was wrong — the opposite of the lesson. Letter-counting, decimal comparison, and +"strawberry"-style tasks are tokenization artifacts and must not be used here. + +## Output constraint (optional section) + +Constraints reduce **output** tokens, so they show best on a **verbose-by-default** task, on a +non-reasoning (base) model. Measured on `openai/gpt-4.1`: + +| Prompt | Avg total tokens | +|---|---| +| "Write a Python function that checks whether a string is a palindrome." | ~190 | +| same + "Output only the code. No explanation, no markdown fences." | ~66 | + +→ **~65% fewer tokens** for the identical function. + +Note: the constraint did **not** help on the already-terse version-sort task (~103 → ~104 on +gpt-4.1), and did not reduce `gpt-5`'s totals (reasoning/"thinking" tokens dominate and aren't +constrained). Pick a verbose task to demonstrate the lever. + +## Models / tasks we evaluated and rejected + +| Candidate | Why rejected | +|---|---| +| `mistral-ai/ministral-3b` as the base pick | Too weak — failed the **easy** task too, breaking the "base wins Part 1" story | +| `gpt-4.1-nano` as the base pick | Reliably fails the hard task, **but is not selectable in the Copilot model picker** — students can't use it | +| Letter-count / decimal / "strawberry" tasks | Tokenization artifacts — results don't track price (base can beat premium); see monotonicity table | +| Coin-change "greedy trap" | Modern small models pass it — no longer discriminates | + +## Reproduce + +```bash +# easy + hard, base vs premium, n=4 +python harness.py --models "openai/gpt-4.1" "openai/gpt-5" --tasks sort carwash -n 4 --sleep 2 +``` diff --git a/076-GitHubCopilotCostOptimization/README.md b/076-GitHubCopilotCostOptimization/README.md new file mode 100644 index 0000000000..73f26ccf6d --- /dev/null +++ b/076-GitHubCopilotCostOptimization/README.md @@ -0,0 +1,54 @@ +# What The Hack - GitHub Copilot Cost Optimization + +## Introduction + +GitHub Copilot transforms developer productivity, but with usage-based billing (UBB), every token counts. This What The Hack teaches engineering teams how to maximize code quality while minimizing AI credit spend through smart engineering practices. + +You'll learn to control the three cost levers: what enters the context (instructions, attachments, MCP tools), what the model produces (model selection, output constraints), and how efficiently you use the session (cache management, context window hygiene, deterministic feedback loops). By the end, you'll understand that most Copilot cost optimization isn't about restricting usage, it's about engineering better inputs and workflows that help the agent succeed faster. + +## Learning Objectives + +In this hack, you will learn practical techniques to optimize GitHub Copilot usage costs: + +- Reduce persistent instruction overhead through scoped, path-specific configuration files +- Choose appropriate models and apply output constraints to control generation costs +- Manage MCP tool sprawl and understand cache invalidation triggers +- Apply proactive context window management to maintain quality and control costs +- Leverage deterministic feedback loops (tests, linters, type checks) as token controls +- Measure and compare credit spend across different coding workflows + +## Challenges + +- Challenge 00: **[Prerequisites](Student/Challenge-00.md)** + - Set up the hack environment +- Challenge 01: **[Spec-Driven Development](Student/Challenge-01.md)** + - Demonstrate that deterministic controls (tests, specs) reduce trial-and-error token burn +- Challenge 02: **[Context Engineering](Student/Challenge-02.md)** + - Audit and restructure global instructions into scoped files, and replace broad references with pinned attachments +- Challenge 03: **[Model Selection & Output Constraints](Student/Challenge-03.md)** + - Compare credit costs across models and reasoning levels, and apply output constraints to reduce token spend +- Challenge 04: **[Session Configuration (Tools + Cache)](Student/Challenge-04.md)** + - Prune unused MCP tools and understand cache invalidation patterns +- Challenge 05: **[Context Window Management](Student/Challenge-05.md)** + - Use /compact proactively to manage growing context windows and avoid context rot +- Challenge 06: **[Token Golf Competition](Student/Challenge-06.md)** + - Complete a coding task with the lowest credit spend while meeting all acceptance criteria + +## Prerequisites + +- GitHub Copilot subscription +- GitHub Copilot Fundamentals +- Visual Studio Code OR IDE of choice with GitHub Copilot extension installed + +## Repository Contents + +- `Student/Resources/` - Starter codebase with intentionally sub-optimal Copilot configuration for teams to optimize +- `Coach/Solutions/` - Reference implementations and measurement data for each challenge + +## Contributors + +- Andy Huang +- Andressa Siqueira +- Anastasia Nefedova +- Anahita Afshari +- Devanshi Thakar diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-00.md b/076-GitHubCopilotCostOptimization/Student/Challenge-00.md new file mode 100644 index 0000000000..9b10f20f94 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-00.md @@ -0,0 +1,55 @@ +# Challenge 00 - Prerequisites & Baseline + +**[Home](../README.md)** - [Next Challenge >](./Challenge-01.md) + +## Introduction + +Thank you for participating in the GitHub Copilot Cost Optimization What The Hack. Before you begin optimizing, you need to set up your environment to measure GitHub Copilot usage costs accurately. This challenge establishes your baseline—the "before" measurement that all subsequent challenges will compare against. + +Understanding where credits go is essential to optimization. In this challenge, you'll confirm that usage-based billing (UBB) is active, learn how to read token breakdowns in VS Code, and complete a baseline coding task to capture your starting credit spend. + +## Description + +Your coach will provide you with a Resources.zip file that contains a starter codebase for this hack. The codebase is intentionally configured with sub-optimal GitHub Copilot settings—you'll be optimizing these in later challenges. + +Please complete the following setup steps: + +1. Extract the Resources.zip file to your local workstation +2. Open Visual Studio Code and install/update the GitHub Copilot extension to the latest version +3. Verify your GitHub Copilot subscription has usage-based billing (UBB) enabled by checking your billing settings +4. Enable token usage visibility in VS Code: + - Open the Output panel (View > Output) + - Select "GitHub Copilot Chat" from the dropdown + - Verify you can see token counts and credit usage per interaction +5. Confirm you can access your GitHub Copilot usage dashboard to view cumulative credit spend +6. Open the starter codebase in VS Code and familiarize yourself with its structure + +### Baseline Measurement + +Once your environment is configured, complete the baseline coding task: + +1. Locate the `baseline-task.md` file in the `/Challenge00/` folder of the Resources.zip +2. Complete the coding task described using GitHub Copilot Chat in your normal working style (don't optimize yet!) +3. Record the total credits consumed for this task from the VS Code Output panel +4. Document your approach: which features you used (inline completions, chat, edits, workspace references) +5. Save your baseline measurements—you'll compare all future challenges against this + +This baseline represents your "before" state. In subsequent challenges, you'll apply optimization techniques and measure the credit reduction compared to this baseline. + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Verify that usage-based billing (UBB) is active on your GitHub Copilot subscription +- Demonstrate that the VS Code Output panel shows token breakdowns for GitHub Copilot interactions +- Show that you can access the GitHub Copilot usage dashboard and view your current credit consumption +- Verify that the starter codebase from Resources.zip is open in VS Code +- Demonstrate that you have completed the baseline coding task and recorded the credit spend +- Show your documented baseline measurements including total credits and approach used + +## Learning Resources + +- [GitHub Copilot Usage-Based Billing Overview](https://docs.github.com/en/copilot/about-github-copilot/what-is-github-copilot#pricing-for-github-copilot) +- [Understanding GitHub Copilot Tokens and Credits](https://docs.github.com/en/copilot) +- [Visual Studio Code Output Panel Documentation](https://code.visualstudio.com/docs/getstarted/userinterface#_output-panel) +- [GitHub Copilot Extension for VS Code](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot) diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-01.md b/076-GitHubCopilotCostOptimization/Student/Challenge-01.md new file mode 100644 index 0000000000..c7176a805d --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-01.md @@ -0,0 +1,268 @@ +# Challenge 01 - Direct Prompt vs. Spec-Driven Development + +[< Previous Challenge](./Challenge-00.md) - **[Home](../README.md)** - [Next Challenge >](./Challenge-02.md) + +## Introduction + +In this challenge, you will build the same application twice with GitHub Copilot: + +1. Using only a direct prompt +2. Using a structured specification in a Markdown file + +The goal is not to prove in advance that one approach is always better. The goal is to observe how the level of detail in the context affects time, credit consumption, the number of interruptions, and the delivered result. + +This activity uses **Spec-Driven Development (SDD)**. The specification will be provided in a single `sdd.md` file, without a constitution, special commands, a separate plan, or a task list. + +> **Key concept:** in SDD, the specification becomes the source of context for implementation. + +## Experiment Objective + +Both approaches must build the same application with the same technology, model, and scope. + +### Application + +A simple notes application built with React and TypeScript. + +Requirements: + +1. Create, read, update, and delete notes (CRUD) +2. Each note must have a required title and optional content +3. Notes must exist only in the application's in-memory state +4. Do not use a backend, database, persistent storage, or automated tests +5. The application must be implemented and run locally + +## Preparation + +To make the comparison more consistent: + +- Use the same computer and environment +- Use the same versions of VS Code and GitHub Copilot +- Use the same model in both runs +- Start each approach in an empty folder and a new conversation +- Do not reuse files generated by one approach in the other +- Start measuring when you send the implementation instruction +- Stop measuring when the application is running and the CRUD operations have been verified manually + +Also record the time and cost required to prepare the specification. They will be analyzed separately from implementation time. + +--- + +## Part 1 - Implementation with a Direct Prompt + +### Step 1 - Create an Empty Folder + +```bash +mkdir notes-app-prompt +cd notes-app-prompt +code . +``` + +### Step 2 - Start a New Conversation + +Select the model chosen for the experiment and send only the prompt below: + +```text +Build a simple notes application using React and TypeScript. + +Requirements: + +1. Create, read, update, and delete notes. +2. Each note must have a required title and optional content. +3. Store notes only in the application's in-memory state. +4. Do not use a backend, database, persistent storage, or automated tests. + +Implement the complete application in local machine. +``` + +### Step 3 - Record the Run + +Record the start and end times, credits consumed, questions asked by Copilot, dependency decisions requested, required interventions, and any errors or rework. + +Answer the questions required to allow the process to continue, but record each interruption. + +### Step 4 - Verify Manually + +Confirm that you can: + +- Create a note with a title and content +- Create a note without content +- Prevent the creation of a note without a title +- View the created notes +- Update a note's title and content +- Delete a note +- Reload the page and observe that the data may be lost + +Keep the project for the final comparison. + +--- + +## Part 2 - Implementation with SDD + +In this part, the same application will be built from a previously prepared specification. + +### Step 1 - Create a New Empty Folder + +```bash +mkdir notes-app-sdd +cd notes-app-sdd +code . +``` + +### Step 2 - Create the Specification File + +In the root of the folder, create a file named `sdd.md` with the following content: + +```markdown +# Specification: Simple Notes Application + +## Objective + +Build a complete notes application using React and TypeScript that runs +on the local machine. The application must allow users to create, read, +update, and delete notes. + +## Technology + +- React +- TypeScript +- Local development environment + +## Note Data Model + +Each note must contain: + +| Field | Type | Required | Description | +|---|---|---|---| +| id | string or number | Yes | Unique identifier generated by the application | +| title | string | Yes | Title of the note | +| content | string | No | Optional content of the note | + +## Functional Requirements + +### REQ-001: Create notes + +The user must be able to create a note with a required title and optional +content. A valid note must be added to the in-memory state and displayed. + +### REQ-002: Read notes + +The application must display the notes created during the current session, +including each note's title and its content when present. + +### REQ-003: Update notes + +The user must be able to update the title and content of an existing note. +The update must replace the existing note without creating a duplicate. + +### REQ-004: Delete notes + +The user must be able to delete an existing note. The deleted note must be +removed from memory and no longer displayed. + +### REQ-005: Required title + +The application must not create or update a note with an empty title. The +content may be empty. + +### REQ-006: In-memory state + +Notes must be managed only in client-side application state. Notes may be +lost when the page is refreshed or the application is restarted. + +## Constraints + +- Use React and TypeScript +- Run on the local machine +- Do not use a backend +- Do not use a database +- Do not use localStorage, IndexedDB, files, or other persistent storage +- Do not include automated tests + +## Definition of Done + +The application is complete when it runs locally and the user can create, +read, update, and delete notes while all constraints above are respected. +``` + +### Step 3 - Start a New Conversation + +Select the same model used in Part 1 and send: + +```text +Read the sdd.md file in this project and implement the complete application +described in it on the local machine. + +Follow the specification exactly. Do not add requirements or functionality +outside the defined scope. +``` + +### Step 4 - Record and Verify + +Record the same measurements as in Part 1 and use exactly the same manual checklist. Do not give SDD an advantage for features that were not included in the specification. + +--- + +## Part 3 - Compare the Results + +Complete the table with data from both runs: + +| Aspect | Direct prompt | SDD with a ready specification | +|---|---:|---:| +| Model used | | | +| Implementation time | | | +| Credits consumed during implementation | | | +| Questions during the run | | | +| User interventions | | | +| Requirements met | | | +| Errors or rework | | | +| Qualitative UX assessment | | | + +### Three Dimensions of Analysis + +1. **Effectiveness:** did both applications meet the same requirements? +2. **Implementation efficiency:** which approach required less time, fewer credits, and fewer interventions after its input was ready? +3. **Full-cycle efficiency:** does the result change when the cost of preparing the specification is included? + +The total SDD time and cost must be calculated as: + +$$ +T_{SDD\ total} = T_{specification} + T_{implementation} +$$ + +$$ +C_{SDD\ total} = C_{specification} + C_{implementation} +$$ + +Do not mix these two perspectives. A ready specification may speed up implementation, but preparing it also has a cost. + + +## Success Criteria + +After completing the challenge, you should be able to: + +- Build the same application with a direct prompt and a Markdown specification +- Keep the technology, model, and scope equivalent in both runs +- Verify the CRUD operations and constraints manually +- Compare time, credits, questions, and interventions +- Separate implementation cost from preparation cost +- Explain why a functional tie can still show differences in efficiency +- Recognize that a single run is an observation, not universal proof + +## Reflection Questions + +1. Did both approaches meet the same functional requirements? +2. Which approach required more decisions during implementation? +3. Did the specification reduce ambiguity, or did it only move decisions to an earlier stage? +4. How much time and how many credits were used to prepare the specification? +5. Does the result change when that cost is included? +6. In which situations could the specification be reused and offset its initial cost? +7. Would a larger application or a later requirements change produce the same result? + +## Experiment Limitations + +- A single run does not control for the natural variation in model responses +- The application is small and may favor the direct prompt over the full cycle +- The visual assessment is subjective without predefined criteria +- Time may be affected by installations, network conditions, and familiarity with the tools +- The results do not demonstrate that SDD is always better or worse + diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-02.md b/076-GitHubCopilotCostOptimization/Student/Challenge-02.md new file mode 100644 index 0000000000..081a3aaa58 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-02.md @@ -0,0 +1,118 @@ +# Challenge 02 - Context Engineering (NYC App) + +[< Previous Challenge](./Challenge-01.md) - **[Home](../README.md)** - [Next Challenge >](./Challenge-03.md) + +## Introduction + +In this challenge, you will optimize Copilot context using a real code task in the NYC starter app under Resources/Challenge-02-NYCApp. + +The app intentionally includes an oversized global `.github/copilot-instructions.md` file and multiple routes (`events`, `restaurants`, `neighborhoods`) so you can compare broad context (`#Codebase`) against precise, task-focused attachments. + +Every token sent to GitHub Copilot has a cost. Context that persists across interactions, such as global instruction files or broad workspace references, can increase recurring spend by adding tokens that are not needed for the current task. + +You will audit the over-stuffed global instructions file, restructure it into scoped path-specific files, and replace broad references with pinned attachments to reduce the base cost of each Copilot interaction while maintaining output quality. + +## Description + +Use the project in: + +`Resources/Challenge-02-NYCApp` + +### Part 1 - Scope Instructions + +1. Audit `.github/copilot-instructions.md` and identify what is: + - universally useful + - specific to events + - irrelevant for this task (for example, restaurants/neighborhoods/frontend/testing rules) +2. Shrink the global file to only truly global guidance. +3. Create scoped instruction files under `.github/instructions/` for rules that should apply only to specific paths/tasks. For example, create an `api.instructions.md` file and an `events.instructions.md` file. +4. Keep the events-specific logic discoverable for work in `src/routes/events.ts`. +5. Create a reusable Copilot skill under `.github/skills/free-events-endpoint/SKILL.md` that captures the pattern for implementing a free events endpoint. This skill should only load when explicitly invoked, making it a conditional cost rather than a recurring one. + +### Part 2 - Implement the Baseline Route Task + +Implement `GET /events/free-this-week` in `src/routes/events.ts` with this behavior: + +- include events where `price === 0` +- include only events within the next 7 days +- sort ascending by date +- return JSON in the shape `{ count, events }` + +Important: + +- modify `src/routes/events.ts` +- do not modify `src/routes/restaurants.ts` +- do not modify `src/routes/neighborhoods.ts` + +### Part 3 - Attachment Precision and Token Usage Check + +In the GitHub Copilot Chat interface, run this exact query twice and compare results: + +"What data or features does this application support?" + +Run it twice and record the token usage each time: + +1. Without scoped files (broad context, such as `#Codebase`) +2. With scoped files pinned (only task-relevant files, for example: `src/routes/events.ts`, `src/data/events.ts`, and relevant instruction files) + +Capture both token counts from the Copilot Chat usage details/output, compare answer quality, and calculate the difference. + +For each run, record context window stats in this format: + +- Used tokens / Total context window tokens +- Context window utilization percentage + +Example format: + +- Run A (broad): 8,200 / 32,000 (25.6%) +- Run B (scoped): 2,100 / 32,000 (6.6%) + +Then compute: + +- Used-token delta (A - B) +- Utilization delta in percentage points + +Use this table to record your measurements: + +| Run | Context Strategy | Query | Used Tokens | Total Context Window Tokens | Utilization % | Notes | +|-----|------------------|-------|-------------|-----------------------------|---------------|-------| +| A | Broad (`#Codebase` or equivalent) | What data or features does this application support? | | | | | +| B | Scoped pinned files | What data or features does this application support? | | | | | + +| Comparison Metric | Value | +|-------------------|-------| +| Used-token delta (A - B) | | +| Utilization delta percentage points (A - B) | | + + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Show that `.github/copilot-instructions.md` is significantly smaller than the original +- Show scoped instruction files created in `.github/instructions/` +- Show a reusable skill created at `.github/skills/free-events-endpoint/SKILL.md` +- Demonstrate a correct `/events/free-this-week` implementation in `src/routes/events.ts` +- Demonstrate that `src/routes/restaurants.ts` and `src/routes/neighborhoods.ts` remained unchanged +- Show token comparison between `#Codebase` and pinned attachments for the same task +- Show Part 3 token and context-window comparison for the query "What data or features does this application support?" in Copilot Chat: + - without scoped files + - with scoped pinned files + - with documented context window stats (used / total) + - with documented utilization percentage for both runs + - with documented token and utilization deltas +- Show measurable credit reduction while preserving output quality + +## Learning Resources + +- [GitHub Copilot Instructions Documentation](https://docs.github.com/en/copilot/customizing-copilot/adding-custom-instructions-for-github-copilot) +- [GitHub Copilot Prompt Files and Attachments](https://docs.github.com/en/copilot/using-github-copilot/asking-github-copilot-questions-in-your-ide#reference-files-in-your-prompts) +- [GitHub Copilot Skills Overview](https://docs.github.com/en/copilot/using-github-copilot/using-github-copilot-skills) + +## Tips + +- Persistent instructions are recurring costs—they load on every call +- Skills are conditional costs—they only load when invoked +- Attachments are per-call costs—you control exactly what's included +- The engineering target: keep always-on global instructions under 1000 tokens +- Scoped instructions should be discoverable: if a rule only applies to `/api`, put the instruction file in `/api/.github/` diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-03.md b/076-GitHubCopilotCostOptimization/Student/Challenge-03.md new file mode 100644 index 0000000000..e40e1e9780 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-03.md @@ -0,0 +1,188 @@ +# Challenge 03 - Model Selection & Output Constraints + +[< Previous Challenge](./Challenge-02.md) - **[Home](../README.md)** - [Next Challenge >](./Challenge-04.md) + +## Introduction + +Not every task needs the most powerful model. Under usage-based billing, cost is **not** just the model's per-token price. It is: + +> **Total cost = token rate × tokens × attempts** + +That last term — *attempts* — is the one people forget. A **base model** that answers a simple task correctly in a few tokens is a bargain. But set that same base model on work that needs real judgment and it will hand you a **plausible-looking answer that's shallow or flat-out wrong** — and you won't always catch it right away. Now you are paying again (and again) to notice the gap and redo the work. The model that looked lower-cost per request just became the costliest one. + +In this challenge you will run the **same prompt against a base model and a premium model** on two very different kinds of task. First an **easy, checkable** task, then an **open-ended** one with no answer key. Every run, you measure the **outcome** — *did it get it right, or how good is it?* — and the **token cost**. From that data you will build the skill this challenge is really about: deciding *which model to use for which job.* + +> **Key Concept:** Lower cost per request is not the same as lower cost per *result*. + +### Why this matters for coding + +This challenge uses one quick checkable prompt and one open-ended planning prompt so you feel both sides of the trade-off. But the lesson is the same one you face every time you code with Copilot: an agent constantly **reasons inside a task** — choosing logic, weighing options, judging whether its own output is any good. That is exactly where a weaker model comes up short — sometimes *confidently wrong*, sometimes just shallow. Model selection governs the quality of that reasoning, not just whether output comes out. Pick the model that fits the *thinking* the task requires. + +## Description + +You will run **two tasks**, each against **two models**: + +- A **base model** — included with your subscription, no premium-request multiplier (e.g. GPT-4.1, GPT-4o). Fast and economical. +- A **premium model** — a frontier / reasoning model billed as a premium request with a cost multiplier (e.g. GPT-5, Claude Sonnet, an o-series model). + +> Use whatever base-tier and premium-tier models your organization exposes in the GitHub Copilot Chat **model picker**. The exact names don't matter — the *pattern* does. The picker shows a multiplier (e.g. `1×`) next to premium models; base models show no multiplier. + +For every run, record the **outcome** (correct / incorrect) and the **token usage** (`View → Output → "GitHub Copilot Chat"`, and/or your Copilot usage view). If your environment doesn't surface an exact per-request token count, you can still see the pattern: the premium model produces far more text. Note *roughly how much more*. + +> **Rule — first shot only:** Run each prompt **once** per model and record what you get. Don't coach, rerun, or rephrase to fish for a better result. The whole point is to see what each model produces *on the first attempt* for each kind of task. + +> **A note on token counts:** Exact token numbers vary run to run — premium reasoning models especially. Focus on the **pattern and the ratio**, not the absolute number. If a result looks odd, run it 2–3 times. + +--- + +## Part 1 — An Easy Task (the base model should win) + +Give **both** models this exact prompt: + +``` +Sort these software version numbers from oldest to newest and list them in +order separated by commas: 1.9.0, 1.10.0, 1.2.0, 1.11.0, 1.9.5 +``` + +✅ Correct answer: `1.2.0, 1.9.0, 1.9.5, 1.10.0, 1.11.0` + +### Record + +| | Base model | Premium model | +|---|---|---| +| Correct? | ? | ? | +| Total tokens | ? | ? | + +### What to notice + +Both models should get this right. Now compare the **token counts**. The premium model often produces far more output — restated reasoning, explanations — for the *exact same correct answer*. On an easy task, that extra spend buys you nothing. + +--- + +## Part 2 — An Open-Ended Task (judge the *quality*, not a right answer) + +Part 1 had a checkable answer. **Most real work doesn't.** When you plan a feature, design a schema, or scope a project with Copilot, there's no answer key — *you* have to judge whether the output is any good. That judgment is the skill this part builds, and it's where model choice matters most. + +Give **both** models this exact prompt: + +``` +I want to build a pickleball app. Produce an implementation plan. +First, ask me any clarifying questions you need. Then identify the +top 5 risks or hidden complexities that could derail this project, +and highlight them in the plan. +``` + +Notice the prompt is **deliberately vague** ("a pickleball app" — to do *what*?). That's on purpose: a strong model should *notice the gap and ask* before it plans. The risk question then forces the model to actually **think**, not just fill in a template. + +### Record & Score + +There's no correct answer, so score each response against this rubric instead: + +| Criterion | Base model | Premium model | +|---|---|---| +| Asked clarifying questions *before* planning? (how many, how sharp) | ? | ? | +| Top-5 risks: generic ("scope creep, budget") or genuinely insightful? | ? | ? | +| Plan specific to *pickleball* — or a generic "any app" template? | ? | ? | +| Would you trust it enough to start building? | ? | ? | +| Total tokens | ? | ? | + +### What to notice + +The base model tends to **charge straight ahead**: it assumes what the app is, produces a plausible-but-generic plan, and lists generic risks (budget, timeline, scope creep). It rarely stops to ask what you're actually building. + +The premium model is more likely to **ask sharp clarifying questions first** (court booking? player matchmaking/ranking? score tracking? league scheduling? who are the users?) and to surface **deeper, domain-specific risks** — double-booking / concurrency on court reservations, ranking fairness, real-time score sync, geolocation privacy, no-show and cancellation handling. Those are exactly the risks that quietly derail a project. + +> **The cost tie-in:** the premium plan costs more tokens up front. But a shallow plan isn't cheap — it's **deferred cost.** Every risk the base model *didn't* flag becomes rework, a rewrite, or a bug you pay for at execution time — more *attempts* to get to something you can actually build. On open-ended, high-leverage work, paying more for *deeper thinking* is usually the economical choice — the mirror image of Part 1, where paying more bought you nothing. + +### Questions to Answer + +1. Which model's plan would you actually trust enough to start building from — and why? +2. Did the extra tokens the premium model spent buy you real value here (unlike Part 1)? What specifically? +3. This task has no "right answer." How did you decide which output was *better*? What made the difference? + +--- + +## Part 3 — Compare Results & Decide + +Combine your measurements into one table: + +| Task | Base: outcome | Base: tokens | Premium: outcome | Premium: tokens | Which model should you use? | +|------|---------------|--------------|------------------|-----------------|-----------------------------| +| Part 1 (easy, checkable) | correct? | ? | correct? | ? | ? | +| Part 2 (open-ended) | judge quality | ? | judge quality | ? | ? | + +### Questions to Answer + +1. On the **easy** task, which model gave the better *value* (correct answer, fewest tokens)? +2. On the **open-ended** task, did the premium model's extra tokens buy you something real — depth, clarifying questions, risks the base model missed? +3. If you ran the **easy** task **10,000 times a day**, which model would you pick, and how much would the wrong choice cost you? +4. How would you decide, *before* running a task, whether it is a "Part 1" (well-defined → base) or a "Part 2" (open-ended, judgment-heavy → premium) task? + +--- + +## Part 4 — Classify Your Own Tasks + +The skill this challenge builds is *prediction*: knowing which tier a task needs **before** you spend anything. + +1. Write down **two real tasks** from your own work — one you think a base model can handle, one you think needs a premium model. +2. For each, predict the tier and **why** (Is it well-defined, or does it need multi-step reasoning / judgment?). +3. Run each task on **both** tiers and check your prediction. Were you right? Note any surprises. + +| Your task | Predicted tier | Actual best tier | Was your prediction right? | +|---|---|---|---| +| 1. | | | | +| 2. | | | | + +--- + +## Optional — Output Constraints + +Output tokens usually cost more than input tokens, and constraining what a model *produces* is a lever you control on **every** model. + +Pick a task where the model is **verbose by default** — for example, asking for a small function: + +``` +Write a Python function that checks whether a string is a palindrome. +``` + +Run it on the base model, then run it again with an output constraint added: + +``` +Write a Python function that checks whether a string is a palindrome. +Output only the code. No explanation, no markdown fences. +``` + +Record the token difference. Constraining output — "code only", "answer only", "explain in under 50 words", a JSON schema — drops the cost while keeping the correct result. + +> Note: constraints reduce **output** tokens. They help most when the model would otherwise pad the answer with explanation. They do **not** turn a shallow answer into a good one (see Part 2), and they do little when the output is already terse or when a reasoning model's hidden "thinking" tokens dominate. + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Run the same prompt against a base and a premium model for both tasks +- Record the outcome (correct/incorrect, or quality score) and token usage for all four runs +- Show that on the easy task the base model was correct **and** used fewer tokens +- On the open-ended planning task, judge each model's *quality* against the rubric (clarifying questions, depth of the top-5 risks, specificity, trust-to-start) and explain which you'd build from +- Classify two of your own real tasks by tier and verify your prediction +- (Optional) Demonstrate a token reduction by adding an output constraint to a verbose task + +## Learning Resources + +- [GitHub Copilot Model Selection Guide](https://docs.github.com/en/copilot) +- [Prompt Engineering: Output Formatting Techniques](https://www.promptingguide.ai/) +- [GitHub Copilot Usage-Based Billing](https://docs.github.com/en/copilot/concepts/billing/usage-based-billing-for-organizations-and-enterprises) + +## Tips + +- Cost = token rate × tokens × **attempts**. A *wrong* answer you have to chase down is the costliest option. +- The base model is great for well-defined, low-ambiguity work — and it is wasteful to *avoid* it there. +- The premium model earns its multiplier on tasks that need real reasoning, where a wrong first answer is costly. +- Output constraints reduce cost on *any* model — ask for the answer only when you don't need narration. +- The best model is not the smartest one — it is the one that returns a *correct* answer at the lowest total cost. + +## Reflection Questions + +1. Describe a real task from your own work that is a "Part 1" task. Which model would you use and why? +2. Describe a real "Part 2" task — open-ended, judgment-heavy. How would you justify the premium multiplier to your team? +3. How could you combine base and premium models in a single workflow to minimize total cost (e.g., base model drafts, premium model reviews)? diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-04.md b/076-GitHubCopilotCostOptimization/Student/Challenge-04.md new file mode 100644 index 0000000000..fc18157d9b --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-04.md @@ -0,0 +1,64 @@ +# Challenge 04 - Session Configuration (Tools + Cache) + +[< Previous Challenge](./Challenge-03.md) - **[Home](../README.md)** - [Next Challenge >](./Challenge-05.md) + +## Introduction + +Every MCP tool and Skill you enable adds token overhead to every agent step. Each tool's name, description, and parameter schema are advertised to the model on every request—even if the tool is never invoked. A session with many unused tools burns credits before you've written a line of code. + +Meanwhile, changing configuration mid-session can trigger cache invalidation, forcing expensive cold restarts where previously cached context must be reprocessed. + +In this challenge, you'll measure the real cost of tool sprawl and observe how session hygiene affects your credit spend. + +## Description + +This challenge has two parts: measuring MCP tool overhead and observing cache invalidation behavior. + +### Part 1: Measure MCP Tool and Skills Overhead + +Use GitHub Copilot to scan/find something in the `microsoft/TypeScript` repository and report back. + +If you don't have MCP servers enabled already, try the GitHub MCP server with all its tools. Complete the task with all tools enabled and record your credit spend. + +Next, identify which MCP tools and skills were actually used. + +Disable the unused tools and skills, repeat the scan, and measure the credit difference. Each unused tool adds token overhead on every agent step because its schema is included in the system prompt. + +### Part 2: Observe Cache Invalidation + +Complete the repository scan twice more under different conditions: + +**Clean Session:** Configure your tools once, then complete the entire scan without making any configuration changes. Record total credits. + +**Messy Session:** Start the scan, but mid-way through deliberately make configuration changes—disable a tool, edit your `.github/copilot-instructions.md` file, or switch to a different reasoning effort level. Finish the scan and record total credits. + +Observe which configuration changes cause measurable credit spikes. These spikes indicate cache invalidation—when GitHub Copilot must reprocess previously cached context. + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Show which MCP tools were enabled at the start using the `/context` command +- Complete the TypeScript repository scan and generate a report with file paths and dependencies +- Identify which tools were actually invoked by examining VS Code Output panel logs +- Demonstrate measurable credit savings from disabling unused tools on the same scan task +- Show comparative credit costs between clean session and messy session approaches +- Explain which configuration changes you observed causing cache invalidation +- Document your findings: tool count before/after, credits before/after, credit difference between clean/messy sessions + +## Learning Resources + +- [Managing Context in GitHub Copilot CLI](https://docs.github.com/en/copilot/concepts/agents/copilot-cli/context-management) +- [Model Context Protocol Tools Specification](https://modelcontextprotocol.io/specification/2025-06-18/server/tools.md) +- [Anthropic Prompt Caching](https://claude.com/blog/prompt-caching) +- [GitHub Copilot Usage-Based Billing](https://docs.github.com/en/copilot/concepts/billing/usage-based-billing-for-organizations-and-enterprises) + +## Tips + +- Use `/context` to see which tools are enabled and how much of your token budget they consume +- The VS Code Output panel (View → Output → "GitHub Copilot") shows actual tool invocations +- Each MCP tool's schema (name, description, parameters) is sent to the model on every request +- System prompt changes (like editing instruction files or toggling tools) can invalidate cached context +- Use `/usage` to see your credit breakdown after each task +- The GitHub MCP server typically includes tools like: search_code, get_file_contents, create_issue, search_users, list_pull_requests +- For this scanning task, you likely only need search_code and get_file_contents diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-05.md b/076-GitHubCopilotCostOptimization/Student/Challenge-05.md new file mode 100644 index 0000000000..37c82ca05d --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-05.md @@ -0,0 +1,80 @@ +# Challenge 05 - Context Window Management + +[< Previous Challenge](./Challenge-04.md) - **[Home](../README.md)** - [Next Challenge >](./Challenge-06.md) + +## Pre-requisites + +- Complete Challenge 02 (Context Engineering) — you will extend the CityScout NYC App from that challenge +- Complete Challenge 04 (Session Configuration: Tools + Cache) + +## Introduction + +Context windows grow with every interaction. As a session accumulates messages, attachments, and generated code, the model has more history to process each turn. + +When context grows too large, quality often drops: missed requirements, repeated questions, inconsistent edits, and higher credit spend. In this challenge, you will compare unmanaged context growth against proactive context management and measure the credit difference. + +## Description + +You will extend the **CityScout NYC App** (from Challenge 02) with four new features, then compare two session strategies side-by-side: one where you let context grow unchecked, and one where you actively manage it. The features deliberately span all three routes and force topic switching—exactly the conditions that grow context fast. + +### The Task: CityScout Feature Sprint + +Using the NYC App you worked with in Challenge 02 (in `Resources/Challenge-02-NYCApp`), implement these four features in order: + +- **Feature 1 – Restaurant Filtering:** Add query-parameter support to `GET /restaurants` so callers can filter by `borough` and/or `cuisine` (e.g., `/restaurants?borough=Brooklyn&cuisine=Mexican`). Return the full list when no filters are provided. +- **Feature 2 – Neighborhood Detail with Restaurants:** Add `GET /neighborhoods/:name` that returns the matching neighborhood object plus an embedded array of restaurants in that neighborhood (a cross-file join between `neighborhoods.ts` and `restaurants.ts` data). +- **Feature 3 – Pagination (cross-cutting):** Add `?page=N&limit=N` support to all three list endpoints (`/events/free-this-week`, `/restaurants`, `/neighborhoods`). Default to page 1, limit 10. Return `{ page, limit, total, data }`. +- **Feature 4 – Input Validation & Error Handling:** Add proper `400 Bad Request` responses for invalid query parameters and `404 Not Found` for missing resources across all routes. + +These four features touch multiple files, require cross-file references, and create natural breakpoints between each feature—ideal for testing context management strategies. We recommend not committing these files to make it easier to revert. + +### Run A – Unmanaged Context Growth + +Implement all four features in a single session without ever using `/compact`: + +- Let context accumulate naturally as you move from feature to feature +- Do not start new sessions between features +- Track credit usage after each feature completes +- Watch for quality degradation signals as context grows: + - Copilot forgets earlier requirements (e.g., pagination format differs between routes) + - Contradictory suggestions (e.g., different error shapes in different routes) + - Repeated clarifying questions you already answered + - Increased rework or manual corrections + +Record a brief evidence note after each feature: what worked, what degraded. + +### Run B – Proactive Context Management + +Implement the same four features, but actively manage context throughout. You choose your own strategy: + +- `/compact` to summarize and continue in the same session +- `/clear` to wipe and restart with a fresh prompt +- Start a new session (`/new`) at any breakpoint you choose + +The goal is to keep context lean and credit spend low. Track total credits across the entire sprint (including any new sessions you start) and compare with Run A. Document which context management actions you took and why. + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Demonstrate at least two concrete quality degradation signals from Run A +- Show a side-by-side credit comparison between Run A and Run B for the same four features +- Verify that your Run B context management strategy reduced total credit spend compared to Run A +- Explain the trade-offs between `/compact`, `/clear`, and starting a new session +- Show a concise evidence log for both runs (context management actions taken, credits per feature, total credits) + +## Learning Resources + +- [Understanding Context Windows and Token Limits](https://www.anthropic.com/index/prompting-long-context) +- [GitHub Copilot Chat Commands Documentation](https://docs.github.com/en/copilot/using-github-copilot/using-github-copilot-chat) +- [Managing Long Conversations with AI Assistants](https://www.promptingguide.ai/) + +## Tips + +- Do not optimize for speed first; optimize for clean credit comparisons between runs +- Keep the four features identical between Run A and Run B so the credit comparison is meaningful +- Record credits after each feature, not just at the end — this shows where costs accelerate +- There is no single "right" strategy for Run B; `/compact`, `/clear`, and `/new` all have trade-offs +- `/compact` preserves some context continuity but costs tokens to summarize +- `/clear` and `/new` give you a cold start — cheap per-turn but you re-establish context from scratch +- The best strategy often mixes approaches depending on how much earlier context the next feature needs diff --git a/076-GitHubCopilotCostOptimization/Student/Challenge-06.md b/076-GitHubCopilotCostOptimization/Student/Challenge-06.md new file mode 100644 index 0000000000..67b24185e8 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Challenge-06.md @@ -0,0 +1,49 @@ +# Challenge 06 - Token Golf Competition 🏌️ + +[< Previous Challenge](./Challenge-05.md) - **[Home](../README.md)** + +## Introduction + +Token golf is the art of achieving high-quality code output with minimal credit spend. In this final challenge, teams compete to complete the same coding task with the lowest credit consumption while meeting all acceptance criteria. This is where you apply everything you've learned: context engineering, model selection, session hygiene, and spec-driven development. + +Your coach has already completed this task and established a "par" score—the credit cost of a well-optimized solution. Can you beat par? + +## Description + +All teams will receive the same coding task with defined acceptance criteria from your coach. The task specification (found in the `/Challenge06/` folder of Resources.zip) includes functional requirements, tests that must pass, required documentation, and code quality standards. + +### Competition Rules + +Complete all functional requirements, ensure provided tests pass, include required documentation, and meet code quality standards. Document your complete approach in a "scorecard" showing total credits consumed, prompt sequence used, models selected, and optimization techniques applied from Challenges 1-5. + +Track every GitHub Copilot interaction throughout the challenge: inline completions, chat, edits, workspace references, and cumulative total credits. The team with the lowest total credit score that meets all acceptance criteria wins (tie-breaker: fastest completion time). + +### Strategy + +Consider applying techniques from previous challenges: scoped instructions, efficient model selection, pruned MCP tools, strategic `/compact` usage, and spec-first development with tests written before implementation. + +## Success Criteria + +To complete this challenge successfully, you should be able to: + +- Verify that all functional requirements from the task specification are implemented +- Demonstrate that all provided tests pass +- Show that required documentation exists and meets standards +- Verify that code meets quality standards (no linting errors, proper types) +- Show your documented scorecard with total credits consumed and techniques applied +- Demonstrate that your solution meets or beats the coach's "par" score (bonus achievement!) + +## Learning Resources + +- [GitHub Copilot Best Practices](https://docs.github.com/en/copilot/using-github-copilot/best-practices-for-using-github-copilot) +- [Prompt Engineering Guide](https://www.promptingguide.ai/) +- [Test-Driven Development Fundamentals](https://martinfowler.com/bliki/TestDrivenDevelopment.html) + +## Tips + +- Start with the spec, not the code—what are the acceptance criteria? +- Write failing tests first to establish your deterministic feedback loop +- Choose your model based on the complexity of each subtask +- Track credits as you go, not just at the end +- If you get stuck in a high-cost loop, reset: use `/clear` and try a different approach +- Remember that the fastest path isn't always the cheapest—an extra minute planning can save 20 credits executing \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json b/076-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..bc0dfb3948 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +{ + "name": "076-GitHubCopilotCostOptimization", + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + "hostRequirements": { + "cpus": 2 + }, + "features": { + "ghcr.io/devcontainers/features/python:1": { "version": "3.12" } + }, + "customizations": { + "vscode": { + "extensions": [ + "github.copilot", + "github.copilot-chat", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode" + ] + }, + "codespaces": { + "openFiles": [ + "README.md" + ] + } + }, + "postCreateCommand": "curl -LsSf https://astral.sh/uv/install.sh | sh && ~/.local/bin/uv tool install specify-cli --from git+https://github.com/github/spec-kit.git", + + "forwardPorts": [3000], + "portsAttributes": { + "3000": { + "label": "CityScout NYC App", + "onAutoForward": "notify" + } + } +} diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/.gitkeep b/076-GitHubCopilotCostOptimization/Student/Resources/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/.github/copilot-instructions.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/.github/copilot-instructions.md new file mode 100644 index 0000000000..e2469755d4 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/.github/copilot-instructions.md @@ -0,0 +1,147 @@ + +# Copilot Instructions (INTENTIONALLY BLOATED) + +## General Engineering Rules +- Always write clean, maintainable, scalable, and production-ready code +- Always validate all inputs at every layer of the application +- Always handle errors gracefully with meaningful error messages +- Always use descriptive variable and function naming +- Always include comments explaining business logic +- Avoid deeply nested logic whenever possible +- Prefer modular and reusable components +- Ensure all code is optimized for performance +- Avoid unnecessary computations and loops +- Minimize memory usage and API calls + +## API Design Rules +- Use RESTful conventions for all endpoints +- Ensure consistent naming for endpoints (plural nouns) +- Always return JSON responses +- Always include HTTP status codes +- Validate all query parameters and request bodies +- Use middleware for validation and error handling +- Return responses in the format: + { + "count": number, + "data": [] + } +- Never expose internal implementation details in responses + +## Frontend Rules (NOT RELEVANT BUT INCLUDED) +- Use React functional components only +- Prefer hooks over class components +- Use useEffect and useState appropriately +- Ensure accessibility (ARIA labels, keyboard navigation) +- Optimize rendering performance +- Use responsive design principles +- Avoid unnecessary re-renders +- Ensure clean UI/UX design patterns + +## Data Modeling Rules +- Normalize JSON structures +- Avoid duplicate data +- Use consistent field naming across all datasets +- Prefer arrays of objects for collections +- Always use ISO date formats (YYYY-MM-DD) +- Avoid null values when possible +- Ensure backward compatibility of schema changes + +## Restaurant Domain Rules (NOT NEEDED FOR EVENTS) +- Restaurants must include: + - name + - borough + - cuisine + - rating + - priceLevel +- Include vegetarian-friendly indicators +- Include neighborhood information +- Include vibe descriptions (e.g., cozy, lively, romantic) +- Include wait time estimates for popular locations + +## Neighborhood Domain Rules (NOT NEEDED FOR EVENTS) +- Neighborhoods must include: + - borough + - vibe + - transit options + - average rent tier + - bestFor categories +- Include walkability information +- Include proximity to parks, restaurants, and nightlife +- Include safety considerations +- Use engaging and descriptive language + +## Events Domain Rules (ACTUALLY RELEVANT) +- Events must include: + - name + - date + - location + - borough + - price +- Free events must have price === 0 +- Events must be filtered based on date when required +- Events must be sorted by date ascending +- Exclude past events +- Ensure event data is complete and valid + +## Testing Rules (NOT USED IN THIS TASK) +- Write unit tests for business logic +- Write integration tests for endpoints +- Mock dependencies when necessary +- Ensure proper test coverage (at least 80%) +- Test edge cases thoroughly +- Validate error handling paths + +## Logging and Monitoring Rules +- Log all important actions +- Log errors with stack traces (but not in responses) +- Ensure logs are structured +- Include timestamps in logs +- Avoid logging sensitive data + +## Security Rules +- Validate all inputs to prevent injection attacks +- Sanitize user input +- Avoid exposing secrets or keys +- Use environment variables for configuration +- Follow principle of least privilege + +## NYC Content & Tone Rules +- Keep examples fun and NYC-specific +- Reference locations like Central Park, Brooklyn, SoHo, Williamsburg +- Use relatable, local descriptions +- Avoid generic or bland wording +- Make the application feel like it’s built for a real NYC resident + +## Performance Optimization Rules +- Optimize loops and filtering logic +- Avoid redundant computations +- Cache results where possible +- Minimize memory footprint +- Reduce unnecessary API calls + +--- + +# ⚠️ THIS FILE IS INTENTIONALLY OVERLOADED + +It includes: +- ✅ relevant rules (events) +- ❌ irrelevant domains (restaurants, neighborhoods) +- ❌ unrelated layers (frontend, security, logging, testing) + +👉 Students should REDUCE this drastically. +`` \ No newline at end of file diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/README.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/README.md new file mode 100644 index 0000000000..87ba71fab4 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/README.md @@ -0,0 +1,39 @@ + +# CityScout NYC - Challenge 02 Starter (FULL VERSION) + +## ✅ WHERE YOU SHOULD MAKE CHANGES + +### MODIFY THESE: +1. `.github/copilot-instructions.md` +2. Create `.github/instructions/` +3. Update `src/routes/events.ts` + +### DO NOT MODIFY: +- `restaurants.ts` +- `neighborhoods.ts` + +--- + +## Baseline Task + +Implement `/events/free-this-week`: +- price === 0 +- within next 7 days +- sorted by date +- return { count, events } + +--- + +## Demo Goal + +Compare: + +❌ #Codebase (loads EVERYTHING) +✅ @events + specific files (loads ONLY events) + +Now repo includes: +✅ Events +✅ Restaurants +✅ Neighborhoods + +So you can visibly see context differences diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/events.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/events.ts new file mode 100644 index 0000000000..44040e6bd4 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/events.ts @@ -0,0 +1,7 @@ + +export const events = [ + { name: 'Central Park Yoga', date: '2026-06-30', price: 0 }, + { name: 'Brooklyn Flea Market Walk', date: '2026-07-01', price: 0 }, + { name: 'Rooftop Movie Night', date: '2026-07-02', price: 15 }, + { name: 'Jazz in Washington Square Park', date: '2026-07-10', price: 0 } +]; diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/neighborhoods.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/neighborhoods.ts new file mode 100644 index 0000000000..a1b5a81feb --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/neighborhoods.ts @@ -0,0 +1,5 @@ + +export const neighborhoods = [ + { name: 'West Village', borough: 'Manhattan' }, + { name: 'Williamsburg', borough: 'Brooklyn' } +]; diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/restaurants.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/restaurants.ts new file mode 100644 index 0000000000..65c0434049 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/data/restaurants.ts @@ -0,0 +1,5 @@ + +export const restaurants = [ + { name: 'West Village Pasta', borough: 'Manhattan', cuisine: 'Italian' }, + { name: 'Williamsburg Tacos', borough: 'Brooklyn', cuisine: 'Mexican' } +]; diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/events.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/events.ts new file mode 100644 index 0000000000..9395b7efbd --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/events.ts @@ -0,0 +1,14 @@ + +import { Router } from 'express'; +import { events } from '../data/events.js'; + +export const eventsRouter = Router(); + +// ✅ STUDENTS MODIFY THIS + +eventsRouter.get('/free-this-week', (req, res) => { + return res.json({ + count: events.length, + events + }); +}); diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/neighborhoods.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/neighborhoods.ts new file mode 100644 index 0000000000..7accf971dc --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/neighborhoods.ts @@ -0,0 +1,9 @@ + +import { Router } from 'express'; +import { neighborhoods } from '../data/neighborhoods.js'; + +export const neighborhoodsRouter = Router(); + +neighborhoodsRouter.get('/', (_req, res) => { + res.json(neighborhoods); +}); diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/restaurants.ts b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/restaurants.ts new file mode 100644 index 0000000000..ce7d338504 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-02-NYCApp/src/routes/restaurants.ts @@ -0,0 +1,9 @@ + +import { Router } from 'express'; +import { restaurants } from '../data/restaurants.js'; + +export const restaurantsRouter = Router(); + +restaurantsRouter.get('/', (_req, res) => { + res.json(restaurants); +}); diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/README.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/README.md new file mode 100644 index 0000000000..88140c3f47 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/README.md @@ -0,0 +1,33 @@ +# Challenge 03 — Model Selection — Resources + +This folder contains everything you need for Challenge 03. There is **no codebase to run** — +the challenge is about sending two short prompts to two different models and measuring what +comes back. + +## What's here + +| File | Purpose | +|---|---| +| `prompts.md` | The two exact prompts (easy + hard) and their correct answers | +| `measurements-template.md` | Blank tables to record outcome + tokens for each run | +| `output-constraints.md` | Cheat sheet of output constraints to reduce token cost | + +## How to run it (VS Code + GitHub Copilot Chat) + +1. Open GitHub Copilot Chat in VS Code. +2. Use the **model picker** (bottom of the chat box) to select a **base-tier** model — one with + *no* premium-request multiplier (e.g. GPT-4.1 / GPT-4o). +3. Paste a prompt from `prompts.md` exactly as written. Record the answer and the token + usage (`View → Output → "GitHub Copilot Chat"`, and/or your Copilot usage view). +4. Switch the model picker to a **premium-tier** model — one that shows a cost multiplier + (e.g. GPT-5 / Claude Sonnet / an o-series model) — and run the **same** prompt. Record again. +5. Repeat for both the easy and the hard prompt. +6. Fill in `measurements-template.md` and answer the Part 3 questions, then do the Part 4 + "classify your own tasks" exercise. + +> **Rule:** Do not retry or coach the base model when it gets the hard task wrong. A wrong +> first answer is a valid — and important — result. You'll reason about the cost of retrying +> in Part 2 instead. + +> **If your picker shows only one tier:** ask your coach which models map to "base" and +> "premium" for your org, or use the coach's reference numbers as the demonstration. diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/measurements-template.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/measurements-template.md new file mode 100644 index 0000000000..5bf1dccafa --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/measurements-template.md @@ -0,0 +1,77 @@ +# Challenge 03 — Measurement Template + +Fill in the tables as you run each prompt. Get token counts from +`View → Output → "GitHub Copilot Chat"` and/or your Copilot usage view. If exact per-request +counts aren't shown, record *roughly* how much more text the premium model produced. + +> Token counts vary run to run (premium reasoning models especially). Focus on the **pattern / +> ratio**, not the absolute number. + +Record the **base-tier** and **premium-tier** model names you actually used: + +- Base model: `____________________` +- Premium model: `____________________` + +--- + +## Part 1 — Easy task (version sort) + +| | Base model | Premium model | +|---|---|---| +| Answer given | | | +| Correct? (Y/N) | | | +| Total tokens | | | + +**Observation:** _Were both correct? Which used fewer tokens for the same answer?_ + +--- + +## Part 2 — Hard task (the car wash) + +| | Base model | Premium model | +|---|---|---| +| Answer (walk / drive) | | | +| Correct? (Y/N) | | | +| Total tokens | | | + +**Observation:** _Did the base model say "walk"? Each attempt was low-cost — but was it useful?_ + +### Cost-of-retries reasoning + +- Base model cost per attempt: `__________` | Premium model cost (one call): `__________` +- **How many base-model attempts equal one premium call?** `__________` +- On this task, would *any* number of base-model attempts reliably get the right answer? `__________` + +--- + +## Optional — Output constraint (verbose task, base model) + +| | Base model (no constraint) | Base model (code/answer only) | +|---|---|---| +| Total tokens | | | +| Token reduction | — | | + +--- + +## Part 3 — Decision table + +| Task | Base correct? | Base tokens | Premium correct? | Premium tokens | Use which model? | +|------|---------------|-------------|------------------|----------------|------------------| +| Part 1 (easy) | | | | | | +| Part 2 (hard) | | | | | | + +### Answers + +1. Easy task — better value (correct + fewest tokens)? → +2. Hard task — what did the base model's wrong answer really cost? → +3. At 10,000 runs/day, how does your choice differ for Part 1 vs Part 2? → +4. How do you decide *up front* whether a task is "easy → base" or "hard → premium"? → + +--- + +## Part 4 — Classify your own tasks + +| Your task | Predicted tier | Actual best tier | Prediction right? | +|---|---|---|---| +| 1. | | | | +| 2. | | | | diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/output-constraints.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/output-constraints.md new file mode 100644 index 0000000000..6407b380c4 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/output-constraints.md @@ -0,0 +1,38 @@ +# Output Constraints — Cheat Sheet + +Output tokens usually cost more than input tokens. Constraining what the model *produces* is +a cost lever you control on **every** model. Use these patterns when you don't need narration. + +| Goal | Add to your prompt | Effect | +|---|---|---| +| Answer only | "Answer with only the number/value, nothing else." | Removes restated reasoning + preamble | +| Code only | "Output only the code. No explanation, no markdown fences." | Cuts the "Here's how it works..." paragraphs | +| Cap verbosity | "Explain in 50 words or fewer." | Hard ceiling on output length | +| Structured output | "Respond as JSON matching: `{ \"result\": number }`" | Prevents rambling; easy to parse | +| No restatement | "Do not repeat the question or summarize the task." | Removes echo of the input | +| Bounded list | "List at most 3 items." | Stops over-generation | + +## Why it matters + +A model asked a question it would normally *explain* will spend most of its tokens narrating an +answer you already trust. The constraint keeps the correctness and drops the cost. The effect is +biggest on **verbose-by-default** tasks (e.g. "write a function") — there the model otherwise +wraps the answer in explanation and markdown. + +## Try it + +Take a verbose task on the base model and re-run it with an output constraint appended: + +``` +Write a Python function that checks whether a string is a palindrome. +Output only the code. No explanation, no markdown fences. +``` + +Compare total tokens — the code is identical, the cost is much lower. + +> Notes: +> - Constraints reduce **output** tokens. They do little when the output is already terse +> (e.g. a one-line answer) or when a reasoning model's hidden "thinking" tokens dominate. +> - Constraints do **not** fix a model that is *wrong* (see Part 2). For a task the base model +> can't reason through, a constraint just gives you a wrong answer faster — model selection, +> not output shape, is the fix there. diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/prompts.md b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/prompts.md new file mode 100644 index 0000000000..75aa0afcb8 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge-03-ModelSelection/prompts.md @@ -0,0 +1,56 @@ +# Challenge 03 — Prompts + +Run each prompt **verbatim**. Do not add context, attachments, or instructions — keeping the +input identical across models is what makes the token comparison fair. + +--- + +## Part 1 — Easy task (version sort) + +``` +Sort these software version numbers from oldest to newest and list them in +order separated by commas: 1.9.0, 1.10.0, 1.2.0, 1.11.0, 1.9.5 +``` + +✅ **Correct answer:** `1.2.0, 1.9.0, 1.9.5, 1.10.0, 1.11.0` + +A wrong answer usually sorts the versions as text (so `1.10.0` and `1.11.0` land before +`1.9.0`). Modern models — base or premium — almost always get this right. The differentiator +here is **token cost**, not correctness. + +--- + +## Part 2 — Hard task (the car wash) + +``` +The car wash is 40 meters from my home. I want to wash my car. +Should I walk or drive there? +``` + +✅ **Correct answer:** **Drive.** You can't wash a car that isn't there — the car has to be +*at* the car wash. The "40 meters" is a deliberate distraction; the base model tends to fixate +on the short distance and tell you to **walk** (leaving your car, and the whole point, at home). + +A premium model reasons about the actual goal — *the car must be present to be washed* — and +answers **drive**. The wrong answer is obvious to any human in one second: that is exactly why +this task works. + +--- + +## Optional — Output constraint variant (verbose coding task) + +Pick a task the model is verbose about by default, then constrain the output: + +``` +Write a Python function that checks whether a string is a palindrome. +``` + +then + +``` +Write a Python function that checks whether a string is a palindrome. +Output only the code. No explanation, no markdown fences. +``` + +Run both on the **base model** and compare total tokens. The code is the same; the constrained +run drops the surrounding explanation, cutting cost. diff --git a/076-GitHubCopilotCostOptimization/Student/Resources/Challenge06/data/sales.csv b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge06/data/sales.csv new file mode 100644 index 0000000000..def8a67565 --- /dev/null +++ b/076-GitHubCopilotCostOptimization/Student/Resources/Challenge06/data/sales.csv @@ -0,0 +1,16 @@ +date,category,amount,note +2026-01-03,Electronics,199.99, +2026-01-05,Books,12.50, +2026-01-06,books,9.99,"reorder, please" +2026-01-07,Books ,0.10, +2026-01-08, BOOKS,0.20, +2026-01-09,Electronics,50.00, +2026-01-10,"Toys, Games",24.99, +2026-01-11,Food,5.25, +2026-01-12,food, 5.75, +2026-01-13, FOOD ,4.00,"bulk order +spanning two lines" +2026-01-14,,13.00, +2026-01-15,Electronics,N/A, +2026-01-16,,7.00, +2026-01-17,Books,0.20, diff --git a/xxx-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json b/xxx-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..bc0dfb3948 --- /dev/null +++ b/xxx-GitHubCopilotCostOptimization/Student/Resources/.devcontainer/devcontainer.json @@ -0,0 +1,34 @@ +{ + "name": "076-GitHubCopilotCostOptimization", + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + "hostRequirements": { + "cpus": 2 + }, + "features": { + "ghcr.io/devcontainers/features/python:1": { "version": "3.12" } + }, + "customizations": { + "vscode": { + "extensions": [ + "github.copilot", + "github.copilot-chat", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode" + ] + }, + "codespaces": { + "openFiles": [ + "README.md" + ] + } + }, + "postCreateCommand": "curl -LsSf https://astral.sh/uv/install.sh | sh && ~/.local/bin/uv tool install specify-cli --from git+https://github.com/github/spec-kit.git", + + "forwardPorts": [3000], + "portsAttributes": { + "3000": { + "label": "CityScout NYC App", + "onAutoForward": "notify" + } + } +}