azure-sdk-automation · Oct 28, 2026 · Oct 29, 2026 · Oct 29, 2026 · Oct 30, 2026
@@ -37,6 +37,7 @@ extends:
       - 'workflows/live/*.eval.yaml'
     # Run each shard under AzureCLI@2 so the real MCP's DevOps calls are authenticated.
     UseAzSdkAuthentication: true
+    azureSubscription: opensource-api-connection
     failOnFailedTests: true
     # Live scenarios are end-to-end, so give each shard more headroom than the report-only tiers.
     shardTimeoutInMinutes: 45

@@ -6,10 +6,31 @@ parameters:
   - name: vallyRoot
     # Required (no default) so each repo passes its own eval root explicitly.
     type: string
+  - name: matrixStageName
+    # Stage containing the generate_eval_matrix job.
+    type: string
+    default: Prepare
+  - name: artifactPrefix
+    # Literal prefix after 'eval-result-' / 'eval-debug-'; include any separator (e.g. 'forced-').
+    # Match Summary's prefix and keep final artifact names unique across this build's tracks.
+    type: string
+    default: ''
   - name: evalInvokeTemplate
     # Swap point: runs `vally eval` for the shard (the common template is auth-conditional).
     type: string
     default: /eng/common/pipelines/templates/steps/eval-invoke.yml
+  - name: vallyArguments
+    # --tag does not filter the supplied matrix; each planned eval must retain matching stimuli.
+    type: object
+    default: []
+  - name: failOnFailedTests
+    # Match Summary's gate; multi-run gating is not currently supported.
+    type: boolean
+    default: false
+  - name: azureSubscription
+    # Caller-owned service connection; required by the common authenticated invoke template.
+    type: string
+    default: ''
   - name: UseAzSdkAuthentication
     # Forwarded to the invoke template: true runs the shard under AzureCLI@2.
     type: boolean
@@ -44,7 +65,7 @@ jobs:
     workspace:
       clean: all
     strategy:
-      matrix: $[ stageDependencies.Prepare.generate_eval_matrix.outputs['detect.matrix'] ]
+      matrix: $[ stageDependencies.${{ parameters.matrixStageName }}.generate_eval_matrix.outputs['detect.matrix'] ]
       maxParallel: 10
     templateContext:
       outputs:
@@ -53,12 +74,12 @@ jobs:
         # The Summary job keeps only the highest attempt per shard so retries supersede cleanly.
         - output: pipelineArtifact
           displayName: 'Publish $(shardName) raw results, JUnit and completeness'
-          artifactName: 'eval-result-$(shardName)-$(System.JobAttempt)'
+          artifactName: 'eval-result-${{ parameters.artifactPrefix }}$(shardName)-$(System.JobAttempt)'
           targetPath: $(Build.ArtifactStagingDirectory)/eval-result
           condition: always()
         - output: pipelineArtifact
           displayName: 'Publish $(shardName) transcripts (on failure)'
-          artifactName: 'eval-debug-$(shardName)-$(System.JobAttempt)'
+          artifactName: 'eval-debug-${{ parameters.artifactPrefix }}$(shardName)-$(System.JobAttempt)'
           targetPath: $(Build.ArtifactStagingDirectory)/debug
           condition: failed()
     steps:
@@ -71,6 +92,14 @@ jobs:
         inputs:
           version: 24.x
 
+      - ${{ if eq(parameters.evalInvokeTemplate, '/eng/common/pipelines/templates/steps/eval-invoke.yml') }}:
+        - script: |
+            node --experimental-strip-types "$(Build.SourcesDirectory)/eng/common/scripts/eval/invoke-eval-shard.ts" --validate-only
+          displayName: 'Validate eval run configuration'
+          env:
+            VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}
+            EVAL_FAIL_ON_FAILED_TESTS: ${{ parameters.failOnFailedTests }}
+
       # 1ES Official image blocks registry.npmjs.org; restore via the azure-sdk npm mirror.
       - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml
         parameters:
@@ -113,12 +142,16 @@ jobs:
 
       - template: /eng/common/pipelines/templates/steps/login-to-copilot.yml
 
-      # Runs the eval via the invoke template (auth-conditional in the common default).
       - template: ${{ parameters.evalInvokeTemplate }}
         parameters:
           vallyRoot: ${{ parameters.vallyRoot }}
           UseAzSdkAuthentication: ${{ parameters.UseAzSdkAuthentication }}
           threshold: ${{ parameters.threshold }}
+          # Existing custom templates keep their contract unless opting into the argument list.
+          ${{ if eq(parameters.evalInvokeTemplate, '/eng/common/pipelines/templates/steps/eval-invoke.yml') }}:
+            azureSubscription: ${{ parameters.azureSubscription }}
+          ${{ if ne(length(parameters.vallyArguments), 0) }}:
+            vallyArguments: ${{ parameters.vallyArguments }}
 
       # Optional repo-specific teardown after the eval.
       - ${{ if parameters.postEvalTemplate }}:

@@ -2,7 +2,19 @@
 # gates on it when failOnFailedTests is true.
 
 parameters:
+  - name: matrixStageName
+    type: string
+    default: Prepare
+  - name: evalStageName
+    type: string
+    default: Eval
+  - name: artifactPrefix
+    # Literal prefix, including any separator (e.g. 'forced-'); must match the shard job.
+    # Callers are responsible for unique final artifact names across tracks in one build.
+    type: string
+    default: ''
   - name: failOnFailedTests
+    # Only single-run evaluations support this gate; multi-run observations must leave it off.
     type: boolean
     default: false
   - name: createDashboardBundle
@@ -22,15 +34,22 @@ jobs:
     workspace:
       clean: all
     variables:
-      EvalExpectedMatrix: $[ stageDependencies.Prepare.generate_eval_matrix.outputs['detect.matrix'] ]
+      EvalExpectedMatrix: $[ stageDependencies.${{ parameters.matrixStageName }}.generate_eval_matrix.outputs['detect.matrix'] ]
     templateContext:
       outputs:
         - output: pipelineArtifact
           displayName: 'Retain selected build summary and saved bundle'
-          artifactName: 'eval-summary-$(System.JobAttempt)'
+          artifactName: 'eval-summary-${{ parameters.artifactPrefix }}$(System.JobAttempt)'
           targetPath: $(Build.ArtifactStagingDirectory)/eval-summary
           condition: always()
     steps:
+      # Dashboard archives identify a single build-wide summary, not separate tracks.
+      - ${{ if and(ne(parameters.artifactPrefix, ''), or(parameters.createDashboardBundle, parameters.publishDashboardResults)) }}:
+        - script: |
+            echo "##vso[task.logissue type=error]Prefixed eval tracks support artifact-only summaries, not dashboard bundles or publication."
+            exit 1
+          displayName: 'Reject ambiguous multi-track dashboard identity'
+
       - checkout: self
         clean: true
         fetchDepth: 1
@@ -40,15 +59,15 @@ jobs:
         inputs:
           version: 24.x
 
-      # Allowlist result data only, excluding debug, MCP, SDL and signing artifacts.
+      # Download only this track's result data; an empty prefix retains all eval-result-* artifacts.
       - task: DownloadPipelineArtifact@2
         displayName: 'Download all shard results'
         inputs:
           buildType: 'current'
           itemPattern: |
-            eval-result-*/results.jsonl
-            eval-result-*/shard.json
-            eval-result-*/**/*.junit.xml
+            eval-result-${{ parameters.artifactPrefix }}*/results.jsonl
+            eval-result-${{ parameters.artifactPrefix }}*/shard.json
+            eval-result-${{ parameters.artifactPrefix }}*/**/*.junit.xml
           targetPath: $(Pipeline.Workspace)/eval-results
 
       # Only attempt verification needs the current build token; rendering uses saved evidence.
@@ -60,6 +79,7 @@ jobs:
         env:
           SYSTEM_ACCESSTOKEN: $(System.AccessToken)
           EVAL_EXPECTED_MATRIX: $(EvalExpectedMatrix)
+          EVAL_STAGE_NAME: ${{ parameters.evalStageName }}
 
       - script: |
           node --experimental-strip-types "$(Build.SourcesDirectory)/eng/common/scripts/eval/build-eval-summary.ts" \
@@ -71,6 +91,7 @@ jobs:
         condition: always()
         env:
           EVAL_EXPECTED_MATRIX: $(EvalExpectedMatrix)
+          EVAL_ARTIFACT_PREFIX: ${{ parameters.artifactPrefix }}
 
       - ${{ if or(parameters.createDashboardBundle, parameters.publishDashboardResults) }}:
         - template: ../steps/eval-publish-results.yml

@@ -43,6 +43,18 @@ parameters:
     # When true, the invoke step runs under AzureCLI@2 (live MCP inherits the service connection).
     type: boolean
     default: false
+  - name: azureSubscription
+    # Caller-owned service connection; unused when authentication is disabled.
+    type: string
+    default: ''
+  - name: vallyArguments
+    # --tag does not filter Prepare's matrix; align evalGlobs so every eval has matching stimuli.
+    type: object
+    default: []
+  - name: artifactPrefix
+    # Literal artifact prefix, including any separator (e.g. 'forced-').
+    type: string
+    default: ''
   - name: threshold
     # Pass-rate gate forwarded to `vally eval --threshold` and the shard verdict. Repos may override.
     type: number
@@ -100,8 +112,12 @@ extends:
           - template: /eng/common/pipelines/templates/jobs/eval-shard.yml
             parameters:
               vallyRoot: ${{ parameters.vallyRoot }}
+              artifactPrefix: ${{ parameters.artifactPrefix }}
               evalInvokeTemplate: ${{ parameters.evalInvokeTemplate }}
               UseAzSdkAuthentication: ${{ parameters.UseAzSdkAuthentication }}
+              azureSubscription: ${{ parameters.azureSubscription }}
+              vallyArguments: ${{ parameters.vallyArguments }}
+              failOnFailedTests: ${{ parameters.failOnFailedTests }}
               threshold: ${{ parameters.threshold }}
               preEvalTemplate: ${{ parameters.preEvalTemplate }}
               postEvalTemplate: ${{ parameters.postEvalTemplate }}
@@ -114,6 +130,7 @@ extends:
         jobs:
           - template: /eng/common/pipelines/templates/jobs/eval-summarize.yml
             parameters:
+              artifactPrefix: ${{ parameters.artifactPrefix }}
               failOnFailedTests: ${{ parameters.failOnFailedTests }}
               createDashboardBundle: ${{ parameters.createDashboardBundle }}
               publishDashboardResults: ${{ parameters.publishDashboardResults }}
@@ -14,11 +14,24 @@ parameters:
     type: number
     default: 0.8
   - name: azureSubscription
+    # Caller-owned service connection; required only for authenticated runs.
     type: string
-    default: opensource-api-connection
+    default: ''
+  - name: vallyArguments
+    # Supplementary --tag/--runs/--workers/--timeout argument pairs, not shell text.
+    # --tag does not filter the matrix; callers must keep every planned eval non-empty.
+    # JSON-escape dollar signs below so agent macros cannot rewrite argument data.
+    type: object
+    default: []
 
 steps:
-  - ${{ if parameters.UseAzSdkAuthentication }}:
+  - ${{ if and(parameters.UseAzSdkAuthentication, eq(trim(parameters.azureSubscription), '')) }}:
+    - script: |
+        echo "##vso[task.logissue type=error]azureSubscription must be provided when UseAzSdkAuthentication is true."
+        exit 1
+      displayName: 'Validate authenticated eval service connection'
+
+  - ${{ elseif parameters.UseAzSdkAuthentication }}:
     - task: AzureCLI@2
       displayName: 'vally eval $(shardName) (authenticated)'
       inputs:
@@ -34,6 +47,7 @@ steps:
             --threshold ${{ parameters.threshold }}
       env:
         COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)
+        VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}
 
   - ${{ else }}:
     - script: |
@@ -46,3 +60,4 @@ steps:
       workingDirectory: ${{ parameters.vallyRoot }}
       env:
         COPILOT_GITHUB_TOKEN: $(COPILOT_GITHUB_TOKEN)
+        VALLY_ARGUMENTS_JSON: ${{ replace(convertToJson(parameters.vallyArguments), '$', '\u0024') }}
@@ -13,6 +13,85 @@ templates.
   its own pnpm package/lock for **fflate only** and CLI/PowerShell tests. Shard evaluator
   restores do not install it; publication adds no Node Azure SDK dependencies.
 
+## Extending a shard job
+
+The shared `eng/common/pipelines/templates/jobs/eval-shard.yml` owns checkout,
+Node/Vally installation, MCP staging, git fixture initialization and Copilot login.
+Its defaults preserve the standard Prepare/Eval/Summary pipeline behavior.
+Repo-local adapters can customize it without copying that infrastructure:
+
+| Parameter | Contract |
+| --- | --- |
+| `matrixStageName` | Stage containing `generate_eval_matrix`; defaults to `Prepare`. The eval stage must depend on it. |
+| `artifactPrefix` | Optional literal prefix, e.g. `forced-`. Common adds no separator: `eval-result-forced-<shard>-<attempt>` / `eval-debug-forced-<shard>-<attempt>` / `eval-summary-forced-<attempt>`. Empty preserves existing names. Available through the stage and jobs; shard and summary prefixes must match. |
+| `preEvalTemplate` / `postEvalTemplate` | Existing parameter-free setup/teardown step templates. |
+| `evalInvokeTemplate` | Existing invocation template accepting `vallyRoot`, `UseAzSdkAuthentication` and `threshold`. |
+| `azureSubscription` | Caller-owned Azure service connection, forwarded by the stage/job to the common invoke template. Defaults to empty. Required when `UseAzSdkAuthentication: true`; unauthenticated runs need no connection. |
+| `vallyArguments` | Optional array of supplementary Vally option/value pairs. Strings are required except that trial/worker counts also accept numbers. Available through the stage, shard job and common invoke template. Defaults to `[]`. |
+
+Supported supplementary options are `--tag`, `--runs`, `--workers` and `--timeout`.
+For example, `['--tag', 'mode=forced', '--runs', '3']` selects a local benchmark
+configuration without replacing the common runner. Trial/worker counts must be
+positive integer strings or safe integer numbers. Azure template expansion can
+serialize numeric parameters as JSON numbers even when using `format`; the runner
+validates and converts only `--runs` / `--workers` numeric values to strings.
+All other options and values must be strings. Both authentication branches forward the array as JSON in
+`VALLY_ARGUMENTS_JSON`; the runner passes literal arguments to the pinned CLI using
+Node directly, without a shell or whitespace splitting of supplementary values.
+The template JSON-escapes dollar signs before agent macro expansion; the runner's
+JSON decoding restores them. Tags such as `build=$(Build.BuildId)` stay literal,
+even when the pipeline variable exists.
+Unsupported flags (including aliases and `--flag=value`) are rejected. Eval
+selection, output paths, JUnit, plugins and verdict settings remain framework-owned.
+Legacy custom invoke templates retain their existing parameter contract when the
+array is empty. A custom template opting into a non-empty array must declare its
+own `vallyArguments` parameter; otherwise template compilation rejects it.
+
+`--runs` greater than 1 must not be combined with `failOnFailedTests: true`.
+Use `failOnFailedTests: false` (the default) for multi-run observations;
+explicit `--runs 1` may still be used with the gate enabled. Unsupported
+combinations fail the common shard's configuration check before installation or
+evaluation. When composing shard and summary jobs directly, pass the same
+`failOnFailedTests` value to both. This check covers explicit `--runs` values in
+`vallyArguments` only; Summary does not infer run counts from results. Callers must
+ensure that repetition configured in `.vally.yaml` or custom invoke templates is
+not combined with the enabled gate.
+
+`--tag` filters stimuli during execution only; it does not filter Prepare's matrix.
+The caller must align `evalGlobs` (or its custom matrix) with the tags so every
+planned eval retains at least one matching stimulus. For example, `--tag area=github`
+alone does not remove other areas from the matrix: also restrict the selected files
+to the intended GitHub evals. Recheck this alignment when adding evals or changing tags.
+Zero-match evals are not treated as successful skips; a zero-trial shard still fails
+the verdict/completeness checks even with `failOnFailedTests: false`.
+
+Use the existing pre/post-eval and MCP setup hooks for repository-specific work.
+Authenticated callers must explicitly configure `azureSubscription`; common has
+no deployment-specific connection fallback. An empty or whitespace-only value
+fails with a configuration error without creating an `AzureCLI@2` task.
+Do not replace authentication, result collection or retry/completeness checks.
+Supplementary reports must not rewrite JUnit thresholds or raw results.
+
+For multiple tracks in one build, use an artifact prefix and configure
+the common summary's `matrixStageName`, `evalStageName` and `artifactPrefix`
+to match the shard job. Each Summary downloads only result-file patterns matching
+its prefix; an empty prefix retains the build-wide `eval-result-*` download.
+Each Summary stage must depend on both its Prepare and
+Eval stages. Timeline attempts are checked against that exact Eval stage, and
+only that track's latest verified attempts are selected. Prefixed tracks
+support artifact-only summaries: dashboard bundles/publication are rejected
+because the archive identity currently represents one build-wide Summary.
+Prefixes and shard names must start with an ASCII letter, digit or underscore,
+contain only ASCII letters, digits, underscores or hyphens, and be at most 200
+characters; prefixes may also be empty. Include any desired separator
+in the prefix: `forced-` and `forced` are used exactly as supplied. Existing shard
+names, including names with hyphens, remain supported.
+
+The caller must ensure final artifact names are unique across all tracks in the
+build, including unprefixed tracks. Different prefixes alone do not guarantee this:
+`forced-` + `area-foo` and `forced-area-` + `foo` produce the same name.
+Common does not perform cross-track collision detection.
+
 ## Complete build results and direct Blob publishing
 
 Shards now retain the newest invocation's raw JSONL, JUnit and a completion marker
@@ -149,6 +228,9 @@ then `node --experimental-strip-types --test` from this directory.
 These shared Node tests do not need Tools-owned CI files, PowerShell tests or
 the consuming repository's redirect settings. A consumer-copy regression runs
 them without Tools files or Git history, with absent and different redirect templates.
+Template contract checks use bounded text assertions, not a YAML parser, and need
+no evaluator dependencies. They do not evaluate Azure template expressions;
+server-side expansion and service-connection authorization require separate validation.
 
 In Tools, run `Invoke-Pester -Path eng/common-tests/eval/Publish-EvalResults.Tests.ps1 -CI`
 from the repository root for the **PowerShell 7.4+ / Pester 5 / Azure CLI** tests. That Tools-only

@@ -379,7 +379,8 @@ async function main(argv) {
       }
     }
     selection = selectSummaryResults({ resultsRoot: options.resultsRoot,
-      selectedRoot: options.selectedRoot, expectedShards, jobAttempts });
+      selectedRoot: options.selectedRoot, expectedShards, jobAttempts,
+      artifactPrefix: process.env.EVAL_ARTIFACT_PREFIX ?? "" });
     if (!matrixValid) {
       selection.status.push({ shard: "Prepare", attempt: null, complete: false,
         reason: "The expected shard matrix is unavailable or invalid; check the Prepare stage." });
@@ -413,7 +414,9 @@ async function main(argv) {
 
   if (process.env.TF_BUILD) {
     console.log(`##vso[task.uploadsummary]${path.resolve(options.outputPath)}`);
-    if (selection) console.log(`##vso[task.setvariable variable=EvalSummaryComplete]${selection.complete}`);
+  }
+  if (process.env.TF_BUILD && selection) {
+    console.log(`##vso[task.setvariable variable=EvalSummaryComplete]${selection.complete}`);
   }
 
   // Completed evaluation failures remain governed by failOnFailedTests. Missing result