Add SDK-owned blob purge service contract and shared task integration

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add `Microsoft.DurableTask.LargePayloadPurge.Abstractions` with shared fetch/report and service-setting interfaces, and expose the existing blob purge tasks for host integration ([#805](https://github.com/microsoft/durabletask-dotnet/pull/805)).
 - Preserve the original orchestration version when restarting through the orchestration-service client shim ([#463](https://github.com/microsoft/durabletask-dotnet/issues/463)).
 - Support configurable scheduler token audiences and government defaults ([#806](https://github.com/microsoft/durabletask-dotnet/pull/806))
 

Microsoft.DurableTask.sln

@@ -149,6 +149,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Extensions", "Extensions",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureBlobPayloads.Tests", "test\Extensions\AzureBlobPayloads.Tests\AzureBlobPayloads.Tests.csproj", "{3E509481-3CCC-4006-BCB2-9E8FA7C275F1}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LargePayloadPurge.Abstractions", "src\LargePayloadPurge.Abstractions\LargePayloadPurge.Abstractions.csproj", "{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LargePayloadPurge.Abstractions.Tests", "test\LargePayloadPurge.Abstractions.Tests\LargePayloadPurge.Abstractions.Tests.csproj", "{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -855,6 +859,30 @@ Global
 		{3E509481-3CCC-4006-BCB2-9E8FA7C275F1}.Release|x64.Build.0 = Release|Any CPU
 		{3E509481-3CCC-4006-BCB2-9E8FA7C275F1}.Release|x86.ActiveCfg = Release|Any CPU
 		{3E509481-3CCC-4006-BCB2-9E8FA7C275F1}.Release|x86.Build.0 = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|x64.Build.0 = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Debug|x86.Build.0 = Debug|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|x64.ActiveCfg = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|x64.Build.0 = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|x86.ActiveCfg = Release|Any CPU
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28}.Release|x86.Build.0 = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|x64.Build.0 = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Debug|x86.Build.0 = Debug|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|x64.ActiveCfg = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|x64.Build.0 = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|x86.ActiveCfg = Release|Any CPU
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -929,6 +957,8 @@ Global
 		{3B8F957E-7773-4C0C-ACD7-91A1591D9312} = {5B448FF6-EC42-491D-A22E-1DC8B618E6D5}
 		{00205C88-F000-28F2-A910-C6FA00E065EE} = {E5637F81-2FB9-4CD7-900D-455363B142A7}
 		{3E509481-3CCC-4006-BCB2-9E8FA7C275F1} = {00205C88-F000-28F2-A910-C6FA00E065EE}
+		{CF0A6A55-1DEC-4027-8B8B-4B0DEE3C7A28} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B}
+		{BE1D641F-1F96-483E-A1A9-0F8A64E9BAFE} = {E5637F81-2FB9-4CD7-900D-455363B142A7}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71}

README.md

@@ -200,6 +200,58 @@ For runnable DTS emulator examples that demonstrate versioning, see the [WorkerV
 
 The [on-demand sandbox activities sample](samples/on-demand-sandbox/README.md) shows how to declare selected activities for Durable Task Scheduler (DTS)-managed on-demand sandbox execution and build the remote worker container image separately from the declarer app.
 
+### Blob auto-purge infrastructure integration
+
+The optional service contract is maintained in
+[`Microsoft.DurableTask.LargePayloadPurge.Abstractions`](src/LargePayloadPurge.Abstractions/README.md).
+It defines `Microsoft.DurableTask.LargePayloadPurge.Abstractions.IOrchestrationServiceLargePayloadPurgeClient` and
+`Microsoft.DurableTask.LargePayloadPurge.Abstractions.ILargePayloadPurgeClient`, using the canonical SDK Client models
+without duplicating them. This package follows the repository's shared release version, the same as Client
+and most other `Microsoft.DurableTask.*` packages, rather than versioning independently, and is not BCL-only:
+its Client dependency transitively depends on SDK Abstractions and Durable Task Core. The Azure Blob
+implementation depends on these contracts; the contracts do not depend on Blob storage, gRPC or worker
+implementations. The service capability inherits the shared fetch/report interface and adds only the setting
+operation.
+
+`Microsoft.DurableTask.Extensions.AzureBlobPayloads` exposes reusable orchestration and activity
+implementations: `BlobPurgeJobOrchestrator`, `GetLargePayloadTombstonesActivity`, `DeleteExternalBlobActivity`,
+and `ReportLargePayloadPurgeResultsActivity`. Preserve their exact task names, empty version, input/output
+types, and retry/event/continue-as-new behavior. Keep the tasks registered even when auto-purge is disabled
+so existing work can finish. The existing client API manages the reserved per-task-hub orchestration instance
+and its configuration.
+
+The companion **.NET isolated Durable Functions** integration uses the optional
+`Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureBlobPayloads` package. It supplies four ordinary
+`[Function]` methods that delegate to the shared tasks, plus worker-side payload-store configuration.
+The base Functions worker extension does not carry these function definitions. Referencing only this shared
+SDK package does not register Functions or enable auto-purge.
+
+The Functions Worker SDK discovers the compiled methods during the normal build and generates their metadata
+and invocation paths. The functions use ordinary trigger and `DurableClient` bindings in the isolated worker,
+passing the bound `TaskOrchestrationContext` and a `TaskActivityContext` with the invoking orchestration's
+instance ID to the shared tasks. Normal serialization and failure propagation preserve structured failure
+details and unprocessed events across continue-as-new.
+
+Construct the fetch/report activities with an `ILargePayloadPurgeClient` and their typed loggers, and the
+delete activity with the worker's configured `PayloadStore` and logger. Reuse `BlobPayloadStore` with
+`LargePayloadStorageOptions` for storage access rather than copying its ownership checks or deletion policy.
+Deletion runs in the language worker; purge RPCs do not carry storage credentials. The narrow purge client
+must honor UTC deadlines and preserve opaque tombstone tokens; the activities classify fetch/report gRPC failures.
+
+The Functions integration routes setting, fetch, and report operations through the bound client's local
+host endpoint to the provider's authenticated transport for the **same task hub**. The existing
+`LargePayloadPurge` gRPC service is separate from `TaskHubSidecarService`. The Functions client wrapper can
+forward the setting through the existing infrastructure `ILargePayloadAutoPurgeClient` interface so the
+original `client.SetLargePayloadAutoPurgeAsync(enabled, batchSize, cancellationToken)` extension retains
+ownership of bootstrap behavior. The integration owns client/store lifetimes, hub binding, and reconnection;
+this SDK surface alone does not supply Functions metadata or the local-host bridge.
+
+Enabling explicitly writes the setting, starts the reserved instance with live-status deduplication and an
+empty version, verifies its identity and Running status, then sends `SetBatchSize`. Disabling **only** writes
+the setting and ignores batch size. Calling neither leaves the setting untouched. These steps are not
+transactional; failures propagate without rollback. Existing standalone gRPC client and worker behavior
+is unchanged.
+
 ### Token audiences and Azure Government
 
 `DurableTaskSchedulerClientOptions.ResourceId` and `DurableTaskSchedulerWorkerOptions.ResourceId`

doc/release_process.md

@@ -10,6 +10,71 @@ This repo publishes multiple NuGet packages. Most share a single version defined
 
 We follow an approach of releasing everything together, even if a package has no changes — unless we intentionally hold a package back.
 
+`LargePayloadPurge.Abstractions` versions with the shared repository-wide `VersionPrefix`/`VersionSuffix` in
+`eng/targets/Release.props`, the same as Client, Abstractions, and most other `Microsoft.DurableTask.*`
+packages (a few, such as `Generators`, deliberately keep their own independent version); it has no
+package-local version override. It uses source references to the SDK Client models, so publish it in the
+same release as the Client and Abstractions packages that contain those models; released Client `1.26.0`
+predates them and is not sufficient on its own until a release that includes them is published. The package
+uses the repository's MIT license and SDK strong-name key.
+Its package and assembly name is `Microsoft.DurableTask.LargePayloadPurge.Abstractions`, covered by the
+standard `Microsoft.DurableTask.*.dll` signing pattern. The existing source traversal, SBOM inclusion,
+NuGet signing and per-package approval-gated publication steps apply.
+
+Contract publication also waits for successful Client and Abstractions publication. If either prerequisite
+fails, including a duplicate-version upload failure, or is skipped or canceled, contract publication is
+skipped rather than treating that result as success.
+
+`Microsoft.DurableTask.Extensions.AzureBlobPayloads` depends on the contract package at every target
+framework, so its publication job also waits for successful contract publication (in addition to approval).
+If contract publication fails for any reason above, including a duplicate-version upload failure, or is
+skipped or canceled, Blob publication is skipped rather than treating that result as success. This extends
+the same fail-closed chain: approval, then Client and Abstractions, then the contract, then Blob. A
+prerequisite that deliberately skips — for example because its exact version is already published — still
+skips the dependent job; this is not a general-purpose publication-idempotency mechanism, just the minimum
+ordering these two packages require. Other packages retain their own independent publication jobs gated
+only on approval.
+
+Keep its `RELEASENOTES.md`: `eng/targets/Release.targets` reads it into NuGet package metadata, whereas the
+root `CHANGELOG.md` retains repository release history. The shared target also appends a link using the
+package's own version, which is now the shared repository-wide version, so the **Prepare Release**
+workflow's existing `release/vX.Y.Z` branch and `vX.Y.Z` tag cover this package automatically — no separate
+version or tag step is needed for it.
+
+### Cross-repository release order for the blob auto-purge integration
+
+This feature spans independently released repositories. Each downstream repository's merge and release
+depends on the previous one having an actual **published** (not local or session-only) compatible package,
+in this order:
+
+1. **Core** publishes a compatible `Microsoft.Azure.DurableTask.Core` release first.
+2. **This repo** (`durabletask-dotnet`) then releases actual new SDK `Client`/`Abstractions` packages (and
+   `Grpc`/`Worker` as needed) against that published Core, before the `LargePayloadPurge.Abstractions`
+   contract and `Extensions.AzureBlobPayloads` packages. Since the contract now shares the same repository-wide
+   version as `Client`/`Abstractions` (it has no independent version of its own), all of these publish from
+   the same release run; "before" here means publish job order within that run, not a separate version or a
+   later release. The existing publication-pipeline `dependsOn` gates above enforce only `nugetApproval` →
+   `Client`/`Abstractions` → the contract → Blob; they do not gate Core publication, and they do not gate
+   every package in the SDK's actual nuspec dependency closure (for example `Grpc` or `Worker`, which
+   `Extensions.AzureBlobPayloads` also depends on but whose release jobs are not inputs to the contract or
+   Blob gate). Before treating this step as complete, or merging a downstream repository against it, a
+   release operator must separately confirm that the SDK's entire actual nuspec dependency closure is
+   published and restorable, not only the three packages the pipeline gates on. The contract's actual first
+   release uses whichever shared SDK version is approved and published next; do not assume or pin to a
+   specific future version, and do not treat the existing published `1.26.0` line as already containing it.
+3. **Durable Functions** must pin the actual published compatible Core/SDK/contract versions, not local or
+   session-only ones, before merging and releasing its host and optional packages.
+4. The private AzureManaged provider releases last, after the Durable Functions host and this repo's SDK
+   are published, with its own committed dependency-version upgrade and a clean restore and test pass
+   against the published packages.
+
+This order reduces, but does not eliminate, the risk of a downstream repository depending on an unpublished
+or incompatible upstream version. It does not substitute for verifying that every repository's committed
+package references are already pinned to real published versions; do not invent release versions or bump a
+committed dependency pin to a local or session-only one to make this order appear satisfied. In particular,
+do not assume a previously published SDK version already contains these new types merely because its number
+precedes an unreleased one — confirm against the actual release notes or package contents.
+
 ### Versioning Scheme
 
 We follow [semver](https://semver.org/) with optional pre-release tags:

eng/publish/publish.yml

@@ -445,8 +445,10 @@ extends:
       # add it for Microsoft.DurableTask.Extensions.AzureBlobPayloads
       - job: nugetRelease_Microsoft_DurableTask_Extensions_AzureBlobPayloads
         displayName: NuGet Release (Microsoft.DurableTask.Extensions.AzureBlobPayloads)
-        dependsOn: nugetApproval
-        condition: succeeded('nugetApproval') # nuget packages need to be on ADO first
+        dependsOn:
+        - nugetApproval
+        - nugetRelease_Microsoft_DurableTask_LargePayloadPurge_Abstractions
+        condition: succeeded() # the Blob package depends on the new contract package, which must publish first
         templateContext:
           type: releaseJob
           isProduction: true
@@ -463,4 +465,30 @@ extends:
             nuGetFeedType: external
             publishFeedCredentials: 'DurableTask org NuGet API Key'
             packagesToPush: '$(System.DefaultWorkingDirectory)/drop/Microsoft.DurableTask.Extensions.AzureBlobPayloads.*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg' # Despite this being a custom command, we need to keep this for 1ES validation
-            packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling
+            packageParentPath: $(System.DefaultWorkingDirectory) # This needs to be set to some prefix of the `packagesToPush` parameter. Apparently it helps with SDL tooling
+
+      # NuGet release (Microsoft.DurableTask.LargePayloadPurge.Abstractions)
+      - job: nugetRelease_Microsoft_DurableTask_LargePayloadPurge_Abstractions
+        displayName: NuGet Release (Microsoft.DurableTask.LargePayloadPurge.Abstractions)
+        dependsOn:
+        - nugetApproval
+        - nugetRelease_Microsoft_DurableTask_Abstractions
+        - nugetRelease_Microsoft_DurableTask_Client
+        condition: succeeded()
+        templateContext:
+          type: releaseJob
+          isProduction: true
+          inputs:
+          - input: pipelineArtifact
+            pipeline: officialPipeline
+            artifactName: drop
+            targetPath: $(System.DefaultWorkingDirectory)/drop
+        steps:
+        - task: 1ES.PublishNuget@1
+          displayName: 'NuGet push (Microsoft.DurableTask.LargePayloadPurge.Abstractions)'
+          inputs:
+            command: push
+            nuGetFeedType: external
+            publishFeedCredentials: 'DurableTask org NuGet API Key'
+            packagesToPush: '$(System.DefaultWorkingDirectory)/drop/Microsoft.DurableTask.LargePayloadPurge.Abstractions.*.nupkg;!$(System.DefaultWorkingDirectory)/**/*.symbols.nupkg'
+            packageParentPath: $(System.DefaultWorkingDirectory)

list-nuget-packages-links.ps1

@@ -72,6 +72,7 @@ $packages = @(
     "Microsoft.DurableTask.Worker.Grpc",
     "Microsoft.DurableTask.Client.OrchestrationServiceClientShim",
     "Microsoft.DurableTask.Extensions.AzureBlobPayloads",
+    "Microsoft.DurableTask.LargePayloadPurge.Abstractions",
     "Microsoft.DurableTask.Client.AzureManaged",
     "Microsoft.DurableTask.Worker.AzureManaged",
     "Microsoft.DurableTask.ScheduledTasks",