diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java index ce6eb70701..3b0549ddf8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java @@ -9,6 +9,9 @@ import java.util.function.Function; import java.util.function.Supplier; import software.amazon.lambda.durable.config.CallbackConfig; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.MapConfig; import software.amazon.lambda.durable.config.ParallelConfig; @@ -18,6 +21,8 @@ import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.config.WithRetryConfig; import software.amazon.lambda.durable.context.BaseContext; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.MapResult; import software.amazon.lambda.durable.model.WaitForConditionResult; @@ -543,6 +548,212 @@ default ParallelDurableFuture parallel(String name) { */ ParallelDurableFuture parallel(String name, ParallelConfig config); + // =============== distributedMap ================ + + /** + * Runs a distributed map with default config, blocking until the run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapSummary distributedMap( + String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + DistributedMapConfig.builder().build()) + .get(); + } + + /** + * Runs a distributed map, blocking until the run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapSummary distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, config) + .get(); + } + + /** + * Runs a distributed map that collects per-item results with a TypeToken, blocking until the run reaches a terminal + * state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, resultType, config) + .get(); + } + + /** + * Runs a distributed map that collects per-item results with a result Class, blocking until the run reaches a + * terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config) + .get(); + } + + /** + * Runs a distributed map that collects per-item results with a result Class and default config, blocking until the + * run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + TypeToken.get(resultType), + DistributedMapConfig.builder().build()) + .get(); + } + + /** + * Runs a distributed map that collects per-item results with a TypeToken and default config, blocking until the run + * reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + resultType, + DistributedMapConfig.builder().build()) + .get(); + } + + /** + * Asynchronously runs a distributed map with default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DurableFuture distributedMapAsync( + String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + DistributedMapConfig.builder().build()); + } + + /** + * Asynchronously runs a distributed map. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + DurableFuture distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config); + + /** + * Asynchronously runs a distributed map that collects per-item results with a TypeToken. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config); + + /** + * Asynchronously runs a distributed map that collects per-item results with a result Class. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config); + } + + /** + * Asynchronously runs a distributed map that collects per-item results with a result Class and default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + TypeToken.get(resultType), + DistributedMapConfig.builder().build()); + } + + /** + * Asynchronously runs a distributed map that collects per-item results with a TypeToken and default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + resultType, + DistributedMapConfig.builder().build()); + } + /** * Executes a submitter function and waits for an external callback, blocking until the callback completes. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java new file mode 100644 index 0000000000..736a636e44 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java @@ -0,0 +1,138 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.time.Duration; +import software.amazon.lambda.durable.serde.SerDes; + +/** Optional configuration for a distributed map run. */ +public class DistributedMapConfig { + private static final long MAX_TIMEOUT_SECONDS = 7776000; // 90 days + + private final DistributedMapDestination destination; + private final CompletionConfig completionConfig; + private final Duration timeout; + private final SerDes resultSerDes; + + private DistributedMapConfig(Builder builder) { + if (builder.timeout != null + && (builder.timeout.toSeconds() <= 0 || builder.timeout.toSeconds() > MAX_TIMEOUT_SECONDS)) { + throw new IllegalArgumentException( + "timeout must be positive and at most 90 days, got: " + builder.timeout.toSeconds() + "s"); + } + this.destination = builder.destination; + this.completionConfig = builder.completionConfig; + this.timeout = builder.timeout; + this.resultSerDes = builder.resultSerDes; + } + + public DistributedMapDestination destination() { + return destination; + } + + public CompletionConfig completionConfig() { + return completionConfig; + } + + public Duration timeout() { + return timeout; + } + + public SerDes resultSerDes() { + return resultSerDes; + } + + public static Builder builder() { + return new Builder(); + } + + public Builder toBuilder() { + var builder = new Builder(); + builder.destination = destination; + builder.completionConfig = completionConfig; + builder.timeout = timeout; + builder.resultSerDes = resultSerDes; + return builder; + } + + /** Builder for DistributedMapConfig. */ + public static class Builder { + private DistributedMapDestination destination; + private CompletionConfig completionConfig; + private Duration timeout; + private SerDes resultSerDes; + + private Builder() {} + + public Builder destination(DistributedMapDestination destination) { + this.destination = destination; + return this; + } + + public Builder completionConfig(CompletionConfig completionConfig) { + this.completionConfig = completionConfig; + return this; + } + + public Builder timeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public Builder resultSerDes(SerDes resultSerDes) { + this.resultSerDes = resultSerDes; + return this; + } + + public DistributedMapConfig build() { + return new DistributedMapConfig(this); + } + } + + /** Failure-tolerance configuration for a distributed map run. */ + public record CompletionConfig( + Integer toleratedFailureCount, Integer toleratedFailurePercentage, Integer minimumSampleSize) { + + public CompletionConfig { + if (toleratedFailureCount != null && toleratedFailurePercentage != null) { + throw new IllegalArgumentException( + "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive"); + } + if (minimumSampleSize != null && toleratedFailurePercentage == null) { + throw new IllegalArgumentException("minimumSampleSize is only valid with toleratedFailurePercentage"); + } + // Omitting the config means zero tolerated failures, so an empty one is rejected rather than omitted. + if (toleratedFailureCount == null && toleratedFailurePercentage == null) { + throw new IllegalArgumentException( + "a completion config must set toleratedFailureCount or toleratedFailurePercentage"); + } + if (toleratedFailureCount != null && toleratedFailureCount < 0) { + throw new IllegalArgumentException( + "toleratedFailureCount must be non-negative, got: " + toleratedFailureCount); + } + if (toleratedFailurePercentage != null + && (toleratedFailurePercentage < 0 || toleratedFailurePercentage > 100)) { + throw new IllegalArgumentException( + "toleratedFailurePercentage must be between 0 and 100, got: " + toleratedFailurePercentage); + } + if (minimumSampleSize != null && minimumSampleSize < 1) { + throw new IllegalArgumentException("minimumSampleSize must be at least 1, got: " + minimumSampleSize); + } + } + + /** Stops the run after more than this many items permanently fail. */ + public static CompletionConfig failureCount(int count) { + return new CompletionConfig(count, null, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100). */ + public static CompletionConfig failurePercentage(int percentage) { + return new CompletionConfig(null, percentage, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100), gated by a minimum sample size. */ + public static CompletionConfig failurePercentage(int percentage, int minimumSampleSize) { + return new CompletionConfig(null, percentage, minimumSampleSize); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java new file mode 100644 index 0000000000..400f777b62 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java @@ -0,0 +1,90 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** Destination routing for distributed map run results. */ +public record DistributedMapDestination(Success onSuccess, Failure onFailure) { + + /** Routes only succeeded item records. */ + public static DistributedMapDestination of(Success onSuccess) { + return new DistributedMapDestination(onSuccess, null); + } + + /** Routes only failed item records. */ + public static DistributedMapDestination of(Failure onFailure) { + return new DistributedMapDestination(null, onFailure); + } + + /** Routes both succeeded and failed item records. */ + public static DistributedMapDestination of(Success onSuccess, Failure onFailure) { + return new DistributedMapDestination(onSuccess, onFailure); + } + + /** Route succeeded item records to an S3 prefix (defaults to include output only). */ + public static Success s3Successes(String prefixUri) { + return s3Successes(prefixUri, false, true, null); + } + + /** Route succeeded item records to an S3 prefix. */ + public static Success s3Successes( + String prefixUri, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return new Success( + parsed.bucket(), + parsed.path() != null ? parsed.path() : "", + includeInput, + includeOutput, + expectedBucketOwner); + } + + /** Route permanently-failed item records to an S3 prefix (defaults to include input and error). */ + public static Failure s3Failures(String prefixUri) { + return s3Failures(prefixUri, true, true, null); + } + + /** Route permanently-failed item records to an S3 prefix. */ + public static Failure s3Failures( + String prefixUri, boolean includeInput, boolean includeError, String expectedBucketOwner) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return new Failure( + parsed.bucket(), + parsed.path() != null ? parsed.path() : "", + includeInput, + includeError, + expectedBucketOwner); + } + + /** S3 destination for succeeded distributed map items. */ + public record Success( + String bucket, String prefix, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { + + public Success { + if (bucket == null || bucket.isEmpty()) { + throw new IllegalArgumentException("success destination bucket cannot be empty"); + } + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + DistributedMapValidation.validateDestinationPrefix(prefix); + if (!includeInput && !includeOutput) { + throw new IllegalArgumentException("success destination must include at least one of input or output"); + } + } + } + + /** S3 destination for permanently-failed distributed map items. */ + public record Failure( + String bucket, String prefix, boolean includeInput, boolean includeError, String expectedBucketOwner) { + + public Failure { + if (bucket == null || bucket.isEmpty()) { + throw new IllegalArgumentException("failure destination bucket cannot be empty"); + } + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + DistributedMapValidation.validateDestinationPrefix(prefix); + if (!includeInput && !includeError) { + throw new IllegalArgumentException("failure destination must include at least one of input or error"); + } + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java new file mode 100644 index 0000000000..6c70ee05e1 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** Processor configuration for a distributed map run. */ +public class DistributedMapProcessor { + + /** Per-batch response mode reported by the processor. */ + public enum ResponseMode { + BATCH, + ITEM_OUTCOMES + } + + private static final int MIN_BATCH_SIZE = 1; + private static final int MAX_BATCH_SIZE = 10000; + private static final int MAX_NAME_PREFIX_LENGTH = 36; + + private final String functionName; + private final ResponseMode responseMode; + private final Integer batchSize; + private final DistributedMapProcessorRetryConfig retry; + private final String durableExecutionNamePrefix; + + private DistributedMapProcessor(Builder builder) { + DistributedMapValidation.validateFunctionName(builder.functionName); + if (builder.batchSize != null && (builder.batchSize < MIN_BATCH_SIZE || builder.batchSize > MAX_BATCH_SIZE)) { + throw new IllegalArgumentException("batchSize must be between 1 and 10000, got: " + builder.batchSize); + } + if (builder.durableExecutionNamePrefix != null + && (builder.durableExecutionNamePrefix.isEmpty() + || builder.durableExecutionNamePrefix.length() > MAX_NAME_PREFIX_LENGTH)) { + throw new IllegalArgumentException("durableExecutionNamePrefix must be between 1 and 36 characters, got: " + + builder.durableExecutionNamePrefix.length()); + } + this.functionName = builder.functionName; + this.responseMode = builder.responseMode; + this.batchSize = builder.batchSize; + this.retry = builder.retry; + this.durableExecutionNamePrefix = builder.durableExecutionNamePrefix; + } + + public String functionName() { + return functionName; + } + + public ResponseMode responseMode() { + return responseMode; + } + + public Integer batchSize() { + return batchSize; + } + + /** Returns the retry settings, or null for the service defaults. */ + public DistributedMapProcessorRetryConfig retry() { + return retry; + } + + public String durableExecutionNamePrefix() { + return durableExecutionNamePrefix; + } + + /** Processor that reports a single pass/fail outcome for the whole batch, with no per-item results. */ + public static Builder reportBatchOutcome(String functionName) { + return new Builder(functionName, ResponseMode.BATCH); + } + + /** + * Processor that may report a failure or an output for each item. Every item it does not list succeeds with no + * output. + */ + public static Builder reportItemOutcomes(String functionName) { + return new Builder(functionName, ResponseMode.ITEM_OUTCOMES); + } + + /** Builder for DistributedMapProcessor. */ + public static class Builder { + private final String functionName; + private final ResponseMode responseMode; + private Integer batchSize; + private DistributedMapProcessorRetryConfig retry; + private String durableExecutionNamePrefix; + + private Builder(String functionName, ResponseMode responseMode) { + this.functionName = functionName; + this.responseMode = responseMode; + } + + public Builder batchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + /** + * Sets the per-item retry policy. Omit it to take the service defaults of 2 attempts within 6 hours. + * + * @param retry the retry settings + * @return this builder + */ + public Builder retry(DistributedMapProcessorRetryConfig retry) { + this.retry = retry; + return this; + } + + public Builder durableExecutionNamePrefix(String durableExecutionNamePrefix) { + this.durableExecutionNamePrefix = durableExecutionNamePrefix; + return this; + } + + public DistributedMapProcessor build() { + return new DistributedMapProcessor(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java new file mode 100644 index 0000000000..bda75df6a5 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.time.Duration; + +/** + * Retry settings for a distributed map processor. + * + *

Retry policy belongs to the processor reference rather than the run, because it defines how the service invokes + * that function. One Lambda function can back several processor references with different policies. + */ +public class DistributedMapProcessorRetryConfig { + + /** Sentinel for unbounded retry attempts (wire value -1). */ + public static final int UNLIMITED = -1; + + private static final long MIN_RETRY_DURATION_SECONDS = 60; + private static final long MAX_RETRY_DURATION_SECONDS = 21600; + + private final Integer maxRetryAttempts; + private final Duration maxRetryDuration; + + private DistributedMapProcessorRetryConfig(Builder builder) { + this.maxRetryAttempts = builder.maxRetryAttempts; + this.maxRetryDuration = builder.maxRetryDuration; + } + + /** Returns the max retry attempts, UNLIMITED for unbounded, or null for the service default of 2. */ + public Integer maxRetryAttempts() { + return maxRetryAttempts; + } + + /** Returns the cumulative retry duration budget, or null for the service default of 6 hours. */ + public Duration maxRetryDuration() { + return maxRetryDuration; + } + + /** + * Creates a new builder for {@code DistributedMapProcessorRetryConfig}. + * + * @return a new builder instance + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for DistributedMapProcessorRetryConfig. */ + public static class Builder { + private Integer maxRetryAttempts; + private Duration maxRetryDuration; + + private Builder() {} + + /** + * Sets the number of function-error retries before an item permanently fails. Pass {@link #UNLIMITED} for no + * attempt limit within the duration budget, or 0 to disable retries. + * + * @param maxRetryAttempts a non-negative count, or UNLIMITED + * @return this builder + */ + public Builder maxRetryAttempts(int maxRetryAttempts) { + if (maxRetryAttempts < 0 && maxRetryAttempts != UNLIMITED) { + throw new IllegalArgumentException( + "maxRetryAttempts must be non-negative or DistributedMapProcessorRetryConfig.UNLIMITED, got: " + + maxRetryAttempts); + } + this.maxRetryAttempts = maxRetryAttempts; + return this; + } + + /** + * Sets the cumulative backoff budget across all of an item's retries, from 1 minute through 6 hours. + * + * @param maxRetryDuration the budget + * @return this builder + */ + public Builder maxRetryDuration(Duration maxRetryDuration) { + if (maxRetryDuration != null + && (maxRetryDuration.toSeconds() < MIN_RETRY_DURATION_SECONDS + || maxRetryDuration.toSeconds() > MAX_RETRY_DURATION_SECONDS)) { + throw new IllegalArgumentException("maxRetryDuration must be between 1 minute and 6 hours, got: " + + maxRetryDuration.toSeconds() + "s"); + } + this.maxRetryDuration = maxRetryDuration; + return this; + } + + public DistributedMapProcessorRetryConfig build() { + return new DistributedMapProcessorRetryConfig(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java new file mode 100644 index 0000000000..620e460567 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java @@ -0,0 +1,303 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.util.DistributedMapValidation; +import software.amazon.lambda.durable.util.ParameterValidator; + +/** + * Source of items for a distributed map run. + * + * @param the item type produced by the source + */ +public class DistributedMapSource { + + private static final long MAX_ITEMS_TO_READ = 100_000_000L; + + /** The kind of source backing a distributed map. */ + public enum SourceType { + INLINE, + S3, + READER_FUNCTION + } + + /** How the bytes of an S3 object are split into items. */ + public enum Format { + JSON_LINES, + JSON_ARRAY, + CSV + } + + /** What an S3 source reads. */ + public enum S3SourceType { + OBJECT, + LISTED_OBJECT_METADATA, + LISTED_OBJECTS + } + + private final SourceType sourceType; + private final Long maxItems; + private final List inlineItems; + private final SerDes inlineSerDes; + private final S3SourceConfig s3; + private final ReaderSourceConfig reader; + + private DistributedMapSource( + SourceType sourceType, + Long maxItems, + List inlineItems, + SerDes inlineSerDes, + S3SourceConfig s3, + ReaderSourceConfig reader) { + if (maxItems != null && (maxItems < 1 || maxItems > MAX_ITEMS_TO_READ)) { + throw new IllegalArgumentException( + "maxItems must be between 1 and " + MAX_ITEMS_TO_READ + ", got: " + maxItems); + } + this.sourceType = sourceType; + this.maxItems = maxItems; + // A null item is allowed and is sent as JSON null, so the copy must accept null elements. + this.inlineItems = inlineItems != null ? Collections.unmodifiableList(new ArrayList<>(inlineItems)) : null; + this.inlineSerDes = inlineSerDes; + this.s3 = s3; + this.reader = reader; + } + + public SourceType sourceType() { + return sourceType; + } + + public Long maxItems() { + return maxItems; + } + + public List inlineItems() { + return inlineItems; + } + + public SerDes inlineSerDes() { + return inlineSerDes; + } + + public S3SourceConfig s3() { + return s3; + } + + public ReaderSourceConfig reader() { + return reader; + } + + /** An in-memory list of items embedded in the start checkpoint. */ + public static DistributedMapSource inline(Collection items) { + return inline(items, null); + } + + /** An in-memory list of items embedded in the start checkpoint, serialized with the given SerDes. */ + public static DistributedMapSource inline(Collection items, SerDes serDes) { + ParameterValidator.validateOrderedCollection(items); + return new DistributedMapSource<>(SourceType.INLINE, null, new ArrayList<>(items), serDes, null, null); + } + + /** Read a single S3 object, treating each line as an item. */ + public static DistributedMapSource s3JsonLines(String uri) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3JsonLines requires an S3 object key"); + } + return s3(object(parsed.bucket(), parsed.path(), Format.JSON_LINES, null)); + } + + /** Read a single S3 object holding a JSON array, treating each element as an item. */ + public static DistributedMapSource s3JsonArray(String uri) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3JsonArray requires an S3 object key"); + } + return s3(object(parsed.bucket(), parsed.path(), Format.JSON_ARRAY, null)); + } + + /** Read a single S3 object, treating each record as an item. */ + public static DistributedMapSource s3Csv(String uri, CsvFormat format) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3Csv requires an S3 object key"); + } + return s3(object(parsed.bucket(), parsed.path(), Format.CSV, format)); + } + + /** Read each object under a prefix as one item (object contents are not read). */ + public static DistributedMapSource s3Objects(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(new S3SourceConfig( + S3SourceType.LISTED_OBJECT_METADATA, + parsed.bucket(), + null, + null, + new ListedObjectMetadataConfig(prefixOrEmpty(parsed.path())), + null)); + } + + /** Read a prefix, flattening each object's lines into items. */ + public static DistributedMapSource s3FlattenedJsonLines(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.JSON_LINES, null)); + } + + /** Read a prefix, flattening each object's JSON array elements into items. */ + public static DistributedMapSource s3FlattenedJsonArray(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.JSON_ARRAY, null)); + } + + /** Read a prefix, flattening each object's records into items. */ + public static DistributedMapSource s3FlattenedCsv(String prefixUri, CsvFormat format) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.CSV, format)); + } + + private static S3SourceConfig object(String bucket, String key, Format format, CsvFormat csvFormat) { + return new S3SourceConfig( + S3SourceType.OBJECT, bucket, null, new ObjectConfig(key, format, csvFormat), null, null); + } + + private static S3SourceConfig listedObjects(String bucket, String keyPrefix, Format format, CsvFormat csvFormat) { + return new S3SourceConfig( + S3SourceType.LISTED_OBJECTS, + bucket, + null, + null, + null, + new ListedObjectsConfig(keyPrefix, format, csvFormat)); + } + + /** Page items from a customer-supplied reader Lambda function. */ + public static DistributedMapSource reader(String functionName) { + return reader(functionName, null, null); + } + + /** Page items from a customer-supplied reader Lambda function with an initial state. */ + public static DistributedMapSource reader(String functionName, S initialState, SerDes stateSerDes) { + return new DistributedMapSource<>( + SourceType.READER_FUNCTION, + null, + null, + null, + null, + new ReaderSourceConfig(functionName, initialState, stateSerDes)); + } + + /** Returns a copy limiting the total number of items read. */ + public DistributedMapSource maxItems(long maxItems) { + return new DistributedMapSource<>(sourceType, maxItems, inlineItems, inlineSerDes, s3, reader); + } + + /** Returns a copy asserting the expected S3 bucket owner (S3 sources only). */ + public DistributedMapSource expectedBucketOwner(String accountId) { + if (s3 == null) { + throw new IllegalStateException("expectedBucketOwner only applies to S3 sources"); + } + DistributedMapValidation.validateBucketOwner(accountId); + return new DistributedMapSource<>( + sourceType, maxItems, inlineItems, inlineSerDes, s3.withExpectedBucketOwner(accountId), reader); + } + + private static DistributedMapSource s3(S3SourceConfig s3) { + return new DistributedMapSource<>(SourceType.S3, null, null, null, s3, null); + } + + private static String prefixOrEmpty(String path) { + return path != null ? path : ""; + } + + /** Resolved S3 source configuration. Exactly one config block matching {@code type} is set. */ + public record S3SourceConfig( + S3SourceType type, + String bucket, + String expectedBucketOwner, + ObjectConfig objectConfig, + ListedObjectMetadataConfig listedObjectMetadataConfig, + ListedObjectsConfig listedObjectsConfig) { + + S3SourceConfig withExpectedBucketOwner(String owner) { + return new S3SourceConfig( + type, bucket, owner, objectConfig, listedObjectMetadataConfig, listedObjectsConfig); + } + } + + /** Reads the single object at {@code key}. */ + public record ObjectConfig(String key, Format format, CsvFormat csvFormat) {} + + /** Lists the objects under {@code keyPrefix} without reading their contents. */ + public record ListedObjectMetadataConfig(String keyPrefix) {} + + /** Lists the objects under {@code keyPrefix} and reads every listed object as {@code format}. */ + public record ListedObjectsConfig(String keyPrefix, Format format, CsvFormat csvFormat) {} + + /** Resolved reader-function source configuration. */ + public record ReaderSourceConfig(String functionName, Object initialState, SerDes stateSerDes) { + public ReaderSourceConfig { + DistributedMapValidation.validateFunctionName(functionName); + } + } + + /** CSV parsing options for a distributed map S3 source. */ + public record CsvFormat(HeaderLocation headerLocation, List columns, CsvDelimiter delimiter) { + + /** Where the column header row comes from. */ + public enum HeaderLocation { + FIRST_ROW, + GIVEN + } + + public CsvFormat { + if (headerLocation == null) { + throw new IllegalArgumentException("CsvFormat requires a headerLocation"); + } + if (headerLocation == HeaderLocation.GIVEN) { + if (columns == null) { + throw new IllegalArgumentException("CsvFormat with headerLocation GIVEN requires headers"); + } + DistributedMapValidation.validateColumns("headers", columns); + } else { + DistributedMapValidation.validateColumns("expectedColumns", columns); + } + columns = columns != null ? List.copyOf(columns) : null; + delimiter = delimiter != null ? delimiter : CsvDelimiter.COMMA; + } + + /** + * File has its own header row. Column names are used only for compile-time typing and are not sent on the wire. + */ + public static CsvFormat expectedColumns(List columns) { + return new CsvFormat(HeaderLocation.FIRST_ROW, columns, CsvDelimiter.COMMA); + } + + /** File has no header row. The given names are the columns, sent on the wire, and the first row is data. */ + public static CsvFormat headers(List headers) { + return new CsvFormat(HeaderLocation.GIVEN, headers, CsvDelimiter.COMMA); + } + + /** File has its own header row and no column names are declared. */ + public static CsvFormat firstRow() { + return new CsvFormat(HeaderLocation.FIRST_ROW, null, CsvDelimiter.COMMA); + } + + /** Returns a copy with the given delimiter. */ + public CsvFormat withDelimiter(CsvDelimiter delimiter) { + return new CsvFormat(headerLocation, columns, delimiter); + } + } + + /** Delimiter for CSV distributed map sources. */ + public enum CsvDelimiter { + COMMA, + PIPE, + SEMICOLON, + SPACE, + TAB + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java index 0c79165ec3..d755231fa9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java @@ -10,6 +10,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; import software.amazon.lambda.durable.DurableCallbackFuture; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; @@ -18,6 +19,9 @@ import software.amazon.lambda.durable.StepContext; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.CallbackConfig; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.MapConfig; import software.amazon.lambda.durable.config.ParallelConfig; @@ -31,12 +35,16 @@ import software.amazon.lambda.durable.execution.OperationIdGenerator; import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.MapResult; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.operation.CallbackOperation; import software.amazon.lambda.durable.operation.ChildContextOperation; +import software.amazon.lambda.durable.operation.DistributedMapOperation; import software.amazon.lambda.durable.operation.InvokeOperation; import software.amazon.lambda.durable.operation.MapOperation; import software.amazon.lambda.durable.operation.ParallelOperation; @@ -371,6 +379,86 @@ public DurableFuture waitForConditionAsync( return operation; } + // =============== distributedMap ================ + + @Override + public DurableFuture distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config) { + return startDistributedMap( + name, + source, + processor, + maxConcurrency, + config, + false, + TypeToken.get(DistributedMapSummary.class), + DistributedMapOperation::toSummary); + } + + @Override + public DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config) { + Objects.requireNonNull(resultType, "resultType cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + var resultSerDes = config.resultSerDes() != null + ? config.resultSerDes() + : getDurableConfig().getSerDes(); + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeToken> token = (TypeToken) TypeToken.get(DistributedMapResult.class); + return startDistributedMap( + name, + source, + processor, + maxConcurrency, + config, + true, + token, + (status, details) -> DistributedMapOperation.toResult(status, details, resultType, resultSerDes)); + } + + private DurableFuture startDistributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config, + boolean collectResults, + TypeToken resultType, + BiFunction resultBuilder) { + Objects.requireNonNull(source, "source cannot be null"); + Objects.requireNonNull(processor, "processor cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + ParameterValidator.validateOperationName(name); + if (maxConcurrency < 1) { + throw new IllegalArgumentException("maxConcurrency must be greater than zero, got: " + maxConcurrency); + } + if (!collectResults && config.resultSerDes() != null) { + throw new IllegalArgumentException( + "resultSerDes is set but this distributedMap call does not collect results, use a resultType overload"); + } + var options = DistributedMapOperation.toOptions( + source, processor, maxConcurrency, config, getDurableConfig().getSerDes(), collectResults); + var operationId = nextOperationId(); + var operation = new DistributedMapOperation<>( + OperationIdentifier.of(operationId, name, OperationSubType.DISTRIBUTED_MAP), + options, + resultType, + getDurableConfig().getSerDes(), + resultBuilder, + this); + operation.execute(); + return operation; + } + // =============== withRetry ================ private static final Duration DEFAULT_BACKOFF_DELAY = Duration.ofSeconds(1); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java new file mode 100644 index 0000000000..eb08d7ecc2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java @@ -0,0 +1,320 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import com.amazonaws.services.lambda.runtime.RequestHandler; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.MapConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.MapResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Authoring helpers that wrap a customer function into a distributed map processor or reader Lambda handler. */ +public final class DistributedMapHandlers { + private static final SerDes JSON = new JacksonSerDes(); + private static final TypeToken OUTPUT_TYPE = TypeToken.get(String.class); + private static final int MIN_CONCURRENCY = 1; + private static final int DEFAULT_CONCURRENCY = 1; + private static final TypeToken EVENT_TYPE = TypeToken.get(ProcessorEvent.class); + private static final int READER_STATE_LIMIT = 32 * 1024; + private static final String ITEMS_OP_NAME = "distributed-map-items"; + + private DistributedMapHandlers() {} + + private static void validateConcurrency(int concurrency) { + if (concurrency < MIN_CONCURRENCY) { + throw new IllegalArgumentException( + "concurrency must be at least " + MIN_CONCURRENCY + ", got: " + concurrency); + } + } + + /** Wraps a per-item function as a processor handler, running one item at a time. */ + public static RequestHandler createDistributedMapItemHandler( + Function func, TypeToken itemType, SerDes itemSerDes, SerDes resultSerDes) { + return createDistributedMapItemHandler(func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY); + } + + /** Wraps a per-item function as a processor handler. Pair it with DistributedMapProcessor.reportItemOutcomes. */ + public static RequestHandler createDistributedMapItemHandler( + Function func, TypeToken itemType, SerDes itemSerDes, SerDes resultSerDes, int concurrency) { + validateConcurrency(concurrency); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return (event, context) -> { + var records = event.records(); + var outputs = new String[records.size()]; + var errors = new Throwable[records.size()]; + var pool = Executors.newFixedThreadPool(concurrency); + try { + var futures = new ArrayList>(); + for (var i = 0; i < records.size(); i++) { + var index = i; + futures.add(pool.submit(() -> { + I item = toItem(inSerdes, records.get(index).body(), itemType); + O output = func.apply(item); + outputs[index] = toOutput(outSerdes, output); + return null; + })); + } + for (var i = 0; i < futures.size(); i++) { + try { + futures.get(i).get(); + } catch (ExecutionException e) { + errors[i] = e.getCause() != null ? e.getCause() : e; + } + } + } catch (InterruptedException e) { + pool.shutdownNow(); + Thread.currentThread().interrupt(); + throw new IllegalStateException("Distributed map item handler interrupted", e); + } finally { + pool.shutdown(); + } + + var results = new ArrayList(); + var failures = new ArrayList(); + for (var i = 0; i < records.size(); i++) { + var itemId = records.get(i).itemId(); + if (errors[i] != null) { + failures.add(new ItemFailure(itemId, ItemFailure.ErrorDetail.of(errors[i]))); + } else { + results.add(new ItemResult(itemId, outputs[i])); + } + } + return new ItemHandlerResponse(results, failures); + }; + } + + /** + * Wraps a whole-batch function as a processor handler. Returning succeeds every item, throwing fails the batch. + * Pair it with DistributedMapProcessor.reportBatchOutcome. + */ + public static RequestHandler createDistributedMapBatchHandler( + Consumer> func, TypeToken itemType, SerDes itemSerDes) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return (event, context) -> { + func.accept(toItems(serdes, event, itemType)); + return null; + }; + } + + /** + * Wraps a reader function as a source handler. A null nextState signals the source is exhausted. + * + *

itemSerDes must be the serdes the processor is configured with, since the processor deserializes every item + * body with its own item serdes. The reader and the processor are separate Lambdas, so nothing checks that they + * match. + */ + public static RequestHandler createDistributedMapReader( + BiFunction> func, + TypeToken stateType, + SerDes stateSerDes, + SerDes itemSerDes) { + var serdes = stateSerDes != null ? stateSerDes : JSON; + var itemSerdes = itemSerDes != null ? itemSerDes : JSON; + return (event, context) -> { + S state = event.state() != null ? serdes.deserialize(event.state(), stateType) : null; + + var page = func.apply(state, event.maxItemsPerPage()); + if (page.items().size() > event.maxItemsPerPage()) { + throw new IllegalStateException("reader returned " + + page.items().size() + " items, exceeding maxItemsPerPage " + event.maxItemsPerPage()); + } + var items = new ArrayList(page.items().size()); + var index = 0; + for (I item : page.items()) { + var body = itemSerdes.serialize(item); + if (body == null) { + // The service requires every item to be a string, so a null item is sent as the JSON literal null. + if (item != null) { + throw new IllegalStateException( + "serialization of reader item " + index + " produced no string"); + } + body = "null"; + } + items.add(body); + index++; + } + String nextState = null; + if (page.nextState() != null) { + nextState = serdes.serialize(page.nextState()); + if (nextState == null) { + throw new IllegalStateException("serialization of reader nextState produced no string"); + } + if (nextState.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalStateException( + "reader nextState exceeds the " + (READER_STATE_LIMIT / 1024) + " KB limit"); + } + } + return new ReaderResponse(items, nextState); + }; + } + + /** Durable variant of the item handler, running one item at a time. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY, null); + } + + /** Durable variant of the item handler, running one item at a time, with a durable execution config. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + DurableConfig durableConfig) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY, durableConfig); + } + + /** Durable variant of the item handler. The function receives the per-item DurableContext. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + int concurrency) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, concurrency, null); + } + + /** + * Durable variant of the item handler. The function receives the per-item DurableContext. + * + * @param durableConfig the config the wrapped durable execution runs under, for example to register plugins or + * supply a client. A null uses DurableConfig.defaultConfig. + */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + int concurrency, + DurableConfig durableConfig) { + validateConcurrency(concurrency); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + var records = event.records(); + var bodies = new ArrayList(records.size()); + for (var record : records) { + bodies.add(record.body()); + } + MapResult batch = ctx.map( + ITEMS_OP_NAME, + bodies, + OUTPUT_TYPE, + (body, index, mapContext) -> { + O output = func.apply(mapContext, toItem(inSerdes, body, itemType)); + return toOutput(outSerdes, output); + }, + MapConfig.builder() + .completionConfig(CompletionConfig.allCompleted()) + .maxConcurrency(concurrency) + .build()); + + var results = new ArrayList(); + var failures = new ArrayList(); + for (var i = 0; i < records.size(); i++) { + var itemId = records.get(i).itemId(); + var item = batch.getItem(i); + if (item.status() == MapResult.MapResultItem.Status.SUCCEEDED) { + results.add(new ItemResult(itemId, item.result())); + } else { + var error = item.error(); + failures.add(new ItemFailure( + itemId, + error != null + ? new ItemFailure.ErrorDetail( + error.errorType(), error.errorMessage(), null, error.stackTrace()) + : null)); + } + } + return new ItemHandlerResponse(results, failures); + }, + configOrDefault(durableConfig)); + } + + /** Durable variant of the batch handler. The function receives the DurableContext. */ + public static + RequestHandler + createDistributedMapBatchHandlerWithDurableExecution( + BiConsumer> func, TypeToken itemType, SerDes itemSerDes) { + return createDistributedMapBatchHandlerWithDurableExecution(func, itemType, itemSerDes, null); + } + + /** + * Durable variant of the batch handler. The function receives the DurableContext. + * + * @param durableConfig the config the wrapped durable execution runs under, for example to register plugins or + * supply a client. A null uses DurableConfig.defaultConfig. + */ + public static + RequestHandler + createDistributedMapBatchHandlerWithDurableExecution( + BiConsumer> func, + TypeToken itemType, + SerDes itemSerDes, + DurableConfig durableConfig) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + func.accept(ctx, toItems(serdes, event, itemType)); + return null; + }, + configOrDefault(durableConfig)); + } + + private static DurableConfig configOrDefault(DurableConfig durableConfig) { + return durableConfig != null ? durableConfig : DurableConfig.defaultConfig(); + } + + private static List toItems(SerDes serdes, ProcessorEvent event, TypeToken itemType) { + var records = event.records(); + var items = new ArrayList(records.size()); + for (var record : records) { + items.add(toItem(serdes, record.body(), itemType)); + } + return items; + } + + private static I toItem(SerDes serdes, String body, TypeToken itemType) { + return serdes.deserialize(body, itemType); + } + + /** Serializes an item's output, or returns null when the handler returned nothing. */ + private static String toOutput(SerDes serdes, Object output) { + return output != null ? serdes.serialize(output) : null; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java new file mode 100644 index 0000000000..8620aaffa7 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * A failed item in a processor response. The error is a nested object on the wire, so it is a nested type here. + * + * @param itemIdentifier the item's identifier, copied from the input record + * @param error why the item failed, or null to report a failure with no detail + */ +public record ItemFailure(String itemIdentifier, ErrorDetail error) { + + /** + * Why an item failed. + * + * @param errorType the error's type, conventionally a fully qualified class name + * @param errorMessage the error's message, or null when it has none + * @param errorData machine-readable error detail, or null when there is none + * @param stackTrace the frames where the error was thrown, or null when they were not captured + */ + public record ErrorDetail(String errorType, String errorMessage, String errorData, List stackTrace) { + + /** Builds an error detail from a throwable. */ + public static ErrorDetail of(Throwable e) { + // errorData stays null. A throwable carries no machine-readable error data, so there is nothing + // to put here. + return new ErrorDetail( + e.getClass().getName(), + e.getMessage(), + null, + ExceptionHelper.serializeStackTrace(e.getStackTrace())); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java new file mode 100644 index 0000000000..acb9d80a28 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The reply an item handler returns. + * + *

A null list is omitted from the JSON rather than sent as null, because the Lambda runtime's serializer excludes + * null fields. The service treats an absent, null, or empty list identically, so this is a shape choice rather than a + * behavioural one. + * + * @param batchItemResults per-item outputs, or null to leave the key off the wire + * @param batchItemFailures the items that failed + */ +public record ItemHandlerResponse(List batchItemResults, List batchItemFailures) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java new file mode 100644 index 0000000000..82d50f25f0 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * A succeeded item in a processor response. + * + * @param itemIdentifier the item's identifier, copied from the input record + * @param output the item's output as the result serdes produced it, or null when the handler returned nothing. A null + * output means the item succeeded without an output. + */ +public record ItemResult(String itemIdentifier, String output) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java new file mode 100644 index 0000000000..14046b9cfd --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The batch the fan-out service sends to a processor function. + * + * @param records the items in this batch + */ +public record ProcessorEvent(List records) { + + /** + * Rejects an event that is not a processor batch. This runs on instances the Lambda runtime deserializes as well as + * on hand-constructed ones, because a record can only be built through its canonical constructor. + */ + public ProcessorEvent { + if (records == null) { + throw new IllegalStateException("expected a distributed map processor event carrying a 'records' array, " + + "the function must be triggered by a distributed map, not invoked directly. An event source " + + "such as SQS or S3 sends a capitalised 'Records' array, which is a different contract"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java new file mode 100644 index 0000000000..c4e331fc0c --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * One record of a processor event. + * + * @param itemId the item's identifier, assigned by the service + * @param body the serialized item, as the item serdes produced it + */ +public record ProcessorRecord(String itemId, String body) { + + /** + * Rejects a record with no item id. The wrappers report each outcome against this id, so a record without one would + * report its outcome against no item. This runs on instances the Lambda runtime deserializes as well. + */ + public ProcessorRecord { + if (itemId == null || itemId.isEmpty()) { + throw new IllegalStateException("expected each distributed map processor record to have an 'itemId'"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java new file mode 100644 index 0000000000..3501feffcf --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * The request the fan-out service sends to a reader function. + * + * @param maxItemsPerPage the most items the reader may return in this page. Boxed because the Lambda runtime turns an + * absent primitive into zero, which would be indistinguishable from a real value + * @param state the opaque state from the previous call, or null on the first call + */ +public record ReaderEvent(Integer maxItemsPerPage, String state) { + + /** Rejects an event that is not a reader request. Runs on runtime-deserialized instances too. */ + public ReaderEvent { + if (maxItemsPerPage == null) { + throw new IllegalStateException( + "expected a distributed map reader event carrying a numeric 'maxItemsPerPage', " + + "the function must be registered as a distributed map reader"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java new file mode 100644 index 0000000000..1c428ffbd2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** A page of items and the next state returned by a distributed map reader function. */ +public record ReaderPage(List items, S nextState) { + + /** Applies a defensive copy and defaults. */ + public ReaderPage { + // A null item is valid and is sent as the JSON literal null. List.copyOf rejects null elements. + items = items != null ? Collections.unmodifiableList(new ArrayList<>(items)) : Collections.emptyList(); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java new file mode 100644 index 0000000000..721434dfe1 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The page a reader function returns. + * + *

A null nextState is omitted from the JSON rather than sent as null, because the Lambda runtime's serializer + * excludes null fields. Omitting the key is how the reader contract signals that the source is exhausted. + * + * @param items this page's items, each serialized with the reader's item serdes + * @param nextState the state to resume from, or null when the source is exhausted + */ +public record ReaderResponse(List items, String nextState) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java new file mode 100644 index 0000000000..02f380eef3 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.exception; + +import software.amazon.lambda.durable.model.DistributedMapError; +import software.amazon.lambda.durable.model.DistributedMapStatus; + +/** + * Thrown by throwIfError when a distributed map run did not fully succeed, or when an item failed on a run that + * otherwise succeeded. It reports an aggregate run result and is not a single operation failure, so it stays out of the + * DurableOperationException hierarchy. + */ +public class DistributedMapException extends DurableExecutionException { + private final DistributedMapStatus status; + private final DistributedMapError error; + private final long failureCount; + + /** Run outcome failure describing the run status, failure count, and the run's error details. */ + public DistributedMapException(DistributedMapStatus status, long failureCount, DistributedMapError error) { + super(runMessage(status, failureCount, error)); + this.status = status; + this.error = error; + this.failureCount = failureCount; + } + + /** Item failure surfacing the first failed item's error. */ + public DistributedMapException( + DistributedMapStatus status, long failureCount, DistributedMapError error, String itemId) { + super(itemMessage(error, itemId)); + this.status = status; + this.error = error; + this.failureCount = failureCount; + } + + /** Returns the terminal run status. */ + public DistributedMapStatus status() { + return status; + } + + /** Returns the error details, or null when none were recorded. */ + public DistributedMapError error() { + return error; + } + + /** Returns the count of failed items. */ + public long failureCount() { + return failureCount; + } + + private static String runMessage(DistributedMapStatus status, long failureCount, DistributedMapError error) { + var message = new StringBuilder("Distributed map run ").append(status); + if (failureCount > 0) { + message.append(", ").append(failureCount).append(" item(s) failed"); + } + if (error != null && error.errorMessage() != null) { + message.append(": ").append(error.errorMessage()); + } + return message.toString(); + } + + private static String itemMessage(DistributedMapError error, String itemId) { + var type = error != null ? error.errorType() : null; + var message = error != null ? error.errorMessage() : null; + var hasType = type != null && !type.isEmpty(); + var hasMessage = message != null && !message.isEmpty(); + if (hasType && hasMessage) { + return type + ": " + message; + } + if (hasMessage) { + return message; + } + if (hasType) { + return type; + } + return "Distributed map item " + itemId + " failed"; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java index 34f44134cd..3480d1932e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java @@ -5,6 +5,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -16,6 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.SdkPojo; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationUpdate; @@ -270,6 +273,43 @@ private static int estimateSize(OperationUpdate update) { + update.type().toString().length() + update.action().toString().length() + (update.payload() != null ? update.payload().length() : 0) + + stringSize(update.distributedMapOptions()) + 100; } + + /** + * Total length of every string in a model object, including nested objects, lists and maps. A distributed map START + * update carries its customer-supplied values (inline items, reader state, CSV headers, S3 locations, function + * names) in its options rather than the payload, so they are counted here. + */ + private static int stringSize(Object value) { + if (value == null) { + return 0; + } + if (value instanceof String string) { + return string.length(); + } + if (value instanceof SdkPojo pojo) { + var size = 0; + for (SdkField field : pojo.sdkFields()) { + size += stringSize(field.getValueOrDefault(pojo)); + } + return size; + } + if (value instanceof Collection collection) { + var size = 0; + for (var element : collection) { + size += stringSize(element); + } + return size; + } + if (value instanceof Map map) { + var size = 0; + for (var entry : map.entrySet()) { + size += stringSize(entry.getKey()) + stringSize(entry.getValue()); + } + return size; + } + return 0; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d8db913265..cc838acb79 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -20,6 +20,7 @@ import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.dmap.ProcessorEvent; import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; @@ -31,6 +32,7 @@ import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.PluginInfoConverter; import software.amazon.lambda.durable.plugin.PluginRunner; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.util.ExceptionHelper; @@ -46,6 +48,7 @@ public class DurableExecutor { // Lambda response size limit is 6MB minus small epsilon for envelope private static final int LAMBDA_RESPONSE_SIZE_LIMIT = 6 * 1024 * 1024 - 50; + private static final SerDes JSON = new JacksonSerDes(); private DurableExecutor() {} @@ -56,6 +59,9 @@ public static DurableExecutionOutput execute( BiFunction handler, DurableConfig config) { var pluginRunner = config.getPluginRunner(); + // A distributed map processor's input and result have a format the service defines, so a customer's SerDes + // must not change them. Durable operations inside the processor still use the config's SerDes. + var invocationSerDes = inputType.getType() == ProcessorEvent.class ? JSON : config.getSerDes(); try (var executionManager = new ExecutionManager(input, config, lambdaContext)) { var isFirstInvocation = !executionManager.isReplaying(); var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; @@ -78,7 +84,7 @@ public static DurableExecutionOutput execute( Throwable inputFailure = null; try { userInput = extractUserInput( - executionManager.getExecutionOperation(), config.getSerDes(), inputType); + executionManager.getExecutionOperation(), invocationSerDes, inputType); } catch (Throwable t) { inputFailure = t; } @@ -171,11 +177,11 @@ public static DurableExecutionOutput execute( cause, pluginExecutionInput.get(), null); - return DurableExecutionOutput.failure(buildErrorObject(cause, config.getSerDes())); + return DurableExecutionOutput.failure(buildErrorObject(cause, invocationSerDes)); } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); + var outputPayload = invocationSerDes.serialize(result); var output = DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); fireOnInvocationEnd( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java new file mode 100644 index 0000000000..0ebc1c5f0a --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import java.util.List; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * Error details on a distributed map run or one of its items. + * + *

Held as a record rather than the client's ErrorObject, so it survives serialization across checkpoint-and-replay + * cycles without requiring AWS SDK-specific Jackson modules. + */ +public record DistributedMapError(String errorType, String errorMessage, String errorData, List stackTrace) { + + /** Creates an error from a throwable. */ + public static DistributedMapError of(Throwable e) { + return new DistributedMapError( + e.getClass().getName(), e.getMessage(), null, ExceptionHelper.serializeStackTrace(e.getStackTrace())); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java new file mode 100644 index 0000000000..289069ad1e --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java @@ -0,0 +1,137 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import java.util.Collections; +import java.util.List; +import software.amazon.lambda.durable.exception.DistributedMapException; + +/** + * Outcome of a distributed map run with collected per-item results. + * + *

An item whose status this SDK version does not recognize is kept in {@code items} with status + * {@link Item.Status#UNKNOWN_TO_SDK_VERSION}. It is excluded from {@link #succeeded()}, {@link #failed()}, + * {@link #results()} and {@link #errors()}. + */ +public record DistributedMapResult(DistributedMapSummary summary, List> items) { + + /** Applies a defensive copy and defaults. */ + public DistributedMapResult { + items = items != null ? List.copyOf(items) : Collections.emptyList(); + } + + /** Returns items that succeeded. */ + public List> succeeded() { + return items.stream() + .filter(item -> item.status() == Item.Status.SUCCEEDED) + .toList(); + } + + /** Returns items that failed. */ + public List> failed() { + return items.stream() + .filter(item -> item.status() == Item.Status.FAILED) + .toList(); + } + + /** Returns the outputs of succeeded items (includes null outputs from successful items). */ + public List results() { + return succeeded().stream().map(Item::output).toList(); + } + + /** Returns the errors of failed items (includes nulls for items that failed without error details). */ + public List errors() { + return failed().stream().map(Item::error).toList(); + } + + /** Returns the terminal run status. */ + public DistributedMapStatus status() { + return summary.status(); + } + + /** Returns true only when the service read the complete source, or null before run creation. */ + public Boolean sourceExhausted() { + return summary.sourceExhausted(); + } + + /** Returns the count of succeeded items. */ + public long successCount() { + return summary.successCount(); + } + + /** Returns the count of failed items. */ + public long failureCount() { + return summary.failureCount(); + } + + /** Returns the count of unprocessed items. */ + public long unprocessedCount() { + return summary.unprocessedCount(); + } + + /** Returns the total item count, or null until known. */ + public Long totalCount() { + return summary.totalCount(); + } + + /** Returns the run ARN, or null when the run never started. */ + public String distributedMapRunArn() { + return summary.distributedMapRunArn(); + } + + /** Returns why the run did not succeed, or null when the service recorded no error details. */ + public DistributedMapError error() { + return summary.error(); + } + + /** Returns true when any item failed. */ + public boolean hasFailure() { + return summary.hasFailure(); + } + + /** + * Throws DistributedMapException when the run did not succeed or any item failed. When the run succeeded and the + * result contains a failed item, the exception names the first failed item. + */ + public void throwIfError() { + if (summary.status() != DistributedMapStatus.SUCCEEDED) { + summary.throwIfError(); + return; + } + var failedItems = failed(); + if (!failedItems.isEmpty()) { + var first = failedItems.get(0); + throw new DistributedMapException(summary.status(), summary.failureCount(), first.error(), first.itemId()); + } + if (summary.hasFailure()) { + summary.throwIfError(); + } + } + + /** Outcome of a single distributed map item. */ + public record Item(String itemId, Status status, O output, DistributedMapError error) { + + /** Status of an individual distributed map item. */ + public enum Status { + SUCCEEDED, + FAILED, + /** A status the backend returned that this SDK version does not recognize. */ + UNKNOWN_TO_SDK_VERSION + } + + /** Creates a succeeded item. */ + public static Item succeeded(String itemId, O output) { + return new Item<>(itemId, Status.SUCCEEDED, output, null); + } + + /** Creates a failed item. */ + public static Item failed(String itemId, DistributedMapError error) { + return new Item<>(itemId, Status.FAILED, null, error); + } + + /** Creates an item whose status this SDK version does not recognize. */ + public static Item unknown(String itemId) { + return new Item<>(itemId, Status.UNKNOWN_TO_SDK_VERSION, null, null); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java new file mode 100644 index 0000000000..421f6e836b --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java @@ -0,0 +1,11 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +/** Terminal status of a distributed map run. */ +public enum DistributedMapStatus { + SUCCEEDED, + FAILED, + STOPPED, + TIMED_OUT +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java new file mode 100644 index 0000000000..45a26b03a4 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import software.amazon.lambda.durable.exception.DistributedMapException; + +/** + * Outcome of a distributed map run, without per-item results. + * + * @param sourceExhausted true only when the service read the complete source, null before run creation + * @param error why the run did not succeed, when the service recorded error details + */ +public record DistributedMapSummary( + DistributedMapStatus status, + Boolean sourceExhausted, + long successCount, + long failureCount, + long unprocessedCount, + String distributedMapRunArn, + DistributedMapError error, + Long totalCount) { + + /** Returns true when any item failed. */ + public boolean hasFailure() { + return failureCount > 0; + } + + /** Throws DistributedMapException when the run did not fully succeed. */ + public void throwIfError() { + if (status != DistributedMapStatus.SUCCEEDED || hasFailure()) { + throw new DistributedMapException(status, failureCount, error); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java index c7273b9cde..8ebf4968e3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java @@ -22,7 +22,8 @@ public enum OperationSubType { PARALLEL_BRANCH(OperationType.CONTEXT, "ParallelBranch"), WAIT_FOR_CALLBACK(OperationType.CONTEXT, "WaitForCallback"), WAIT_FOR_CONDITION(OperationType.STEP, "WaitForCondition"), - WITH_RETRY(OperationType.CONTEXT, "WithRetry"); + WITH_RETRY(OperationType.CONTEXT, "WithRetry"), + DISTRIBUTED_MAP(OperationType.DISTRIBUTED_MAP, "DistributedMap"); private final OperationType operationType; private final String value; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 5cd40820e9..249ed42000 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -610,6 +610,10 @@ public static ErrorObject getErrorObject(Operation operation) { : null; case CONTEXT -> operation.contextDetails() != null ? operation.contextDetails().error() : null; + case DISTRIBUTED_MAP -> + operation.distributedMapDetails() != null + ? operation.distributedMapDetails().error() + : null; default -> null; }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java index 8c299cfa4a..2be2d089dc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java @@ -259,7 +259,7 @@ private Throwable translateException(Operation op, ErrorObject errorObject) { case RUN_IN_CHILD_CONTEXT, WITH_RETRY -> new ChildContextFailedException(op); // the following subtypes should not be able to reach here - case PARALLEL, MAP, WAIT_FOR_CONDITION, STEP, WAIT, CALLBACK, CHAINED_INVOKE -> + case PARALLEL, MAP, WAIT_FOR_CONDITION, STEP, WAIT, CALLBACK, CHAINED_INVOKE, DISTRIBUTED_MAP -> new IllegalStateException("Unexpected sub-type: " + getSubType()); }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java new file mode 100644 index 0000000000..576cb17b74 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java @@ -0,0 +1,440 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvDelimiter; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvHeaderLocation; +import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationType; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapFunctionResponseType; +import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureInclude; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessInclude; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapProcessorConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionMode; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3DestinationConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ListedObjectMetadataConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ListedObjectsConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ObjectConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceType; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapDestination; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DurableExecutionException; +import software.amazon.lambda.durable.model.DistributedMapError; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** + * Durable operation that starts a distributed map run and waits for its terminal outcome. + * + * @param the resolved result type (a summary, or a result with collected items) + */ +public class DistributedMapOperation extends SerializableDurableOperation { + private final DistributedMapOptions options; + private final BiFunction resultBuilder; + + private static final SerDes JSON = new JacksonSerDes(); + private static final long INLINE_SIZE_LIMIT = 1024L * 1024L; + private static final long READER_STATE_LIMIT = 32L * 1024L; + + public DistributedMapOperation( + OperationIdentifier operationIdentifier, + DistributedMapOptions options, + TypeToken resultType, + SerDes serDes, + BiFunction resultBuilder, + DurableContextImpl durableContext) { + super(operationIdentifier, resultType, serDes, durableContext); + this.options = options; + this.resultBuilder = resultBuilder; + } + + /** Starts the operation. */ + @Override + protected void start() { + sendOperationUpdate( + OperationUpdate.builder().action(OperationAction.START).distributedMapOptions(options)); + pollUntilTerminal(); + } + + /** Replays the operation. */ + @Override + protected void replay(Operation existing) { + switch (existing.status()) { + // The start checkpoint records the operation as PENDING, and it reaches STARTED only once the + // service confirms the run was created. A replay in that window waits for the outcome. + case PENDING, STARTED -> pollUntilTerminal(); + case SUCCEEDED, FAILED, TIMED_OUT, STOPPED -> markAlreadyCompleted(); + default -> + throw terminateExecutionWithIllegalDurableOperationException( + "Unexpected distributed map status: " + existing.statusAsString()); + } + } + + /** + * Polls until the operation reaches a terminal status. A poll completes on any update, so a PENDING to STARTED + * update would otherwise end polling while get() is still waiting for the outcome. + */ + private void pollUntilTerminal() { + pollForOperationUpdates().thenAccept(op -> { + if (op != null && (op.status() == OperationStatus.PENDING || op.status() == OperationStatus.STARTED)) { + pollUntilTerminal(); + } + }); + } + + /** Blocks until the operation completes and returns the resolved outcome. */ + @Override + public R get() { + var op = waitForOperationCompletion(); + var runStatus = runStatus(op); + var details = op.distributedMapDetails(); + if (details == null) { + throw terminateExecutionWithIllegalDurableOperationException("DISTRIBUTED_MAP operation reached " + + op.statusAsString() + " but carried no DistributedMapDetails"); + } + // SourceExhausted is required at terminal once a run exists, and is absent only when the operation + // failed before one was created, which is also when the run ARN is absent. + if (details.distributedMapRunArn() != null && details.sourceExhausted() == null) { + throw terminateExecutionWithIllegalDurableOperationException( + "DISTRIBUTED_MAP operation reached " + op.statusAsString() + " but carried no SourceExhausted"); + } + return resultBuilder.apply(runStatus, details); + } + + // An unsuccessful run resolves with its summary, so callers can read the counts. Use + // DistributedMapSummary.throwIfError to opt into raising instead. + private DistributedMapStatus runStatus(Operation op) { + return switch (op.status()) { + case SUCCEEDED -> DistributedMapStatus.SUCCEEDED; + case FAILED -> DistributedMapStatus.FAILED; + case TIMED_OUT -> DistributedMapStatus.TIMED_OUT; + case STOPPED -> DistributedMapStatus.STOPPED; + default -> + throw terminateExecutionWithIllegalDurableOperationException( + "Unexpected distributed map operation status: " + op.statusAsString()); + }; + } + + public static DistributedMapOptions toOptions( + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config, + SerDes defaultSerDes, + boolean collectResults) { + var builder = DistributedMapOptions.builder() + .maxConcurrency(maxConcurrency) + .source(sourceConfig(source, defaultSerDes)) + .processor(processorConfig(processor)); + if (config.destination() != null) { + var destination = destinationConfig(config.destination()); + if (destination != null) { + builder.destination(destination); + } + } + var completion = completionConfig(config); + if (completion != null) { + builder.completionConfig(completion); + } + if (collectResults) { + builder.resultCollection(DistributedMapResultCollectionConfig.builder() + .mode(DistributedMapResultCollectionMode.INLINE) + .build()); + } + if (config.timeout() != null) { + builder.timeoutSeconds((int) config.timeout().toSeconds()); + } + return builder.build(); + } + + private static DistributedMapSourceConfig sourceConfig(DistributedMapSource source, SerDes defaultSerDes) { + var builder = DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.fromValue(source.sourceType().name())); + if (source.maxItems() != null) { + builder.maxItemsToRead(source.maxItems().longValue()); + } + switch (source.sourceType()) { + case INLINE -> builder.inlineSourceConfig(inlineSourceConfig(source, defaultSerDes)); + case S3 -> builder.s3SourceConfig(s3SourceConfig(source.s3())); + case READER_FUNCTION -> builder.readerFunctionSourceConfig(readerConfig(source.reader(), defaultSerDes)); + } + return builder.build(); + } + + private static DistributedMapInlineSourceConfig inlineSourceConfig( + DistributedMapSource source, SerDes defaultSerDes) { + var serdes = source.inlineSerDes() != null ? source.inlineSerDes() : defaultSerDes; + var bodies = new ArrayList(source.inlineItems().size()); + var items = source.inlineItems(); + for (int index = 0; index < items.size(); index++) { + var item = items.get(index); + var body = serdes.serialize(item); + if (body == null) { + // The service requires every item to be a string. The default serdes returns null for a null item, + // so a null item is sent as the JSON literal null. Any other item with no serialized form is an error. + if (item != null) { + throw new IllegalArgumentException("serialization of inline item " + index + " produced no string"); + } + body = "null"; + } + bodies.add(body); + } + var arrayBytes = JSON.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; + if (arrayBytes > INLINE_SIZE_LIMIT) { + throw new IllegalArgumentException( + "inline source exceeds the 1 MB limit (serialized size: " + arrayBytes + " bytes)"); + } + return DistributedMapInlineSourceConfig.builder().items(bodies).build(); + } + + private static DistributedMapS3SourceConfig s3SourceConfig(DistributedMapSource.S3SourceConfig s3) { + var builder = DistributedMapS3SourceConfig.builder() + .type(DistributedMapS3SourceType.fromValue(s3.type().name())) + .bucket(s3.bucket()); + if (s3.expectedBucketOwner() != null) { + builder.expectedBucketOwner(s3.expectedBucketOwner()); + } + if (s3.objectConfig() != null) { + var object = s3.objectConfig(); + var block = DistributedMapS3ObjectConfig.builder() + .key(object.key()) + .format(DistributedMapSourceFormat.fromValue(object.format().name())); + if (object.csvFormat() != null) { + block.csvFormatOptions(csvFormatOptions(object.csvFormat())); + } + builder.objectConfig(block.build()); + } + if (s3.listedObjectMetadataConfig() != null) { + builder.listedObjectMetadataConfig(DistributedMapS3ListedObjectMetadataConfig.builder() + .keyPrefix(s3.listedObjectMetadataConfig().keyPrefix()) + .build()); + } + if (s3.listedObjectsConfig() != null) { + var listed = s3.listedObjectsConfig(); + var block = DistributedMapS3ListedObjectsConfig.builder() + .keyPrefix(listed.keyPrefix()) + .format(DistributedMapSourceFormat.fromValue(listed.format().name())); + if (listed.csvFormat() != null) { + block.csvFormatOptions(csvFormatOptions(listed.csvFormat())); + } + builder.listedObjectsConfig(block.build()); + } + return builder.build(); + } + + private static DistributedMapCsvFormatOptions csvFormatOptions(DistributedMapSource.CsvFormat format) { + var builder = DistributedMapCsvFormatOptions.builder() + .headerLocation(DistributedMapCsvHeaderLocation.fromValue( + format.headerLocation().name())) + .delimiter( + DistributedMapCsvDelimiter.fromValue(format.delimiter().name())); + // Column names go on the wire only for headerless files (GIVEN), expectedColumns (FIRST_ROW) is compile-time + // only. + if (format.headerLocation() == DistributedMapSource.CsvFormat.HeaderLocation.GIVEN + && format.columns() != null) { + builder.headers(format.columns()); + } + return builder.build(); + } + + private static DistributedMapReaderFunctionSourceConfig readerConfig( + DistributedMapSource.ReaderSourceConfig reader, SerDes defaultSerDes) { + var builder = DistributedMapReaderFunctionSourceConfig.builder().functionName(reader.functionName()); + if (reader.initialState() != null) { + var serdes = reader.stateSerDes() != null ? reader.stateSerDes() : defaultSerDes; + var state = serdes.serialize(reader.initialState()); + if (state == null) { + throw new IllegalArgumentException("serialization of reader initialState produced no string"); + } + if (state.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalArgumentException("reader initialState exceeds the 32 KB limit"); + } + builder.initialState(state); + } + return builder.build(); + } + + private static DistributedMapProcessorConfig processorConfig(DistributedMapProcessor processor) { + var builder = DistributedMapProcessorConfig.builder().functionName(processor.functionName()); + if (processor.durableExecutionNamePrefix() != null) { + builder.durableExecutionNamePrefix(processor.durableExecutionNamePrefix()); + } + if (processor.responseMode() == DistributedMapProcessor.ResponseMode.ITEM_OUTCOMES) { + builder.functionResponseTypes(List.of(DistributedMapFunctionResponseType.REPORT_BATCH_ITEM_OUTCOMES)); + } + if (processor.batchSize() != null) { + builder.batchSize(processor.batchSize()); + } + if (processor.retry() != null) { + var retry = processor.retry(); + if (retry.maxRetryAttempts() != null) { + builder.maxRetryAttempts(retry.maxRetryAttempts()); + } + if (retry.maxRetryDuration() != null) { + builder.maxRetryDurationSeconds((int) retry.maxRetryDuration().toSeconds()); + } + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig destinationConfig( + software.amazon.lambda.durable.config.DistributedMapDestination destination) { + if (destination.onSuccess() == null && destination.onFailure() == null) { + return null; // omit an empty destination config from the wire + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig.builder(); + DistributedMapDestination.Success onSuccess = destination.onSuccess(); + if (onSuccess != null) { + var include = new ArrayList(); + if (onSuccess.includeInput()) { + include.add(DistributedMapOnSuccessInclude.INPUT); + } + if (onSuccess.includeOutput()) { + include.add(DistributedMapOnSuccessInclude.OUTPUT); + } + builder.onSuccess(DistributedMapOnSuccessConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig( + s3Destination(onSuccess.bucket(), onSuccess.prefix(), onSuccess.expectedBucketOwner())) + .build()); + } + DistributedMapDestination.Failure onFailure = destination.onFailure(); + if (onFailure != null) { + var include = new ArrayList(); + if (onFailure.includeInput()) { + include.add(DistributedMapOnFailureInclude.INPUT); + } + if (onFailure.includeError()) { + include.add(DistributedMapOnFailureInclude.ERROR); + } + builder.onFailure(DistributedMapOnFailureConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig( + s3Destination(onFailure.bucket(), onFailure.prefix(), onFailure.expectedBucketOwner())) + .build()); + } + return builder.build(); + } + + private static DistributedMapS3DestinationConfig s3Destination( + String bucket, String keyPrefix, String expectedBucketOwner) { + var builder = DistributedMapS3DestinationConfig.builder().bucket(bucket).keyPrefix(keyPrefix); + if (expectedBucketOwner != null) { + builder.expectedBucketOwner(expectedBucketOwner); + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig completionConfig( + DistributedMapConfig config) { + var completion = config.completionConfig(); + if (completion == null) { + return null; + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig.builder(); + if (completion.toleratedFailureCount() != null) { + builder.toleratedFailureCount(completion.toleratedFailureCount()); + } + if (completion.toleratedFailurePercentage() != null) { + builder.toleratedFailurePercentage( + completion.toleratedFailurePercentage().floatValue()); + } + if (completion.minimumSampleSize() != null) { + builder.minimumSampleSize(completion.minimumSampleSize()); + } + return builder.build(); + } + + public static DistributedMapSummary toSummary(DistributedMapStatus runStatus, DistributedMapDetails details) { + // The service sends all three counts at every terminal state, and TotalCount once the run succeeded. + // Reading an absent count as zero would report a run that failed items as having none, so a missing + // count is a contract violation rather than a default. + var successCount = requireCount(details.successCount(), "SuccessCount"); + var failureCount = requireCount(details.failureCount(), "FailureCount"); + var unprocessedCount = requireCount(details.unprocessedCount(), "UnprocessedCount"); + if (runStatus == DistributedMapStatus.SUCCEEDED && details.totalCount() == null) { + throw new DurableExecutionException("missing distributed map TotalCount from backend"); + } + return new DistributedMapSummary( + runStatus, + details.sourceExhausted(), + successCount, + failureCount, + unprocessedCount, + details.distributedMapRunArn(), + toError(details.error()), + details.totalCount()); + } + + private static long requireCount(Long count, String field) { + if (count == null) { + throw new DurableExecutionException("missing distributed map " + field + " from backend"); + } + return count; + } + + public static DistributedMapResult toResult( + DistributedMapStatus runStatus, DistributedMapDetails details, TypeToken resultType, SerDes serdes) { + var summary = toSummary(runStatus, details); + var items = new ArrayList>(); + if (details.results() != null) { + for (var wire : details.results()) { + if (wire.itemId() == null || wire.itemId().isEmpty()) { + throw new DurableExecutionException("missing distributed map item id from backend"); + } + var status = wire.statusAsString(); + if (status == null) { + throw new DurableExecutionException("distributed map item " + wire.itemId() + + " has no status, which the service model" + " requires"); + } + if (DistributedMapResult.Item.Status.SUCCEEDED.name().equals(status)) { + O output = wire.output() != null ? serdes.deserialize(wire.output(), resultType) : null; + items.add(DistributedMapResult.Item.succeeded(wire.itemId(), output)); + } else if (DistributedMapResult.Item.Status.FAILED.name().equals(status)) { + items.add(DistributedMapResult.Item.failed(wire.itemId(), toError(wire.error()))); + } else { + // A status added after this SDK version. Keep the item so the other items survive. + items.add(DistributedMapResult.Item.unknown(wire.itemId())); + } + } + } + return new DistributedMapResult<>(summary, items); + } + + private static DistributedMapError toError(ErrorObject error) { + return error == null + ? null + : new DistributedMapError( + error.errorType(), error.errorMessage(), error.errorData(), error.stackTrace()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java new file mode 100644 index 0000000000..92608e682e --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.util; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.regex.Pattern; + +/** Shared validation and S3 URI parsing for distributed map configuration. */ +public final class DistributedMapValidation { + private static final Pattern ACCOUNT_ID = Pattern.compile("\\d{12}"); + private static final String S3_SCHEME = "s3://"; + private static final int MAX_FUNCTION_NAME_LENGTH = 170; + private static final int MAX_CSV_COLUMNS = 1000; + private static final int MAX_CSV_COLUMN_LENGTH = 256; + private static final int MAX_DESTINATION_KEY_PREFIX_BYTES = 900; + + private DistributedMapValidation() {} + + public static void validateFunctionName(String functionName) { + if (functionName == null || functionName.isBlank()) { + throw new IllegalArgumentException("function name cannot be empty"); + } + if (functionName.length() > MAX_FUNCTION_NAME_LENGTH) { + throw new IllegalArgumentException( + "function name must be at most 170 characters, got: " + functionName.length()); + } + } + + public static void validateBucketOwner(String expectedBucketOwner) { + if (expectedBucketOwner != null + && !ACCOUNT_ID.matcher(expectedBucketOwner).matches()) { + throw new IllegalArgumentException( + "expectedBucketOwner must be a 12-digit account id, got: " + expectedBucketOwner); + } + } + + /** + * Rejects a destination key prefix the service would reject. The limit is 900 UTF-8 bytes, including a trailing + * slash when the URI has one, and it is measured in bytes rather than characters, so a multi-byte prefix under 900 + * characters can still exceed it. The service applies the same limit when the start checkpoint is written, so + * checking it here fails the call that built the destination instead of the operation. + */ + public static void validateDestinationPrefix(String prefix) { + if (prefix == null) { + return; + } + var bytes = prefix.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_DESTINATION_KEY_PREFIX_BYTES) { + throw new IllegalArgumentException("destination key prefix must be at most " + + MAX_DESTINATION_KEY_PREFIX_BYTES + " UTF-8 bytes, got: " + bytes); + } + } + + public static void validateColumns(String parameterName, List columns) { + if (columns == null) { + return; + } + if (columns.isEmpty()) { + throw new IllegalArgumentException(parameterName + " cannot be empty"); + } + if (columns.size() > MAX_CSV_COLUMNS) { + throw new IllegalArgumentException( + parameterName + " must hold at most " + MAX_CSV_COLUMNS + " names, got: " + columns.size()); + } + for (var column : columns) { + if (column != null && column.length() > MAX_CSV_COLUMN_LENGTH) { + throw new IllegalArgumentException("each " + parameterName + " name must be at most " + + MAX_CSV_COLUMN_LENGTH + " characters, got: " + column.length()); + } + } + if (new HashSet<>(columns).size() != columns.size()) { + throw new IllegalArgumentException(parameterName + " must not contain duplicates"); + } + } + + public static ParsedS3Uri parseS3Uri(String uri) { + if (uri == null || !uri.startsWith(S3_SCHEME)) { + throw new IllegalArgumentException("S3 URI must start with s3://, got: " + uri); + } + var remainder = uri.substring(S3_SCHEME.length()); + var slash = remainder.indexOf('/'); + var bucket = slash < 0 ? remainder : remainder.substring(0, slash); + var path = slash < 0 ? null : remainder.substring(slash + 1); + if (bucket.isEmpty()) { + throw new IllegalArgumentException("S3 URI must contain a bucket, got: " + uri); + } + return new ParsedS3Uri(bucket, path == null || path.isEmpty() ? null : path); + } + + public record ParsedS3Uri(String bucket, String path) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java new file mode 100644 index 0000000000..71003c3544 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class DistributedMapConfigTest { + + @Test + void failureCount_setsCount() { + var config = DistributedMapConfig.CompletionConfig.failureCount(3); + + assertEquals(3, config.toleratedFailureCount()); + assertNull(config.toleratedFailurePercentage()); + assertNull(config.minimumSampleSize()); + } + + @Test + void failureCount_withZero_shouldPass() { + assertEquals(0, DistributedMapConfig.CompletionConfig.failureCount(0).toleratedFailureCount()); + } + + @Test + void failurePercentage_setsPercentage() { + var config = DistributedMapConfig.CompletionConfig.failurePercentage(25); + + assertNull(config.toleratedFailureCount()); + assertEquals(25, config.toleratedFailurePercentage()); + assertNull(config.minimumSampleSize()); + } + + @Test + void failurePercentage_withMinimumSampleSize_setsBoth() { + var config = DistributedMapConfig.CompletionConfig.failurePercentage(25, 10); + + assertEquals(25, config.toleratedFailurePercentage()); + assertEquals(10, config.minimumSampleSize()); + } + + @Test + void failurePercentage_atBoundaries_shouldPass() { + assertEquals( + 0, DistributedMapConfig.CompletionConfig.failurePercentage(0).toleratedFailurePercentage()); + assertEquals( + 100, + DistributedMapConfig.CompletionConfig.failurePercentage(100).toleratedFailurePercentage()); + } + + @Test + void countAndPercentage_mutuallyExclusive_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(3, 25, null)); + assertEquals( + "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive", exception.getMessage()); + } + + @Test + void minimumSampleSize_withoutPercentage_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(3, null, 5)); + assertEquals("minimumSampleSize is only valid with toleratedFailurePercentage", exception.getMessage()); + } + + @Test + void completionConfig_onlyMinimumSampleSize_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(null, null, 5)); + assertEquals("minimumSampleSize is only valid with toleratedFailurePercentage", exception.getMessage()); + } + + @Test + void failureCount_negative_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failureCount(-1)); + assertEquals("toleratedFailureCount must be non-negative, got: -1", exception.getMessage()); + } + + @Test + void failurePercentage_negative_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(-1)); + assertEquals("toleratedFailurePercentage must be between 0 and 100, got: -1", exception.getMessage()); + } + + @Test + void failurePercentage_aboveHundred_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(101)); + assertEquals("toleratedFailurePercentage must be between 0 and 100, got: 101", exception.getMessage()); + } + + @Test + void minimumSampleSize_belowOne_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(25, 0)); + assertEquals("minimumSampleSize must be at least 1, got: 0", exception.getMessage()); + } + + @Test + void emptyCompletionConfig_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(null, null, null)); + assertEquals( + "a completion config must set toleratedFailureCount or toleratedFailurePercentage", + exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java new file mode 100644 index 0000000000..d2680e7c79 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java @@ -0,0 +1,216 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; + +class DistributedMapDestinationTest { + + @Test + void successes_defaultsIncludeOutputOnly() { + var destination = DistributedMapDestination.s3Successes("s3://bucket/results"); + + assertEquals("bucket", destination.bucket()); + assertEquals("results", destination.prefix()); + assertFalse(destination.includeInput()); + assertTrue(destination.includeOutput()); + assertNull(destination.expectedBucketOwner()); + } + + @Test + void successes_withoutPath_prefixIsEmpty() { + var destination = DistributedMapDestination.s3Successes("s3://bucket"); + + assertEquals("bucket", destination.bucket()); + assertEquals("", destination.prefix()); + } + + @Test + void successes_withAllArguments() { + var destination = DistributedMapDestination.s3Successes("s3://bucket/out", true, true, "123456789012"); + + assertEquals("bucket", destination.bucket()); + assertEquals("out", destination.prefix()); + assertTrue(destination.includeInput()); + assertTrue(destination.includeOutput()); + assertEquals("123456789012", destination.expectedBucketOwner()); + } + + @Test + void successes_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/out", false, false, null)); + assertEquals("success destination must include at least one of input or output", exception.getMessage()); + } + + @Test + void successDestination_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Success("bucket", "", false, false, null)); + assertEquals("success destination must include at least one of input or output", exception.getMessage()); + } + + @ParameterizedTest + @NullAndEmptySource + void successDestination_nullOrEmptyBucket_shouldThrow(String bucket) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Success(bucket, "", false, true, null)); + assertEquals("success destination bucket cannot be empty", exception.getMessage()); + } + + @Test + void failures_defaultsIncludeInputAndError() { + var destination = DistributedMapDestination.s3Failures("s3://bucket/errors"); + + assertEquals("bucket", destination.bucket()); + assertEquals("errors", destination.prefix()); + assertTrue(destination.includeInput()); + assertTrue(destination.includeError()); + assertNull(destination.expectedBucketOwner()); + } + + @Test + void failures_withAllArguments() { + var destination = DistributedMapDestination.s3Failures("s3://bucket/err", true, false, "123456789012"); + + assertEquals("bucket", destination.bucket()); + assertEquals("err", destination.prefix()); + assertTrue(destination.includeInput()); + assertFalse(destination.includeError()); + assertEquals("123456789012", destination.expectedBucketOwner()); + } + + @Test + void failures_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Failures("s3://bucket/err", false, false, null)); + assertEquals("failure destination must include at least one of input or error", exception.getMessage()); + } + + @Test + void failureDestination_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Failure("bucket", "", false, false, null)); + assertEquals("failure destination must include at least one of input or error", exception.getMessage()); + } + + @ParameterizedTest + @NullAndEmptySource + void failureDestination_nullOrEmptyBucket_shouldThrow(String bucket) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Failure(bucket, "", true, true, null)); + assertEquals("failure destination bucket cannot be empty", exception.getMessage()); + } + + @Test + void successes_invalidBucketOwner_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/out", false, true, "123")); + assertEquals("expectedBucketOwner must be a 12-digit account id, got: 123", exception.getMessage()); + } + + @Test + void successes_invalidUri_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapDestination.s3Successes("bucket/out")); + assertEquals("S3 URI must start with s3://, got: bucket/out", exception.getMessage()); + } + + @Test + void destinationConfig_onSuccess_routesSuccessOnly() { + var success = DistributedMapDestination.s3Successes("s3://bucket/ok"); + var config = DistributedMapDestination.of(success); + + assertSame(success, config.onSuccess()); + assertNull(config.onFailure()); + } + + @Test + void destinationConfig_onFailure_routesFailureOnly() { + var failure = DistributedMapDestination.s3Failures("s3://bucket/bad"); + var config = DistributedMapDestination.of(failure); + + assertNull(config.onSuccess()); + assertSame(failure, config.onFailure()); + } + + @Test + void destinationConfig_of_routesBoth() { + var success = DistributedMapDestination.s3Successes("s3://bucket/ok"); + var failure = DistributedMapDestination.s3Failures("s3://bucket/bad"); + var config = DistributedMapDestination.of(success, failure); + + assertSame(success, config.onSuccess()); + assertSame(failure, config.onFailure()); + } + + // ------------------------------------------------------------------ + // key prefix length + // ------------------------------------------------------------------ + + private static String repeat(String unit, int times) { + return unit.repeat(times); + } + + @Test + void successes_prefixAtTheByteLimit_isAccepted() { + var prefix = repeat("a", 900); + + assertEquals( + prefix, + DistributedMapDestination.s3Successes("s3://bucket/" + prefix).prefix()); + } + + @Test + void successes_prefixOverTheByteLimit_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + repeat("a", 901))); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } + + @Test + void failures_prefixOverTheByteLimit_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Failures("s3://bucket/" + repeat("a", 901))); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } + + @Test + void successes_prefixCountsUtf8BytesNotCharacters() { + // This character is two UTF-8 bytes, so 450 of them are exactly 900 bytes while being only 450 characters. + var atTheLimit = repeat("\u00e9", 450); + assertEquals(450, atTheLimit.length()); + assertEquals( + atTheLimit, + DistributedMapDestination.s3Successes("s3://bucket/" + atTheLimit) + .prefix()); + + var overTheLimit = atTheLimit + "\u00e9"; + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + overTheLimit)); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 902", exception.getMessage()); + } + + @Test + void successes_prefixLengthCountsTheTrailingSlash() { + // 900 characters plus the trailing slash is 901 bytes, so the slash is what pushes it over. + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + repeat("a", 900) + "/")); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java new file mode 100644 index 0000000000..67a8481e67 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class DistributedMapProcessorRetryConfigTest { + + @Test + void build_defaultsAreNull() { + var retry = DistributedMapProcessorRetryConfig.builder().build(); + + assertNull(retry.maxRetryAttempts()); + assertNull(retry.maxRetryDuration()); + } + + @Test + void maxRetryAttempts_zero_isStored() { + var retry = + DistributedMapProcessorRetryConfig.builder().maxRetryAttempts(0).build(); + + assertEquals(0, retry.maxRetryAttempts()); + } + + @Test + void maxRetryAttempts_unlimited_isStored() { + var retry = DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .build(); + + assertEquals(-1, retry.maxRetryAttempts()); + } + + @Test + void maxRetryAttempts_otherNegative_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(-2)); + assertEquals( + "maxRetryAttempts must be non-negative or DistributedMapProcessorRetryConfig.UNLIMITED, got: -2", + exception.getMessage()); + } + + @Test + void maxRetryDuration_atBoundaries_shouldPass() { + assertEquals( + Duration.ofMinutes(1), + DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(1)) + .build() + .maxRetryDuration()); + assertEquals( + Duration.ofHours(6), + DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofHours(6)) + .build() + .maxRetryDuration()); + } + + @Test + void maxRetryDuration_belowMinimum_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofSeconds(59))); + assertEquals("maxRetryDuration must be between 1 minute and 6 hours, got: 59s", exception.getMessage()); + } + + @Test + void maxRetryDuration_aboveMaximum_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofHours(7))); + assertEquals("maxRetryDuration must be between 1 minute and 6 hours, got: 25200s", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java new file mode 100644 index 0000000000..3a84e923f5 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java @@ -0,0 +1,176 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class DistributedMapProcessorTest { + + @Test + void batch_setsBatchMode() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn").build(); + + assertEquals("fn", processor.functionName()); + assertEquals(DistributedMapProcessor.ResponseMode.BATCH, processor.responseMode()); + } + + @Test + void itemOutcomes_setsItemOutcomesMode() { + var processor = DistributedMapProcessor.reportItemOutcomes("fn").build(); + + assertEquals(DistributedMapProcessor.ResponseMode.ITEM_OUTCOMES, processor.responseMode()); + } + + @Test + void build_defaultsAreNull() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn").build(); + + assertNull(processor.batchSize()); + assertNull(processor.retry()); + assertNull(processor.durableExecutionNamePrefix()); + } + + @Test + void batchSize_atBoundaries_shouldPass() { + assertEquals( + 1, + DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(1) + .build() + .batchSize()); + assertEquals( + 10000, + DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(10000) + .build() + .batchSize()); + } + + @Test + void batchSize_belowMinimum_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(0) + .build()); + assertEquals("batchSize must be between 1 and 10000, got: 0", exception.getMessage()); + } + + @Test + void batchSize_aboveMaximum_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(10001) + .build()); + assertEquals("batchSize must be between 1 and 10000, got: 10001", exception.getMessage()); + } + + @Test + void durableExecutionNamePrefix_atBoundaries_shouldPass() { + assertEquals( + "a", + DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("a") + .build() + .durableExecutionNamePrefix()); + var thirtySix = "a".repeat(36); + assertEquals( + thirtySix, + DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix(thirtySix) + .build() + .durableExecutionNamePrefix()); + } + + @Test + void durableExecutionNamePrefix_empty_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("") + .build()); + assertEquals("durableExecutionNamePrefix must be between 1 and 36 characters, got: 0", exception.getMessage()); + } + + @Test + void durableExecutionNamePrefix_tooLong_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("a".repeat(37)) + .build()); + assertEquals("durableExecutionNamePrefix must be between 1 and 36 characters, got: 37", exception.getMessage()); + } + + @Test + void retryFields_areStored() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(3) + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + + assertEquals(3, processor.retry().maxRetryAttempts()); + assertEquals(Duration.ofMinutes(5), processor.retry().maxRetryDuration()); + } + + @Test + void unlimitedRetryAttempts_isStored() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .build()) + .build(); + + assertEquals( + DistributedMapProcessorRetryConfig.UNLIMITED, processor.retry().maxRetryAttempts()); + } + + @Test + void functionName_empty_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("") + .build()); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void functionName_null_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome(null) + .build()); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void functionName_tooLong_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("a".repeat(171)) + .build()); + assertEquals("function name must be at most 170 characters, got: 171", exception.getMessage()); + } + + @Test + void functionName_atMaxLength_shouldPass() { + var name = "a".repeat(170); + assertEquals( + name, DistributedMapProcessor.reportBatchOutcome(name).build().functionName()); + } + + @Test + void build_allFieldsSet() { + var processor = DistributedMapProcessor.reportItemOutcomes("fn") + .batchSize(50) + .durableExecutionNamePrefix("job") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + + assertEquals("fn", processor.functionName()); + assertEquals(50, processor.batchSize()); + assertEquals("job", processor.durableExecutionNamePrefix()); + assertEquals(Duration.ofMinutes(5), processor.retry().maxRetryDuration()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java new file mode 100644 index 0000000000..75d4d5789a --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java @@ -0,0 +1,415 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +class DistributedMapSourceTest { + + @Test + void inline_storesItems() { + var source = DistributedMapSource.inline(List.of("a", "b")); + + assertEquals(DistributedMapSource.SourceType.INLINE, source.sourceType()); + assertEquals(List.of("a", "b"), source.inlineItems()); + assertNull(source.inlineSerDes()); + assertNull(source.s3()); + assertNull(source.reader()); + } + + @Test + void inline_withSerDes_storesSerDes() { + var serDes = new JacksonSerDes(); + + var source = DistributedMapSource.inline(List.of("a"), serDes); + + assertEquals(DistributedMapSource.SourceType.INLINE, source.sourceType()); + assertEquals(List.of("a"), source.inlineItems()); + assertSame(serDes, source.inlineSerDes()); + } + + @Test + void inline_acceptsNullItem() { + var source = DistributedMapSource.inline(java.util.Arrays.asList("a", null)); + assertEquals(java.util.Arrays.asList("a", null), source.inlineItems()); + } + + @Test + void inline_null_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.inline(null)); + assertEquals("items cannot be null", exception.getMessage()); + } + + @Test + void inline_rejectsUnorderedSet() { + var unordered = new HashSet<>(List.of("a", "b")); + + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.inline(unordered)); + assertEquals("items must have deterministic iteration order", exception.getMessage()); + } + + @Test + void inline_allowsLinkedHashSet() { + var ordered = new LinkedHashSet<>(List.of("a", "b")); + + var source = DistributedMapSource.inline(ordered); + + assertEquals(List.of("a", "b"), source.inlineItems()); + } + + @Test + void s3JsonLines_setsBucketKeyFormat() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + assertEquals(DistributedMapSource.SourceType.S3, source.sourceType()); + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.jsonl", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.JSON_LINES, s3.objectConfig().format()); + assertNull(s3.listedObjectMetadataConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3JsonLines_withoutKey_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3JsonLines("s3://bucket-only")); + assertEquals("s3JsonLines requires an S3 object key", exception.getMessage()); + } + + @Test + void s3JsonArray_setsFormat() { + DistributedMapSource source = DistributedMapSource.s3JsonArray("s3://bucket/data.json"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.json", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.JSON_ARRAY, s3.objectConfig().format()); + } + + @Test + void s3JsonArray_withoutKey_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3JsonArray("s3://bucket-only")); + assertEquals("s3JsonArray requires an S3 object key", exception.getMessage()); + } + + @Test + void s3Csv_setsFormatAndCsvFormat() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")); + + DistributedMapSource source = DistributedMapSource.s3Csv("s3://bucket/data.csv", format); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.csv", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.CSV, s3.objectConfig().format()); + assertSame(format, s3.objectConfig().csvFormat()); + } + + @Test + void s3Csv_withoutKey_throws() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3Csv("s3://bucket-only", format)); + assertEquals("s3Csv requires an S3 object key", exception.getMessage()); + } + + @Test + void s3Objects_setsListedObjectMetadataWithPrefix() { + DistributedMapSource source = DistributedMapSource.s3Objects("s3://bucket/data/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECT_METADATA, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data/", s3.listedObjectMetadataConfig().keyPrefix()); + assertNull(s3.objectConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3Objects_wholeBucket_allowsEmptyPrefix() { + DistributedMapSource source = DistributedMapSource.s3Objects("s3://bucket"); + + var s3 = source.s3(); + assertEquals("bucket", s3.bucket()); + assertEquals("", s3.listedObjectMetadataConfig().keyPrefix()); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECT_METADATA, s3.type()); + } + + @Test + void s3FlattenedJsonLines_setsListedObjects() { + DistributedMapSource source = DistributedMapSource.s3FlattenedJsonLines("s3://bucket/prefix/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("prefix/", s3.listedObjectsConfig().keyPrefix()); + assertEquals( + DistributedMapSource.Format.JSON_LINES, s3.listedObjectsConfig().format()); + assertNull(s3.objectConfig()); + } + + @Test + void s3FlattenedJsonArray_setsListedObjects() { + DistributedMapSource source = DistributedMapSource.s3FlattenedJsonArray("s3://bucket/prefix/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals( + DistributedMapSource.Format.JSON_ARRAY, s3.listedObjectsConfig().format()); + } + + @Test + void s3FlattenedCsv_setsListedObjects() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + DistributedMapSource source = DistributedMapSource.s3FlattenedCsv("s3://bucket/prefix/", format); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals(DistributedMapSource.Format.CSV, s3.listedObjectsConfig().format()); + assertSame(format, s3.listedObjectsConfig().csvFormat()); + } + + @Test + void s3FlattenedCsv_wholeBucket_allowsEmptyPrefix() { + DistributedMapSource source = + DistributedMapSource.s3FlattenedCsv("s3://bucket", DistributedMapSource.CsvFormat.firstRow()); + + assertEquals("", source.s3().listedObjectsConfig().keyPrefix()); + } + + @Test + void reader_storesFunctionName() { + DistributedMapSource source = DistributedMapSource.reader("my-reader"); + + assertEquals(DistributedMapSource.SourceType.READER_FUNCTION, source.sourceType()); + assertEquals("my-reader", source.reader().functionName()); + assertNull(source.reader().initialState()); + assertNull(source.reader().stateSerDes()); + } + + @Test + void reader_withInitialState_storesState() { + var serDes = new JacksonSerDes(); + + DistributedMapSource source = DistributedMapSource.reader("my-reader", "start-token", serDes); + + assertEquals("my-reader", source.reader().functionName()); + assertEquals("start-token", source.reader().initialState()); + assertSame(serDes, source.reader().stateSerDes()); + } + + @Test + void reader_emptyFunctionName_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.reader("")); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void maxItems_setsMaxItems() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .maxItems(500); + + assertEquals(500L, source.maxItems()); + } + + @Test + void maxItems_zero_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.maxItems(0)); + assertEquals("maxItems must be between 1 and 100000000, got: 0", exception.getMessage()); + } + + @Test + void maxItems_atTheLimit_stores() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .maxItems(100_000_000L); + + assertEquals(100_000_000L, source.maxItems()); + } + + @Test + void maxItems_aboveTheLimit_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.maxItems(100_000_001L)); + assertEquals("maxItems must be between 1 and 100000000, got: 100000001", exception.getMessage()); + } + + @Test + void expectedBucketOwner_twelveDigits_stores() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .expectedBucketOwner("123456789012"); + + assertEquals("123456789012", source.s3().expectedBucketOwner()); + } + + @Test + void expectedBucketOwner_invalid_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.expectedBucketOwner("123")); + assertEquals("expectedBucketOwner must be a 12-digit account id, got: 123", exception.getMessage()); + } + + @Test + void expectedBucketOwner_onNonS3Source_throws() { + var source = DistributedMapSource.inline(List.of("a")); + + var exception = assertThrows(IllegalStateException.class, () -> source.expectedBucketOwner("123456789012")); + assertEquals("expectedBucketOwner only applies to S3 sources", exception.getMessage()); + } + + @Test + void s3JsonLines_missingScheme_throws() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.s3JsonLines("bucket/key")); + assertEquals("S3 URI must start with s3://, got: bucket/key", exception.getMessage()); + } + + @Test + void s3Objects_missingBucket_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.s3Objects("s3:///key")); + assertEquals("S3 URI must contain a bucket, got: s3:///key", exception.getMessage()); + } + + @Test + void expectedColumns_setsFirstRowAndColumns() { + var format = DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "b")); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void headers_setsGivenAndColumns() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void firstRow_hasNoColumns() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, format.headerLocation()); + assertNull(format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void withDelimiter_setsDelimiter() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")) + .withDelimiter(DistributedMapSource.CsvDelimiter.PIPE); + + assertEquals(DistributedMapSource.CsvDelimiter.PIPE, format.delimiter()); + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + } + + @Test + void constructor_nullDelimiter_defaultsToComma() { + var format = new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, List.of("a"), null); + + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void expectedColumns_empty_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.expectedColumns(List.of())); + assertEquals("expectedColumns cannot be empty", exception.getMessage()); + } + + @Test + void expectedColumns_aboveTheColumnLimit_throws() { + var columns = + java.util.stream.IntStream.range(0, 1001).mapToObj(i -> "c" + i).toList(); + + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.expectedColumns(columns)); + assertEquals("expectedColumns must hold at most 1000 names, got: 1001", exception.getMessage()); + } + + @Test + void headers_nameAboveTheLengthLimit_throws() { + var columns = List.of("a".repeat(257)); + + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(columns)); + assertEquals("each headers name must be at most 256 characters, got: 257", exception.getMessage()); + } + + @Test + void expectedColumns_duplicates_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "a"))); + assertEquals("expectedColumns must not contain duplicates", exception.getMessage()); + } + + @Test + void headers_empty_throws() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(List.of())); + assertEquals("headers cannot be empty", exception.getMessage()); + } + + @Test + void headers_duplicates_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(List.of("a", "a"))); + assertEquals("headers must not contain duplicates", exception.getMessage()); + } + + @Test + void constructor_givenWithoutColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, null, null)); + assertEquals("CsvFormat with headerLocation GIVEN requires headers", exception.getMessage()); + } + + @Test + void constructor_givenWithDuplicateColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, List.of("a", "a"), null)); + assertEquals("headers must not contain duplicates", exception.getMessage()); + } + + @Test + void constructor_firstRowWithEmptyColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, List.of(), null)); + assertEquals("expectedColumns cannot be empty", exception.getMessage()); + } + + @Test + void csvFormat_withoutHeaderLocation_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapSource.CsvFormat(null, null, null)); + assertEquals("CsvFormat requires a headerLocation", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java b/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java new file mode 100644 index 0000000000..dea8382a63 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java @@ -0,0 +1,194 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.context; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +/** Exercises the ctx.distributedMap entrypoint overloads on DurableContextImpl. */ +class DurableContextDistributedMapTest { + + private static final String EXECUTION_NAME = "349beff4-a89d-4bc8-a56f-af7a8af67a5f"; + private static final String EXECUTION_OP_ID = "20dae574-53da-37a1-bfd5-b0e2e6ec715d"; + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/" + + EXECUTION_NAME + "/" + EXECUTION_OP_ID; + + private static final Operation EXECUTION_OP = Operation.builder() + .id(EXECUTION_OP_ID) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .build(); + + // The first operation created inside a context always hashes the counter value "1". + private static final String FIRST_OP_ID = TestUtils.hashOperationId("1"); + + private DurableContext createTestContext() { + return createTestContext(List.of()); + } + + private DurableContext createTestContext(List initialOperations) { + var client = TestUtils.createMockClient(); + var operations = new ArrayList<>(List.of(EXECUTION_OP)); + operations.addAll(initialOperations); + var initialExecutionState = + CheckpointUpdatedExecutionState.builder().operations(operations).build(); + var executionManager = new ExecutionManager( + new DurableExecutionInput(EXECUTION_ARN, "test-token", initialExecutionState), + DurableConfig.builder().withDurableExecutionClient(client).build(), + null); + var root = DurableContextImpl.createRootContext( + executionManager, + DurableConfig.builder().withDurableExecutionClient(client).build(), + null); + executionManager.registerActiveThread(null); + executionManager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + return root; + } + + private DistributedMapSource source() { + return DistributedMapSource.inline(List.of("a")); + } + + private DistributedMapProcessor processor() { + return DistributedMapProcessor.reportBatchOutcome("test-processor").build(); + } + + // A completed distributed map operation so a replay resolves without suspending. + private Operation completedDmapOp() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L) + .build(); + return Operation.builder() + .id(FIRST_OP_ID) + .status(OperationStatus.SUCCEEDED) + .distributedMapDetails(details) + .build(); + } + + @Test + void testMaxConcurrencyZeroThrows() { + var context = createTestContext(); + + var ex = assertThrows( + IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), 0)); + assertTrue(ex.getMessage().contains("maxConcurrency must be greater than zero")); + } + + @Test + void testMaxConcurrencyNegativeThrows() { + var context = createTestContext(); + + assertThrows(IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), -1)); + } + + @Test + void testResultSerDesOnSummaryCallThrows() { + var context = createTestContext(); + // resultSerDes is only valid on a result-collecting overload, a summary call must reject it. + var config = + DistributedMapConfig.builder().resultSerDes(new JacksonSerDes()).build(); + + var ex = assertThrows( + IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), 2, config)); + assertTrue(ex.getMessage().contains("resultSerDes")); + } + + @Test + void testSummaryOverloadReturnsSummary() { + var context = createTestContext(List.of(completedDmapOp())); + + var summary = context.distributedMap("m", source(), processor(), 2); + + assertNotNull(summary); + assertEquals(1, summary.successCount()); + } + + @Test + void testResultOverloadReturnsResult() { + var context = createTestContext(List.of(completedDmapOp())); + + DistributedMapResult result = context.distributedMap( + "m", + source(), + processor(), + 2, + String.class, + DistributedMapConfig.builder().build()); + + assertNotNull(result); + assertNotNull(result.items()); + assertEquals(1, result.summary().successCount()); + } + + @Test + void testClassOverloadDelegatesToTypeToken() { + // The Class overload forwards to the TypeToken overload, so both must resolve identically. + var classContext = createTestContext(List.of(completedDmapOp())); + DistributedMapResult fromClass = classContext.distributedMap( + "m", + source(), + processor(), + 2, + String.class, + DistributedMapConfig.builder().build()); + + var tokenContext = createTestContext(List.of(completedDmapOp())); + DistributedMapResult fromToken = tokenContext.distributedMap( + "m", + source(), + processor(), + 2, + new TypeToken() {}, + DistributedMapConfig.builder().build()); + + assertEquals(fromToken.summary(), fromClass.summary()); + assertEquals(fromToken.items(), fromClass.items()); + } + + @Test + void testSummaryConfigOptional() { + var context = createTestContext(List.of(completedDmapOp())); + + // The no-config summary overload uses an empty config internally. + DistributedMapSummary summary = context.distributedMap("m", source(), processor(), 2); + + assertNotNull(summary); + } + + @Test + void testResultConfigOptional() { + var context = createTestContext(List.of(completedDmapOp())); + + // The no-config result overload uses an empty config internally. + DistributedMapResult result = context.distributedMap("m", source(), processor(), 2, String.class); + + assertNotNull(result); + assertNotNull(result.items()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java new file mode 100644 index 0000000000..f93b8ed64a --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java @@ -0,0 +1,140 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** + * Tests that the durable processor wrappers run under the durable execution config the caller supplies. The config is + * how a durable processor registers plugins or supplies its own client, so the wrapper must not force the default. + */ +class DistributedMapDurableHandlersTest { + private static final String EXECUTION_OP_ID = "f3d7b0c0-1234-5678-90ab-cdef12345678"; + private static final String EXECUTION_ARN = + "arn:aws:lambda:us-east-1:123456789012:function:test" + "/durable-execution/exec-name/" + EXECUTION_OP_ID; + + /** Counts deserialize calls, so a test can prove the config's own serdes read the event. */ + private static final class RecordingSerDes implements SerDes { + private final SerDes delegate = new JacksonSerDes(); + private final AtomicInteger deserializeCount = new AtomicInteger(); + + @Override + public String serialize(Object value) { + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + deserializeCount.incrementAndGet(); + return delegate.deserialize(data, typeToken); + } + } + + private static DurableConfig configWith(SerDes serDes) { + return DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withSerDes(serDes) + .build(); + } + + // A first invocation of a durable function: one STARTED execution operation carrying the processor batch. + private static DurableExecutionInput durableInput(ProcessorEvent event) { + var executionOp = Operation.builder() + .id(EXECUTION_OP_ID) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.parse("2026-08-15T00:00:00Z")) + .executionDetails(ExecutionDetails.builder() + .inputPayload(new JacksonSerDes().serialize(event)) + .build()) + .build(); + return new DurableExecutionInput( + EXECUTION_ARN, + "token-1", + CheckpointUpdatedExecutionState.builder() + .operations(List.of(executionOp)) + .build()); + } + + private static ProcessorEvent event(String... bodies) { + var records = new java.util.ArrayList(); + for (var i = 0; i < bodies.length; i++) { + records.add(new ProcessorRecord(String.valueOf(i), bodies[i])); + } + return new ProcessorEvent(records); + } + + @Test + void durableBatchHandlerRunsUnderTheSuppliedConfig() { + var serDes = new RecordingSerDes(); + var seen = new java.util.ArrayList(); + BiConsumer> func = + (ctx, items) -> seen.addAll(ctx.step("copy", new TypeToken>() {}, stepCtx -> items)); + + var handler = DistributedMapHandlers.createDistributedMapBatchHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, configWith(serDes)); + var output = handler.handleRequest(durableInput(event("1", "2")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals(List.of(1, 2), seen); + // The processor's event is always read with the default JSON SerDes. The step inside the processor uses the + // config's SerDes, so a call here means the supplied config reached the execution. + assertTrue(serDes.deserializeCount.get() > 0); + } + + @Test + void durableItemHandlerRunsUnderTheSuppliedConfig() { + var serDes = new RecordingSerDes(); + BiFunction func = (ctx, item) -> item * 2; + + var handler = DistributedMapHandlers.createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, configWith(serDes)); + var output = handler.handleRequest(durableInput(event("1", "2")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertTrue(serDes.deserializeCount.get() > 0); + } + + // The service defines the processor's event and response format, so a custom SerDes on the config must not + // rename the envelope's fields. + @Test + void customSerDesDoesNotChangeTheProcessorEnvelope() { + var snakeCase = new JacksonSerDes(new ObjectMapper() + .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)); + BiFunction func = (ctx, item) -> item * 2; + + var handler = DistributedMapHandlers.createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, configWith(snakeCase)); + var output = handler.handleRequest(durableInput(event("1")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertTrue(output.result().contains("\"batchItemResults\""), output.result()); + assertTrue(output.result().contains("\"itemIdentifier\":\"0\""), output.result()); + assertTrue(output.result().contains("\"batchItemFailures\""), output.result()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java new file mode 100644 index 0000000000..9731a22f3e --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java @@ -0,0 +1,460 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.dmap.DistributedMapHandlers.*; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Unit tests for the distributed map authoring helpers (non-durable variants). */ +class DistributedMapHandlersTest { + + // Builds a single processor record. The body is the serialized item, as the wire carries it. + private static ProcessorRecord rec(String itemId, String body) { + return new ProcessorRecord(itemId, body); + } + + // Builds a distributed map processor event carrying the given records. + private static ProcessorEvent processorEvent(ProcessorRecord... records) { + return new ProcessorEvent(List.of(records)); + } + + // Builds a distributed map reader event carrying the state and maxItemsPerPage fields. + private static ReaderEvent readerEvent(String state, int maxItemsPerPage) { + return new ReaderEvent(maxItemsPerPage, state); + } + + private static ItemFailure.ErrorDetail error(String type, String message) { + return new ItemFailure.ErrorDetail(type, message, null, null); + } + + private static void assertFailure(ItemFailure failure, String itemId, String type, String message) { + assertEquals(itemId, failure.itemIdentifier()); + assertEquals(type, failure.error().errorType()); + assertEquals(message, failure.error().errorMessage()); + assertNull(failure.error().errorData()); + assertFalse(failure.error().stackTrace().isEmpty()); + } + + /** Serdes whose wire form is not JSON, so an item that skipped serialization cannot be read back. */ + private static final class PrefixSerDes implements SerDes { + @Override + public String serialize(Object value) { + return "p:" + value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + if (!data.startsWith("p:")) { + throw new IllegalArgumentException("not a prefixed value: " + data); + } + return (T) Integer.valueOf(data.substring(2)); + } + } + + @Test + void itemHandlerReportsResultsInOrder() { + Function func = x -> x * 2; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "2"), rec("1", "3")), null); + + assertEquals(List.of(new ItemResult("0", "4"), new ItemResult("1", "6")), resp.batchItemResults()); + assertEquals(List.of(), resp.batchItemFailures()); + } + + @Test + void itemHandlerCapturesFailuresWithFullyQualifiedErrorType() { + Function func = x -> { + if (x.equals("bad")) { + throw new IllegalStateException("boom"); + } + return x; + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(String.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "\"ok\""), rec("1", "\"bad\"")), null); + + assertEquals(List.of(new ItemResult("0", "\"ok\"")), resp.batchItemResults()); + assertEquals(1, resp.batchItemFailures().size()); + assertFailure(resp.batchItemFailures().get(0), "1", "java.lang.IllegalStateException", "boom"); + } + + @Test + void itemHandlerReportsAnEmptyResultsListWhenEveryItemFails() { + Function func = x -> { + throw new IllegalStateException("nope"); + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("7", "1")), null); + + // Both lists are always reported, so an all-failing batch carries an empty results list rather than no key. + assertEquals(List.of(), resp.batchItemResults()); + assertEquals(1, resp.batchItemFailures().size()); + assertFailure(resp.batchItemFailures().get(0), "7", "java.lang.IllegalStateException", "nope"); + } + + @Test + void itemHandlerHonorsRequestedConcurrency() { + // Each item blocks until all four have arrived, so the batch only finishes if the pool really runs four + // items at once. At a smaller pool size the first item waits out the timeout and every item fails. + var barrier = new CyclicBarrier(4); + Function func = x -> { + try { + barrier.await(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new IllegalStateException("barrier not reached", e); + } + return x; + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 4); + + var resp = + handler.handleRequest(processorEvent(rec("0", "0"), rec("1", "1"), rec("2", "2"), rec("3", "3")), null); + + assertEquals(List.of(), resp.batchItemFailures()); + // The four items are released together, so completion order is arbitrary while results stay in item order. + assertEquals( + List.of( + new ItemResult("0", "0"), + new ItemResult("1", "1"), + new ItemResult("2", "2"), + new ItemResult("3", "3")), + resp.batchItemResults()); + } + + @Test + void itemHandlerDefaultsToOneConcurrentHandler() { + var inFlight = new AtomicInteger(); + var peakInFlight = new AtomicInteger(); + Function func = x -> { + peakInFlight.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + // Held briefly so a regression to a wider pool would actually overlap and be caught. + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return x; + }; + // The overload with no concurrency argument runs one handler at a time. + var handler = createDistributedMapItemHandler(func, TypeToken.get(String.class), null, null); + + var resp = handler.handleRequest(processorEvent(rec("0", "\"a\""), rec("1", "\"b\""), rec("2", "\"c\"")), null); + + assertEquals(1, peakInFlight.get()); + assertEquals(3, resp.batchItemResults().size()); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void itemHandlerRejectsNonPositiveConcurrency(int concurrency) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapHandlers.createDistributedMapItemHandler( + item -> item, TypeToken.get(String.class), null, null, concurrency)); + assertEquals("concurrency must be at least 1, got: " + concurrency, exception.getMessage()); + } + + @Test + void itemHandlerReportsSerializationFailureAgainstTheOffendingItemOnly() { + // A result serdes that refuses exactly one value, standing in for a POJO the caller cannot serialize. + var refusingSerDes = new SerDes() { + private final SerDes delegate = new JacksonSerDes(); + + @Override + public String serialize(Object value) { + if ("boom".equals(value)) { + throw new IllegalStateException("cannot serialize"); + } + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return delegate.deserialize(data, typeToken); + } + }; + + var handler = DistributedMapHandlers.createDistributedMapItemHandler( + item -> item, TypeToken.get(String.class), null, refusingSerDes, 2); + + var resp = handler.handleRequest( + processorEvent(rec("a", "\"ok\""), rec("b", "\"boom\""), rec("c", "\"fine\"")), null); + + // The whole batch must not fail. Only item b is reported as failed. + assertEquals(2, resp.batchItemResults().size()); + assertEquals("a", resp.batchItemResults().get(0).itemIdentifier()); + assertEquals("c", resp.batchItemResults().get(1).itemIdentifier()); + assertEquals(1, resp.batchItemFailures().size()); + assertEquals("b", resp.batchItemFailures().get(0).itemIdentifier()); + } + + @Test + void processorEventRejectsMissingRecords() { + var ex = assertThrows(IllegalStateException.class, () -> new ProcessorEvent(null)); + assertTrue(ex.getMessage().contains("'records' array")); + assertTrue(ex.getMessage().contains("not invoked directly")); + } + + @Test + void readerEventRejectsMissingMaxItemsPerPage() { + var ex = assertThrows(IllegalStateException.class, () -> new ReaderEvent(null, "state")); + assertTrue(ex.getMessage().contains("'maxItemsPerPage'")); + } + + @Test + void batchHandlerSucceedsAllItemsWhenFunctionReturns() { + var seen = new ArrayList(); + Consumer> func = seen::addAll; + var handler = createDistributedMapBatchHandler(func, TypeToken.get(Integer.class), null); + + var resp = handler.handleRequest(processorEvent(rec("0", "1"), rec("1", "2")), null); + + assertNull(resp); + assertEquals(List.of(1, 2), seen); + } + + @Test + void batchHandlerFailsBatchWhenFunctionThrows() { + Consumer> func = items -> { + throw new RuntimeException("batch failed"); + }; + var handler = createDistributedMapBatchHandler(func, TypeToken.get(Integer.class), null); + + var ex = assertThrows(RuntimeException.class, () -> handler.handleRequest(processorEvent(rec("0", "1")), null)); + assertTrue(ex.getMessage().contains("batch failed")); + } + + @Test + void readerReturnsItemsAndNextStateThenExhausts() { + BiFunction> read = (state, maxItemsPerPage) -> + state == null ? new ReaderPage<>(List.of(1, 2), 1) : new ReaderPage<>(List.of(3), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var first = handler.handleRequest(readerEvent(null, 10), null); + assertEquals(List.of("1", "2"), first.items()); + assertEquals("1", first.nextState()); + + var second = handler.handleRequest(readerEvent("1", 10), null); + assertEquals(List.of("3"), second.items()); + // Null rather than empty, so the runtime omits the key, which is how exhaustion is signalled. + assertNull(second.nextState()); + } + + @Test + void readerReceivesMaxItemsPerPage() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(maxItemsPerPage), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var page = handler.handleRequest(readerEvent(null, 7), null); + + assertEquals(List.of("7"), page.items()); + } + + @Test + void readerRejectsPageOverMaxItemsPerPage() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1, 2, 3), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 2), null)); + assertTrue(ex.getMessage().contains("exceeding maxItemsPerPage")); + } + + @Test + void readerRejectsOversizedNextState() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), "x".repeat(40_000)); + var handler = createDistributedMapReader(read, TypeToken.get(String.class), null, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertTrue(ex.getMessage().contains("32 KB")); + } + + @Test + void readerSendsNullItemAsJsonNull() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(java.util.Arrays.asList(1, null), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var page = handler.handleRequest(readerEvent(null, 10), null); + + assertEquals(List.of("1", "null"), page.items()); + } + + @Test + void readerRejectsItemSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, nullSerDes); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertEquals("serialization of reader item 0 produced no string", ex.getMessage()); + } + + @Test + void readerRejectsNextStateSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), 1); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), nullSerDes, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertEquals("serialization of reader nextState produced no string", ex.getMessage()); + } + + @Test + void readerItemsRoundTripThroughTheProcessorUnderACustomItemSerdes() { + var serdes = new PrefixSerDes(); + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1, 2), null); + var reader = createDistributedMapReader(read, TypeToken.get(Integer.class), null, serdes); + + var page = reader.handleRequest(readerEvent(null, 10), null); + assertEquals(List.of("p:1", "p:2"), page.items()); + + var seen = new ArrayList(); + var processor = createDistributedMapItemHandler( + (Function) item -> { + seen.add(item); + return item; + }, + TypeToken.get(Integer.class), + serdes, + null); + processor.handleRequest( + processorEvent( + rec("0", page.items().get(0)), rec("1", page.items().get(1))), + null); + + assertEquals(List.of(1, 2), seen); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void durableItemHandlerRejectsNonPositiveConcurrency(int concurrency) { + BiFunction func = (ctx, item) -> item; + var exception = assertThrows( + IllegalArgumentException.class, + () -> createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, concurrency)); + assertEquals("concurrency must be at least 1, got: " + concurrency, exception.getMessage()); + } + + // The Lambda runtime serializes the response, so these assert the wire shape it produces. The mapper mirrors the + // runtime's null exclusion, which is what keeps an absent key absent. + private static final ObjectMapper WIRE = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Test + void itemHandlerReportsNoOutputWhenTheHandlerReturnsNull() { + // PrefixSerDes stringifies whatever it is handed, so an output is reported only if the handler + // never calls it for an item that returned nothing. + Function func = x -> null; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, new PrefixSerDes(), 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "1")), null); + + assertEquals(List.of(), resp.batchItemFailures()); + assertEquals(List.of(new ItemResult("0", null)), resp.batchItemResults()); + } + + @Test + void itemResultWithNoOutputLeavesTheOutputOffTheWire() throws Exception { + var json = WIRE.writeValueAsString(new ItemResult("0", null)); + + assertEquals("{\"itemIdentifier\":\"0\"}", json); + } + + @Test + void itemFailureCarriesTheStackTraceAndNoErrorData() { + Function func = x -> { + throw new IllegalStateException("nope"); + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var error = handler.handleRequest(processorEvent(rec("0", "1")), null) + .batchItemFailures() + .get(0) + .error(); + + assertNull(error.errorData()); + assertTrue(error.stackTrace().get(0).startsWith(DistributedMapHandlersTest.class.getName())); + } + + @Test + void itemFailureNestsTheErrorObject() throws Exception { + var json = WIRE.writeValueAsString(new ItemFailure("1", error("SomeType", "boom"))); + + assertEquals( + "{\"itemIdentifier\":\"1\",\"error\":{\"errorType\":\"SomeType\",\"errorMessage\":\"boom\"}}", json); + } + + @Test + void itemFailureWritesEveryErrorMember() throws Exception { + var detail = new ItemFailure.ErrorDetail("E", "boom", "{\"k\":1}", List.of("f1")); + var json = WIRE.writeValueAsString(new ItemFailure("1", detail)); + + assertEquals( + "{\"itemIdentifier\":\"1\",\"error\":{\"errorType\":\"E\",\"errorMessage\":\"boom\"," + + "\"errorData\":\"{\\\"k\\\":1}\",\"stackTrace\":[\"f1\"]}}", + json); + } + + @Test + void exhaustedReaderPageLeavesNextStateOffTheWire() throws Exception { + var json = WIRE.writeValueAsString(new ReaderResponse(List.of("1", "2"), null)); + + assertEquals("{\"items\":[\"1\",\"2\"]}", json); + } + + @ParameterizedTest + @NullAndEmptySource + void processorRecordWithoutItemIdIsRejected(String itemId) { + var exception = assertThrows(IllegalStateException.class, () -> new ProcessorRecord(itemId, "1")); + assertEquals("expected each distributed map processor record to have an 'itemId'", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java index 61f534e26a..54e7b7bbe7 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java @@ -9,10 +9,12 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; @@ -20,6 +22,16 @@ import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.services.lambda.model.CheckpointDurableExecutionResponse; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ObjectConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceType; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; import software.amazon.awssdk.services.lambda.model.GetDurableExecutionStateResponse; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -75,6 +87,143 @@ void checkpoint_sendsUpdateAndReturnsCompletedFuture() throws Exception { assertTrue(future.isDone()); } + @Test + void checkpoint_sendsLargeInlineSourceInItsOwnRequest() throws Exception { + // 800 KB of inline items, over the 750 KB batch limit, so this update cannot share a request. + var items = Collections.nCopies(800, "x".repeat(1024)); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.INLINE) + .inlineSourceConfig(DistributedMapInlineSourceConfig.builder() + .items(items) + .build()) + .build()) + .build()) + .build(); + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(dmapStart); + var second = batcher.checkpoint(step); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the oversized update must not share a request"); + assertEquals( + List.of("op-dmap"), + sent.get(0).stream().map(OperationUpdate::id).toList()); + assertEquals( + List.of("op-step"), + sent.get(1).stream().map(OperationUpdate::id).toList()); + } + + @Test + void checkpoint_countsCsvHeadersTowardBatchSize() throws Exception { + // The step payload fills the 750 KB batch limit to within 15,000 characters. The two updates fit in one + // request only if the 30,000 characters of CSV headers are left out of the estimate. + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .payload("x".repeat(750 * 1024 - 243 - 15_000)) + .build(); + var headers = IntStream.range(0, 200) + .mapToObj(i -> String.format("%03d", i) + "h".repeat(147)) + .toList(); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.S3) + .s3SourceConfig(DistributedMapS3SourceConfig.builder() + .type(DistributedMapS3SourceType.OBJECT) + .bucket("bucket") + .objectConfig(DistributedMapS3ObjectConfig.builder() + .key("data.csv") + .format(DistributedMapSourceFormat.CSV) + .csvFormatOptions(DistributedMapCsvFormatOptions.builder() + .headers(headers) + .build()) + .build()) + .build()) + .build()) + .build()) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(step); + var second = batcher.checkpoint(dmapStart); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the CSV headers must count toward the batch size"); + } + + @Test + void checkpoint_countsReaderInitialStateTowardBatchSize() throws Exception { + // The step payload fills the 750 KB batch limit to within 15,000 characters. The two updates fit in one + // request only if the reader's 30,000-character initial state is left out of the estimate. + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .payload("x".repeat(750 * 1024 - 243 - 15_000)) + .build(); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.READER_FUNCTION) + .readerFunctionSourceConfig(DistributedMapReaderFunctionSourceConfig.builder() + .functionName("reader") + .initialState("s".repeat(30_000)) + .build()) + .build()) + .build()) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(step); + var second = batcher.checkpoint(dmapStart); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the reader state must count toward the batch size"); + } + @Test void checkpoint_skipsApiCallWhenExecutionAlreadyCompleted() throws Exception { var finishCheckpointProcessing = mock(Runnable.class); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java new file mode 100644 index 0000000000..a134f7425d --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java @@ -0,0 +1,191 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.DistributedMapException; + +class DistributedMapResultTest { + + private static DistributedMapSummary summary(DistributedMapStatus status, long failureCount) { + return new DistributedMapSummary( + status, + true, + 2, + failureCount, + 1, + "arn:aws:states:us-east-1:123456789012:mapRun:run-42", + new DistributedMapError("RunFailed", "details", null, null), + 4L); + } + + private static DistributedMapError itemError(String message) { + return new DistributedMapError("java.lang.RuntimeException", message, null, null); + } + + @Test + void succeeded_returnsAllSucceededItems() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("fail-b")), + DistributedMapResult.Item.succeeded("c", "out-c"))); + + var succeeded = result.succeeded(); + assertEquals(2, succeeded.size()); + assertEquals("a", succeeded.get(0).itemId()); + assertEquals("c", succeeded.get(1).itemId()); + } + + @Test + void failed_returnsAllFailedItems() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("fail-b")))); + + var failed = result.failed(); + assertEquals(1, failed.size()); + assertEquals("b", failed.get(0).itemId()); + } + + @Test + void results_preservesNullOutputs() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 0), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.succeeded("b", null), + DistributedMapResult.Item.succeeded("c", "out-c"))); + + // cardinality must match succeeded() so callers can align outputs with items + assertEquals(3, result.results().size()); + assertEquals(Arrays.asList("out-a", null, "out-c"), result.results()); + assertEquals(result.succeeded().size(), result.results().size()); + } + + @Test + void errors_preservesNullErrors() { + DistributedMapResult.Item withError = DistributedMapResult.Item.failed("a", itemError("fail-a")); + var withoutError = + new DistributedMapResult.Item("b", DistributedMapResult.Item.Status.FAILED, null, null); + var result = new DistributedMapResult( + summary(DistributedMapStatus.SUCCEEDED, 2), List.of(withError, withoutError)); + + // cardinality must match failed() so callers can align errors with items + var errors = result.errors(); + assertEquals(2, errors.size()); + assertEquals("fail-a", errors.get(0).errorMessage()); + assertNull(errors.get(1)); + assertEquals(result.failed().size(), errors.size()); + } + + @Test + void throwIfError_throwsRunLevelOnNonSucceededStatus() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.FAILED, 0), List.of(DistributedMapResult.Item.succeeded("a", "out-a"))); + + var error = assertThrows(DistributedMapException.class, result::throwIfError); + assertEquals(DistributedMapStatus.FAILED, error.status()); + } + + @Test + void throwIfError_surfacesFirstFailedItemWhenSucceededWithFailures() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("boom-b")))); + + var error = assertThrows(DistributedMapException.class, result::throwIfError); + assertTrue(error.getMessage().contains("boom-b")); + } + + private static String itemFailureMessage(DistributedMapError error) { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), List.of(DistributedMapResult.Item.failed("b", error))); + return assertThrows(DistributedMapException.class, result::throwIfError).getMessage(); + } + + @Test + void throwIfError_itemMessageJoinsTypeAndMessage() { + assertEquals("E: boom", itemFailureMessage(new DistributedMapError("E", "boom", null, null))); + } + + @Test + void throwIfError_itemMessageUsesMessageWhenTypeIsAbsent() { + assertEquals("boom", itemFailureMessage(new DistributedMapError(null, "boom", null, null))); + assertEquals("boom", itemFailureMessage(new DistributedMapError("", "boom", null, null))); + } + + @Test + void throwIfError_itemMessageUsesTypeWhenMessageIsAbsent() { + assertEquals("E", itemFailureMessage(new DistributedMapError("E", null, null, null))); + assertEquals("E", itemFailureMessage(new DistributedMapError("E", "", null, null))); + } + + @Test + void throwIfError_itemMessageFallsBackWhenTypeAndMessageAreAbsent() { + assertEquals( + "Distributed map item b failed", itemFailureMessage(new DistributedMapError(null, null, null, null))); + assertEquals("Distributed map item b failed", itemFailureMessage(new DistributedMapError("", "", null, null))); + assertEquals("Distributed map item b failed", itemFailureMessage(null)); + } + + @Test + void throwIfError_doesNothingOnFullSuccess() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 0), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.succeeded("b", "out-b"))); + + assertDoesNotThrow(result::throwIfError); + } + + @Test + void delegators_returnComposedSummaryValues() { + var summary = summary(DistributedMapStatus.SUCCEEDED, 1); + var result = new DistributedMapResult<>(summary, List.of(DistributedMapResult.Item.succeeded("a", "out-a"))); + + assertEquals(summary.status(), result.status()); + assertEquals(summary.sourceExhausted(), result.sourceExhausted()); + assertEquals(summary.successCount(), result.successCount()); + assertEquals(summary.failureCount(), result.failureCount()); + assertEquals(summary.unprocessedCount(), result.unprocessedCount()); + assertEquals(summary.totalCount(), result.totalCount()); + assertEquals(summary.distributedMapRunArn(), result.distributedMapRunArn()); + assertEquals(summary.error(), result.error()); + assertEquals(summary.hasFailure(), result.hasFailure()); + } + + @Test + void items_defaultToEmptyWhenNull() { + var result = new DistributedMapResult(summary(DistributedMapStatus.SUCCEEDED, 0), null); + + assertTrue(result.items().isEmpty()); + } + + @Test + void of_usesFullyQualifiedClassName() { + var error = DistributedMapError.of(new IllegalStateException("boom")); + + assertEquals("java.lang.IllegalStateException", error.errorType()); + assertEquals("boom", error.errorMessage()); + assertFalse(error.stackTrace().isEmpty()); + } + + @Test + void of_preservesNullMessage() { + var error = DistributedMapError.of(new RuntimeException()); + + assertEquals("java.lang.RuntimeException", error.errorType()); + assertNull(error.errorMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java new file mode 100644 index 0000000000..4c8137524f --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.DistributedMapException; + +class DistributedMapSummaryTest { + + private static DistributedMapSummary summary( + DistributedMapStatus status, long failureCount, String arn, String errorMessage) { + var error = errorMessage != null ? new DistributedMapError("RunFailed", errorMessage, null, null) : null; + return new DistributedMapSummary(status, true, 5, failureCount, 0, arn, error, 5L); + } + + @Test + void distributedMapRunArn_exposedAsGiven() { + var s = summary( + DistributedMapStatus.SUCCEEDED, + 0, + "arn:aws:lambda:us-east-1:123456789012:distributed-map-run/abc", + null); + + assertEquals("arn:aws:lambda:us-east-1:123456789012:distributed-map-run/abc", s.distributedMapRunArn()); + } + + @Test + void distributedMapRunArn_nullWhenAbsent() { + var s = summary(DistributedMapStatus.SUCCEEDED, 0, null, null); + + assertNull(s.distributedMapRunArn()); + } + + @Test + void hasFailure_trueWhenFailureCountPositive() { + assertTrue(summary(DistributedMapStatus.SUCCEEDED, 3, "arn:x", null).hasFailure()); + } + + @Test + void hasFailure_falseWhenNoFailures() { + assertFalse(summary(DistributedMapStatus.SUCCEEDED, 0, "arn:x", null).hasFailure()); + } + + @Test + void throwIfError_throwsOnNonSucceededStatus() { + var s = summary(DistributedMapStatus.FAILED, 0, "arn:x", "run blew up"); + + var error = assertThrows(DistributedMapException.class, s::throwIfError); + assertEquals(DistributedMapStatus.FAILED, error.status()); + assertEquals("run blew up", error.error().errorMessage()); + assertTrue(error.getMessage().contains("run blew up")); + } + + @Test + void throwIfError_throwsWhenSucceededWithFailures() { + var s = summary(DistributedMapStatus.SUCCEEDED, 2, "arn:x", null); + + assertThrows(DistributedMapException.class, s::throwIfError); + } + + @Test + void throwIfError_doesNothingOnFullSuccess() { + var s = summary(DistributedMapStatus.SUCCEEDED, 0, "arn:x", null); + + assertDoesNotThrow(s::throwIfError); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java new file mode 100644 index 0000000000..ed1dcd6608 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java @@ -0,0 +1,333 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiFunction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DistributedMapException; +import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.IllegalDurableOperationException; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +class DistributedMapOperationTest { + private static final String OPERATION_ID = "7"; + private static final String OPERATION_NAME = "test-distributed-map"; + private static final OperationIdentifier OPERATION_IDENTIFIER = + OperationIdentifier.of(OPERATION_ID, OPERATION_NAME, OperationSubType.DISTRIBUTED_MAP); + private static final SerDes SER_DES = new JacksonSerDes(); + + private ExecutionManager executionManager; + private DurableContextImpl durableContext; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + @BeforeEach + void setUp() { + executionManager = mock(ExecutionManager.class); + durableContext = mock(DurableContextImpl.class); + when(durableContext.getExecutionManager()).thenReturn(executionManager); + when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("root", ThreadType.CONTEXT)); + } + + @Test + void getReturnsSummaryWhenOperationSucceeded() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(5L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(5L) + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertTrue(summary.sourceExhausted()); + assertEquals(5L, summary.successCount()); + assertEquals(0L, summary.failureCount()); + assertEquals(5L, summary.totalCount()); + assertEquals("arn:aws:lambda:us-east-1:123456789012:map-run:abc", summary.distributedMapRunArn()); + assertFalse(summary.hasFailure()); + } + + @Test + void getReturnsResultWhenResultCollecting() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(2L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(2L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeToken> resultType = (TypeToken) TypeToken.get(DistributedMapResult.class); + BiFunction> resultBuilder = + (s, d) -> DistributedMapOperation.toResult(s, d, TypeToken.get(Integer.class), SER_DES); + + var operation = newOperation(resultType, resultBuilder); + operation.onCheckpointComplete(op); + + var result = operation.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, result.status()); + assertEquals(2L, result.successCount()); + assertTrue(result.results().isEmpty()); + assertTrue(result.errors().isEmpty()); + } + + @Test + void getResolvesWithSummaryWhenOperationFailed() { + assertGetResolvesWithSummary(OperationStatus.FAILED, DistributedMapStatus.FAILED); + } + + @Test + void getResolvesWithSummaryWhenOperationTimedOut() { + assertGetResolvesWithSummary(OperationStatus.TIMED_OUT, DistributedMapStatus.TIMED_OUT); + } + + @Test + void getResolvesWithSummaryWhenOperationStopped() { + assertGetResolvesWithSummary(OperationStatus.STOPPED, DistributedMapStatus.STOPPED); + } + + @Test + void getTerminatesWhenSucceededWithNoDetails() { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + assertThrows(IllegalDurableOperationException.class, operation::get); + verify(executionManager).terminateExecution(any(IllegalDurableOperationException.class)); + } + + @Test + void getTerminatesWhenARunExistedButNoSourceExhausted() { + var details = DistributedMapDetails.builder() + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + assertThrows(IllegalDurableOperationException.class, operation::get); + verify(executionManager).terminateExecution(any(IllegalDurableOperationException.class)); + } + + @Test + void getResolvesWhenNoRunWasCreatedAndSourceExhaustedIsAbsent() { + var details = DistributedMapDetails.builder() + .successCount(0L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.FAILED) + .distributedMapDetails(details) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(DistributedMapStatus.FAILED, summary.status()); + assertNull(summary.sourceExhausted()); + } + + @Test + void getSuspendsUntilCheckpointCompletesThenReturnsSummary() + throws InterruptedException, ExecutionException, TimeoutException { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + + // get blocks because the operation has not reached a terminal checkpoint yet + var future = executor.submit(operation::get); + try { + future.get(500, TimeUnit.MILLISECONDS); + fail(); + } catch (TimeoutException e) { + // feed the terminal checkpoint, which unblocks get and resolves the summary + operation.onCheckpointComplete(op); + var summary = future.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertEquals(1L, summary.successCount()); + } + } + + // A terminal-failed operation resolves with its summary so the counts are reachable. + private void assertGetResolvesWithSummary(OperationStatus status, DistributedMapStatus expected) { + var details = DistributedMapDetails.builder() + .sourceExhausted(false) + .successCount(3L) + .failureCount(2L) + .unprocessedCount(1L) + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(status) + .distributedMapDetails(details) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(expected, summary.status()); + assertFalse(summary.sourceExhausted()); + assertEquals(3L, summary.successCount()); + assertEquals(2L, summary.failureCount()); + assertEquals(1L, summary.unprocessedCount()); + assertTrue(summary.hasFailure()); + assertThrows(DistributedMapException.class, summary::throwIfError); + } + + @Test + void replayPollsWhileTheOperationIsPending() { + assertReplayPolls(OperationStatus.PENDING); + } + + @Test + void replayPollsWhileTheOperationIsStarted() { + assertReplayPolls(OperationStatus.STARTED); + } + + @Test + void pollingContinuesWhenPendingBecomesStarted() { + var pending = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.DISTRIBUTED_MAP) + .subType(OperationSubType.DISTRIBUTED_MAP.getValue()) + .status(OperationStatus.PENDING) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(pending); + var started = pending.toBuilder().status(OperationStatus.STARTED).build(); + when(executionManager.pollForOperationUpdates(OPERATION_ID)) + .thenReturn(CompletableFuture.completedFuture(started), new CompletableFuture<>()); + + newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary) + .execute(); + + verify(executionManager, times(2)).pollForOperationUpdates(OPERATION_ID); + } + + @Test + void failedRunErrorIsReportedToPlugins() { + var error = ErrorObject.builder() + .errorType("SourceReadError") + .errorMessage("object not found") + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .type(OperationType.DISTRIBUTED_MAP) + .status(OperationStatus.FAILED) + .distributedMapDetails( + DistributedMapDetails.builder().error(error).build()) + .build(); + + var extracted = BaseDurableOperation.extractErrorFromOperation(op); + + var exception = assertInstanceOf(DurableOperationException.class, extracted); + assertEquals(error, exception.getErrorObject()); + } + + // A non-terminal replay waits for the terminal outcome rather than terminating the execution. + private void assertReplayPolls(OperationStatus status) { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.DISTRIBUTED_MAP) + .subType(OperationSubType.DISTRIBUTED_MAP.getValue()) + .status(status) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary) + .execute(); + + verify(executionManager).pollForOperationUpdates(OPERATION_ID); + verify(executionManager, never()).terminateExecution(any()); + } + + private DistributedMapOperation newOperation( + TypeToken resultType, BiFunction resultBuilder) { + return new DistributedMapOperation<>( + OPERATION_IDENTIFIER, + DistributedMapOptions.builder().build(), + resultType, + SER_DES, + resultBuilder, + durableContext); + } + + private Operation succeededOperation(DistributedMapDetails details) { + return Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .distributedMapDetails(details) + .build(); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java new file mode 100644 index 0000000000..607576db5b --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java @@ -0,0 +1,852 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultItem; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapDestination; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapProcessorRetryConfig; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.exception.DurableExecutionException; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Translation tests for DistributedMapOperation toOptions, toSummary and toResult. */ +class DistributedMapOperationTranslationTest { + + private static final SerDes SERDES = new JacksonSerDes(); + + // ------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------ + + private static DistributedMapOptions options( + DistributedMapSource source, + DistributedMapProcessor processor, + DistributedMapConfig config, + boolean collect) { + return DistributedMapOperation.toOptions(source, processor, 4, config, SERDES, collect); + } + + private static DistributedMapOptions options(DistributedMapSource source, DistributedMapProcessor processor) { + return options(source, processor, DistributedMapConfig.builder().build(), false); + } + + private static DistributedMapProcessor batchProcessor() { + return DistributedMapProcessor.reportBatchOutcome("proc").build(); + } + + private static DistributedMapDetails.Builder detailsBuilder() { + return DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L); + } + + private static DistributedMapResultItem succeededItem(String itemId, String output) { + return DistributedMapResultItem.builder() + .itemId(itemId) + .status("SUCCEEDED") + .output(output) + .build(); + } + + private static DistributedMapResultItem failedItem(String itemId, ErrorObject error) { + return DistributedMapResultItem.builder() + .itemId(itemId) + .status("FAILED") + .error(error) + .build(); + } + + /** Serdes that upper-cases on serialize and upper-cases plus counts on deserialize. */ + private static final class UpperSerDes implements SerDes { + int deserializeCount = 0; + + @Override + public String serialize(Object value) { + return String.valueOf(value).toUpperCase(Locale.ROOT); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + deserializeCount++; + return (T) data.toUpperCase(Locale.ROOT); + } + } + + // ------------------------------------------------------------------ + // toOptions - top level + // ------------------------------------------------------------------ + + @Test + void inlineSourceSerializesItemsThroughDefaultSerdes() { + var options = options(DistributedMapSource.inline(List.of("a", "b")), batchProcessor()); + assertEquals(4, options.maxConcurrency().intValue()); + assertEquals("INLINE", options.source().typeAsString()); + assertEquals( + List.of(SERDES.serialize("a"), SERDES.serialize("b")), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceSendsNullItemAsJsonNull() { + var options = options(DistributedMapSource.inline(java.util.Arrays.asList("a", null)), batchProcessor()); + assertEquals( + List.of(SERDES.serialize("a"), "null"), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceRejectsItemSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var exception = assertThrows( + IllegalArgumentException.class, + () -> options(DistributedMapSource.inline(List.of("a"), nullSerDes), batchProcessor())); + assertEquals("serialization of inline item 0 produced no string", exception.getMessage()); + } + + /** One item of "é" characters, padded with "a", whose serialized inline array is exactly totalBytes. */ + private static List inlineItemsOfSerializedSize(int totalBytes) { + // The serdes wraps the item in quotes. The array escapes those quotes, then adds its own quotes and brackets. + // So the array is 8 bytes larger than the item text. Each "é" is 2 UTF-8 bytes. + int bodyBytes = totalBytes - 8; + var body = (bodyBytes % 2 == 1 ? "a" : "") + "é".repeat(bodyBytes / 2); + return List.of(body); + } + + private static int serializedInlineBytes(List items) { + var bodies = items.stream().map(SERDES::serialize).toList(); + return SERDES.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; + } + + @Test + void inlineSourceAcceptsSerializedSizeAtTheLimit() { + var items = inlineItemsOfSerializedSize(1024 * 1024); + assertEquals(1024 * 1024, serializedInlineBytes(items)); + + var options = options(DistributedMapSource.inline(items), batchProcessor()); + assertEquals( + List.of(SERDES.serialize(items.get(0))), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceRejectsSerializedSizeOneByteOverTheLimit() { + var items = inlineItemsOfSerializedSize(1024 * 1024 + 1); + assertEquals(1024 * 1024 + 1, serializedInlineBytes(items)); + // The character count is below the limit, so only a byte count rejects this source. + assertTrue(SERDES.serialize(List.of(SERDES.serialize(items.get(0)))).length() < 1024 * 1024); + + var exception = assertThrows( + IllegalArgumentException.class, () -> options(DistributedMapSource.inline(items), batchProcessor())); + assertEquals("inline source exceeds the 1 MB limit (serialized size: 1048577 bytes)", exception.getMessage()); + } + + @Test + void inlineSourceUsesCustomSerdesForEachItem() { + var options = options(DistributedMapSource.inline(List.of("a", "b"), new UpperSerDes()), batchProcessor()); + assertEquals(List.of("A", "B"), options.source().inlineSourceConfig().items()); + } + + @Test + void maxConcurrencyPassedThrough() { + var options = DistributedMapOperation.toOptions( + DistributedMapSource.inline(List.of("a")), + batchProcessor(), + 42, + DistributedMapConfig.builder().build(), + SERDES, + false); + assertEquals(42, options.maxConcurrency().intValue()); + } + + @Test + void timeoutMapsToWholeSeconds() { + var config = + DistributedMapConfig.builder().timeout(Duration.ofMinutes(30)).build(); + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false); + assertEquals(1800, options.timeoutSeconds().intValue()); + } + + @Test + void timeoutOmittedWhenAbsent() { + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor()); + assertNull(options.timeoutSeconds()); + } + + // ------------------------------------------------------------------ + // toOptions - processor + // ------------------------------------------------------------------ + + @Test + void itemOutcomesSetsResponseTypeAndBatchSize() { + var processor = + DistributedMapProcessor.reportItemOutcomes("proc").batchSize(25).build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals("proc", wire.functionName()); + assertEquals(List.of("REPORT_BATCH_ITEM_OUTCOMES"), wire.functionResponseTypesAsStrings()); + assertEquals(25, wire.batchSize().intValue()); + } + + @Test + void batchOmitsResponseTypes() { + var wire = options(DistributedMapSource.inline(List.of("a")), batchProcessor()) + .processor(); + assertFalse(wire.hasFunctionResponseTypes()); + } + + @Test + void unlimitedRetriesMapToNegativeOne() { + var processor = DistributedMapProcessor.reportItemOutcomes("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .maxRetryDuration(Duration.ofHours(1)) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(-1, wire.maxRetryAttempts().intValue()); + assertEquals(3600, wire.maxRetryDurationSeconds().intValue()); + } + + @Test + void explicitRetryAttemptsPassThrough() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(0) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(0, wire.maxRetryAttempts().intValue()); + } + + @Test + void retryDurationOnlyOmitsAttempts() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(300, wire.maxRetryDurationSeconds().intValue()); + assertNull(wire.maxRetryAttempts()); + } + + @Test + void omittedRetryConfigSendsNeitherRetryField() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc").build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertNull(wire.maxRetryAttempts()); + assertNull(wire.maxRetryDurationSeconds()); + } + + @Test + void durableExecutionNamePrefixSent() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .durableExecutionNamePrefix("pfx") + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals("pfx", wire.durableExecutionNamePrefix()); + } + + // ------------------------------------------------------------------ + // toOptions - source variants + // ------------------------------------------------------------------ + + @Test + void s3JsonLinesSourceSerializesConfig() { + var source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl").maxItems(500); + var wire = options(source, batchProcessor()).source(); + assertEquals("S3", wire.typeAsString()); + assertEquals(500L, wire.maxItemsToRead().longValue()); + assertEquals("OBJECT", wire.s3SourceConfig().typeAsString()); + assertEquals("bucket", wire.s3SourceConfig().bucket()); + assertEquals("data.jsonl", wire.s3SourceConfig().objectConfig().key()); + assertEquals("JSON_LINES", wire.s3SourceConfig().objectConfig().formatAsString()); + } + + @Test + void s3ObjectsSourceUsesListedObjectMetadataAndPrefix() { + var wire = options(DistributedMapSource.s3Objects("s3://b/prefix/"), batchProcessor()) + .source(); + var s3 = wire.s3SourceConfig(); + assertEquals("LISTED_OBJECT_METADATA", s3.typeAsString()); + assertEquals("prefix/", s3.listedObjectMetadataConfig().keyPrefix()); + assertNull(s3.objectConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3FlattenedJsonLinesUsesListedObjects() { + var s3 = options(DistributedMapSource.s3FlattenedJsonLines("s3://b/prefix/"), batchProcessor()) + .source() + .s3SourceConfig(); + assertEquals("LISTED_OBJECTS", s3.typeAsString()); + assertEquals("prefix/", s3.listedObjectsConfig().keyPrefix()); + assertEquals("JSON_LINES", s3.listedObjectsConfig().formatAsString()); + assertNull(s3.objectConfig()); + } + + @Test + void s3FlattenedCsvUsesListedObjectsAndCsvOptions() { + var source = DistributedMapSource.s3FlattenedCsv( + "s3://b/prefix", DistributedMapSource.CsvFormat.headers(List.of("a", "b"))); + var listed = options(source, batchProcessor()).source().s3SourceConfig().listedObjectsConfig(); + assertEquals("CSV", listed.formatAsString()); + assertEquals("GIVEN", listed.csvFormatOptions().headerLocationAsString()); + assertEquals(List.of("a", "b"), listed.csvFormatOptions().headers()); + } + + @Test + void s3ExpectedBucketOwnerSent() { + var source = DistributedMapSource.s3JsonLines("s3://b/k.jsonl").expectedBucketOwner("123456789012"); + var s3 = options(source, batchProcessor()).source().s3SourceConfig(); + assertEquals("123456789012", s3.expectedBucketOwner()); + } + + @Test + void readerSourceSerializesInitialState() { + var source = DistributedMapSource.reader("reader", Map.of("page", 0), null); + var reader = options(source, batchProcessor()).source().readerFunctionSourceConfig(); + assertEquals("reader", reader.functionName()); + assertEquals(SERDES.serialize(Map.of("page", 0)), reader.initialState()); + } + + @Test + void readerSourceRejectsInitialStateSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var source = DistributedMapSource.reader("reader", Map.of("page", 0), nullSerDes); + var exception = assertThrows(IllegalArgumentException.class, () -> options(source, batchProcessor())); + assertEquals("serialization of reader initialState produced no string", exception.getMessage()); + } + + @Test + void readerSourceWithoutInitialStateOmitsState() { + var reader = options(DistributedMapSource.reader("reader"), batchProcessor()) + .source() + .readerFunctionSourceConfig(); + assertEquals("reader", reader.functionName()); + assertNull(reader.initialState()); + } + + // ------------------------------------------------------------------ + // toOptions - CSV header handling + // ------------------------------------------------------------------ + + @Test + void csvHeadersMapToGivenAndAreSent() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", DistributedMapSource.CsvFormat.headers(List.of("a", "b"))); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("GIVEN", csv.headerLocationAsString()); + assertEquals(List.of("a", "b"), csv.headers()); + assertEquals("COMMA", csv.delimiterAsString()); + } + + @Test + void csvFirstRowOmitsHeaders() { + var source = DistributedMapSource.s3Csv("s3://b/data.csv", DistributedMapSource.CsvFormat.firstRow()); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("FIRST_ROW", csv.headerLocationAsString()); + assertFalse(csv.hasHeaders()); + assertEquals("COMMA", csv.delimiterAsString()); + } + + @Test + void csvExpectedColumnsStayClientSide() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "b"))); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("FIRST_ROW", csv.headerLocationAsString()); + assertFalse(csv.hasHeaders()); + } + + @Test + void csvDelimiterMapsToWireValue() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", + DistributedMapSource.CsvFormat.headers(List.of("a")) + .withDelimiter(DistributedMapSource.CsvDelimiter.PIPE)); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("PIPE", csv.delimiterAsString()); + } + + // ------------------------------------------------------------------ + // toOptions - completion config + // ------------------------------------------------------------------ + + @Test + void completionPercentageAndSampleSizeSerialize() { + var config = DistributedMapConfig.builder() + .completionConfig(DistributedMapConfig.CompletionConfig.failurePercentage(5, 200)) + .build(); + var completion = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .completionConfig(); + assertEquals(5.0f, completion.toleratedFailurePercentage(), 0.0001f); + assertEquals(200, completion.minimumSampleSize().intValue()); + assertNull(completion.toleratedFailureCount()); + } + + @Test + void completionFailureCountSerializes() { + var config = DistributedMapConfig.builder() + .completionConfig(DistributedMapConfig.CompletionConfig.failureCount(2)) + .build(); + var completion = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .completionConfig(); + assertEquals(2, completion.toleratedFailureCount().intValue()); + } + + // ------------------------------------------------------------------ + // toOptions - destinations + // ------------------------------------------------------------------ + + @Test + void destinationBothSuccessAndFailureSerialize() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", false, true, null), + new DistributedMapDestination.Failure("out", "bad", true, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + var onSuccess = destination.onSuccess(); + assertEquals("S3", onSuccess.typeAsString()); + assertEquals(List.of("OUTPUT"), onSuccess.includeAsStrings()); + assertEquals("out", onSuccess.s3DestinationConfig().bucket()); + assertEquals("ok", onSuccess.s3DestinationConfig().keyPrefix()); + var onFailure = destination.onFailure(); + assertEquals(List.of("INPUT", "ERROR"), onFailure.includeAsStrings()); + assertEquals("bad", onFailure.s3DestinationConfig().keyPrefix()); + } + + @Test + void destinationOnlySuccessOmitsFailure() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", false, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + assertNotNull(destination.onSuccess()); + assertNull(destination.onFailure()); + } + + @Test + void destinationOnlyFailureOmitsSuccess() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", false, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + assertNotNull(destination.onFailure()); + assertNull(destination.onSuccess()); + } + + @Test + void allNullDestinationOmittedFromWire() { + var config = DistributedMapConfig.builder() + .destination(new DistributedMapDestination(null, null)) + .build(); + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false); + assertNull(options.destination()); + } + + @Test + void successDestinationIncludeInputAndOutputWithOwner() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", true, true, "123456789012"))) + .build(); + var onSuccess = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onSuccess(); + assertEquals(List.of("INPUT", "OUTPUT"), onSuccess.includeAsStrings()); + assertEquals("123456789012", onSuccess.s3DestinationConfig().expectedBucketOwner()); + } + + @Test + void successDestinationInputOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", true, false, null))) + .build(); + var onSuccess = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onSuccess(); + assertEquals(List.of("INPUT"), onSuccess.includeAsStrings()); + } + + @Test + void failureDestinationErrorOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", false, true, "123456789012"))) + .build(); + var onFailure = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onFailure(); + assertEquals(List.of("ERROR"), onFailure.includeAsStrings()); + assertEquals("123456789012", onFailure.s3DestinationConfig().expectedBucketOwner()); + } + + @Test + void failureDestinationInputOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", true, false, null))) + .build(); + var onFailure = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onFailure(); + assertEquals(List.of("INPUT"), onFailure.includeAsStrings()); + } + + // ------------------------------------------------------------------ + // toOptions - result collection + // ------------------------------------------------------------------ + + @Test + void resultCollectionInlineWhenCollecting() { + var options = options( + DistributedMapSource.inline(List.of("a")), + batchProcessor(), + DistributedMapConfig.builder().build(), + true); + assertEquals("INLINE", options.resultCollection().modeAsString()); + } + + @Test + void resultCollectionOmittedWhenNotCollecting() { + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor()); + assertNull(options.resultCollection()); + } + + // ------------------------------------------------------------------ + // toSummary + // ------------------------------------------------------------------ + + @Test + void toSummaryMapsDetailFields() { + var details = detailsBuilder() + .successCount(5L) + .totalCount(5L) + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .build(); + var summary = DistributedMapOperation.toSummary(DistributedMapStatus.SUCCEEDED, details); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertTrue(summary.sourceExhausted()); + assertEquals(5L, summary.successCount()); + assertEquals(5L, summary.totalCount().longValue()); + assertEquals("arn:aws:lambda:us-east-1:123456789012:map-run:abc", summary.distributedMapRunArn()); + } + + @Test + void toSummaryCarriesTheRunError() { + var details = detailsBuilder() + .sourceExhausted(false) + .error(ErrorObject.builder() + .errorType("SourceFailed") + .errorMessage("bad object") + .errorData("{}") + .stackTrace(List.of("frame")) + .build()) + .build(); + var summary = DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details); + assertFalse(summary.sourceExhausted()); + assertEquals("SourceFailed", summary.error().errorType()); + assertEquals("bad object", summary.error().errorMessage()); + assertEquals("{}", summary.error().errorData()); + assertEquals(List.of("frame"), summary.error().stackTrace()); + } + + @Test + void toSummaryUsesTheOperationStatus() { + assertEquals( + DistributedMapStatus.FAILED, + DistributedMapOperation.toSummary( + DistributedMapStatus.FAILED, detailsBuilder().build()) + .status()); + } + + @Test + void toSummaryLeavesSourceExhaustedNullWhenAbsent() { + var details = DistributedMapDetails.builder() + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + assertNull(DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details) + .sourceExhausted()); + } + + @ParameterizedTest + @ValueSource(strings = {"SuccessCount", "FailureCount", "UnprocessedCount"}) + void toSummaryThrowsWhenATerminalCountIsMissing(String missing) { + var details = DistributedMapDetails.builder() + .successCount("SuccessCount".equals(missing) ? null : 1L) + .failureCount("FailureCount".equals(missing) ? null : 0L) + .unprocessedCount("UnprocessedCount".equals(missing) ? null : 0L) + .build(); + + var exception = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details)); + + assertEquals("missing distributed map " + missing + " from backend", exception.getMessage()); + } + + @Test + void toSummaryThrowsWhenASucceededRunHasNoTotalCount() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + + var exception = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toSummary(DistributedMapStatus.SUCCEEDED, details)); + + assertEquals("missing distributed map TotalCount from backend", exception.getMessage()); + } + + @Test + void toSummaryAcceptsAnAbsentTotalCountWhenTheRunDidNotSucceed() { + // TotalCount is required only for a succeeded run, so a failed run may omit it. + var details = DistributedMapDetails.builder() + .successCount(0L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + + assertNull(DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details) + .totalCount()); + } + + @Test + void toSummaryLeavesTheRunErrorNullWhenAbsent() { + assertNull(DistributedMapOperation.toSummary( + DistributedMapStatus.SUCCEEDED, detailsBuilder().build()) + .error()); + } + + // ------------------------------------------------------------------ + // toResult + // ------------------------------------------------------------------ + + @Test + void toResultParsesSucceededAndFailedItems() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of( + succeededItem("0", "42"), + failedItem( + "1", + ErrorObject.builder() + .errorType("E") + .errorMessage("boom") + .build()))) + .build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(Integer.class), SERDES); + assertEquals(List.of(42), result.results()); + assertEquals(1, result.errors().size()); + assertEquals("E", result.errors().get(0).errorType()); + assertEquals("boom", result.errors().get(0).errorMessage()); + } + + @Test + void toResultDecodesOutputExactlyOnceThroughSerdes() { + var serdes = new UpperSerDes(); + var details = + detailsBuilder().results(List.of(succeededItem("0", "abc"))).build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), serdes); + assertEquals(List.of("ABC"), result.results()); + assertEquals(1, serdes.deserializeCount); + } + + @Test + void toResultKeepsNullOutputNull() { + var details = + detailsBuilder().results(List.of(succeededItem("0", null))).build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + assertNull(result.succeeded().get(0).output()); + assertEquals(1, result.results().size()); + assertNull(result.results().get(0)); + } + + @Test + void toResultKeepsNullItemErrorFieldsNull() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem("0", ErrorObject.builder().build()))) + .build(); + var error = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES) + .errors() + .get(0); + assertNull(error.errorType()); + assertNull(error.errorMessage()); + } + + @Test + void toResultCarriesEveryItemErrorMember() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem( + "0", + ErrorObject.builder() + .errorType("E") + .errorMessage("boom") + .errorData("{\"k\":1}") + .stackTrace(List.of("frame-1", "frame-2")) + .build()))) + .build(); + var error = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES) + .errors() + .get(0); + assertEquals("E", error.errorType()); + assertEquals("boom", error.errorMessage()); + assertEquals("{\"k\":1}", error.errorData()); + assertEquals(List.of("frame-1", "frame-2"), error.stackTrace()); + } + + @Test + void toResultKeepsFailedItemErrorNullWhenAbsent() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem("0", null))) + .build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + assertNull(result.failed().get(0).error()); + assertEquals(1, result.errors().size()); + assertNull(result.errors().get(0)); + } + + @Test + void toResultMapsUnrecognizedItemStatusToUnknownAndKeepsTheOtherItems() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of( + succeededItem("0", "\"a\""), + DistributedMapResultItem.builder() + .itemId("1") + .status("BOGUS") + .build(), + failedItem("2", ErrorObject.builder().errorType("E").build()))) + .build(); + + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + + assertEquals(3, result.items().size()); + assertEquals( + DistributedMapResult.Item.Status.UNKNOWN_TO_SDK_VERSION, + result.items().get(1).status()); + assertNull(result.items().get(1).output()); + assertNull(result.items().get(1).error()); + // The unknown item is in items() but in none of the four filtered accessors. + assertEquals( + List.of("0"), + result.succeeded().stream() + .map(DistributedMapResult.Item::itemId) + .toList()); + assertEquals( + List.of("2"), + result.failed().stream().map(DistributedMapResult.Item::itemId).toList()); + assertEquals(List.of("a"), result.results()); + assertEquals(1, result.errors().size()); + } + + @Test + void toResultThrowsOnMissingItemStatus() { + var details = detailsBuilder() + .results(List.of(DistributedMapResultItem.builder().itemId("0").build())) + .build(); + assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES)); + } + + @ParameterizedTest + @NullAndEmptySource + void toResultThrowsOnMissingItemId(String itemId) { + var details = detailsBuilder() + .results(List.of(DistributedMapResultItem.builder() + .itemId(itemId) + .status("SUCCEEDED") + .output("\"x\"") + .build())) + .build(); + var e = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES)); + assertEquals("missing distributed map item id from backend", e.getMessage()); + } +}