From 4237a80fe36b1d0c90efcf8026cd18566d4300e0 Mon Sep 17 00:00:00 2001 From: nvasiu Date: Fri, 28 Aug 2026 23:06:50 +0000 Subject: [PATCH 1/2] feat(sdk): add distributed map operation --- .../durable/DistributedMapHandlers.java | 265 +++++++++++++++ .../amazon/lambda/durable/DurableContext.java | 141 ++++++++ .../lambda/durable/config/CsvDelimiter.java | 23 ++ .../lambda/durable/config/CsvFormat.java | 45 +++ .../DistributedMapCompletionConfig.java | 45 +++ .../durable/config/DistributedMapConfig.java | 91 +++++ .../config/DistributedMapDestination.java | 44 +++ .../DistributedMapDestinationConfig.java | 22 ++ .../config/DistributedMapProcessor.java | 123 +++++++ .../durable/config/DistributedMapSource.java | 212 ++++++++++++ .../durable/config/FailureDestination.java | 17 + .../durable/config/ProcessorRetryConfig.java | 76 +++++ .../durable/config/SuccessDestination.java | 17 + .../durable/context/DurableContextImpl.java | 87 +++++ .../exception/DistributedMapError.java | 93 +++++ .../exception/DistributedMapException.java | 18 + .../model/DistributedMapCompletionReason.java | 47 +++ .../model/DistributedMapItemError.java | 12 + .../durable/model/DistributedMapResult.java | 112 ++++++ .../model/DistributedMapResultItem.java | 23 ++ .../durable/model/DistributedMapStatus.java | 37 ++ .../durable/model/DistributedMapSummary.java | 36 ++ .../durable/model/OperationSubType.java | 4 +- .../lambda/durable/model/ReaderPage.java | 15 + .../operation/DistributedMapOperation.java | 79 +++++ .../durable/operation/DistributedMapWire.java | 318 ++++++++++++++++++ .../util/DistributedMapValidation.java | 62 ++++ 27 files changed, 2063 insertions(+), 1 deletion(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java b/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java new file mode 100644 index 0000000000..28f8ccca41 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java @@ -0,0 +1,265 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import com.amazonaws.services.lambda.runtime.RequestHandler; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor.ResponseMode; +import software.amazon.lambda.durable.config.MapConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.MapResult; +import software.amazon.lambda.durable.model.ReaderPage; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Authoring helpers that wrap a customer function into a distributed map processor or reader Lambda handler. */ +public final class DistributedMapHandlers { + private static final SerDes JSON = new JacksonSerDes(); + private static final TypeToken OBJECT_TYPE = TypeToken.get(Object.class); + private static final TypeToken> EVENT_TYPE = new TypeToken>() {}; + private static final int READER_STATE_LIMIT = 32 * 1024; + private static final String ITEMS_OP_NAME = "distributed-map-items"; + + private DistributedMapHandlers() {} + + private static void validateItemReport(ResponseMode report) { + if (report != ResponseMode.REPORT_ITEM_RESULTS && report != ResponseMode.REPORT_FAILED_ITEMS) { + throw new IllegalArgumentException( + "item handler report mode must be REPORT_ITEM_RESULTS or REPORT_FAILED_ITEMS, got: " + report); + } + } + + /** Wraps a per-item function as a processor handler. Use REPORT_ITEM_RESULTS or REPORT_FAILED_ITEMS. */ + public static RequestHandler, Map> createDistributedMapItemHandler( + Function func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + ResponseMode report, + int concurrency) { + validateItemReport(report); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return (event, context) -> { + var records = records(event); + var workers = concurrency > 0 ? concurrency : Math.max(1, records.size()); + var outputs = new Object[records.size()]; + var errors = new Throwable[records.size()]; + var pool = Executors.newFixedThreadPool(workers); + try { + var futures = new ArrayList>(); + for (var i = 0; i < records.size(); i++) { + var index = i; + futures.add(pool.submit(() -> { + I item = toItem(inSerdes, records.get(index).get("body"), itemType); + outputs[index] = func.apply(item); + return null; + })); + } + for (var i = 0; i < futures.size(); i++) { + try { + futures.get(i).get(); + } catch (ExecutionException e) { + errors[i] = e.getCause() != null ? e.getCause() : e; + } + } + } catch (InterruptedException e) { + pool.shutdownNow(); + Thread.currentThread().interrupt(); + throw new IllegalStateException("Distributed map item handler interrupted", e); + } finally { + pool.shutdown(); + } + + var results = new ArrayList>(); + var failures = new ArrayList>(); + for (var i = 0; i < records.size(); i++) { + var itemId = (String) records.get(i).get("itemId"); + if (errors[i] != null) { + failures.add(errorEntry(itemId, errors[i].getClass().getName(), errors[i].getMessage())); + } else if (report == ResponseMode.REPORT_ITEM_RESULTS) { + results.add(resultEntry(itemId, toJsonValue(outSerdes, outputs[i]))); + } + } + return itemResponse(report, results, failures); + }; + } + + /** Wraps a whole-batch function as a processor handler. Returning succeeds every item, throwing fails the batch. */ + public static RequestHandler, Object> createDistributedMapBatchHandler( + Consumer> func, TypeToken itemType, SerDes itemSerDes) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return (event, context) -> { + func.accept(toItems(serdes, event, itemType)); + return null; + }; + } + + /** Wraps a reader function as a source handler. A null nextState signals the source is exhausted. */ + public static RequestHandler, Map> createDistributedMapReader( + Function> func, TypeToken stateType, SerDes stateSerDes) { + var serdes = stateSerDes != null ? stateSerDes : JSON; + return (event, context) -> { + if (!(event.get("maxItems") instanceof Number maxItemsValue)) { + throw new IllegalStateException("expected a distributed map reader event carrying a numeric 'maxItems' (got keys: " + + event.keySet() + "), the function must be registered as a distributed map reader"); + } + var rawState = (String) event.get("state"); + S state = rawState != null ? serdes.deserialize(rawState, stateType) : null; + var maxItems = maxItemsValue.intValue(); + + var page = func.apply(state); + if (page.items().size() > maxItems) { + throw new IllegalStateException( + "reader returned " + page.items().size() + " items, exceeding maxItems " + maxItems); + } + var response = new LinkedHashMap(); + response.put("items", page.items()); + if (page.nextState() != null) { + var nextState = serdes.serialize(page.nextState()); + if (nextState.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalStateException( + "reader nextState exceeds the " + (READER_STATE_LIMIT / 1024) + " KB limit"); + } + response.put("nextState", nextState); + } + return response; + }; + } + + /** Durable variant of the item handler. The function receives the per-item DurableContext. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + ResponseMode report) { + validateItemReport(report); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + var records = records(event); + var bodies = new ArrayList(records.size()); + for (var record : records) { + bodies.add(record.get("body")); + } + MapResult batch = ctx.map( + ITEMS_OP_NAME, + bodies, + OBJECT_TYPE, + (body, index, mapContext) -> + (Object) func.apply(mapContext, toItem(inSerdes, body, itemType)), + MapConfig.builder() + .completionConfig(CompletionConfig.allCompleted()) + .build()); + + var results = new ArrayList>(); + var failures = new ArrayList>(); + for (var i = 0; i < records.size(); i++) { + var itemId = (String) records.get(i).get("itemId"); + var item = batch.getItem(i); + if (item.status() == MapResult.MapResultItem.Status.SUCCEEDED) { + if (report == ResponseMode.REPORT_ITEM_RESULTS) { + results.add(resultEntry(itemId, toJsonValue(outSerdes, item.result()))); + } + } else { + var error = item.error(); + failures.add(errorEntry( + itemId, + error != null ? error.errorType() : "", + error != null ? error.errorMessage() : "")); + } + } + return itemResponse(report, results, failures); + }, + DurableConfig.defaultConfig()); + } + + /** Durable variant of the batch handler. The function receives the DurableContext. */ + public static + RequestHandler + createDistributedMapBatchHandlerWithDurableExecution( + BiConsumer> func, TypeToken itemType, SerDes itemSerDes) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + func.accept(ctx, toItems(serdes, event, itemType)); + return null; + }, + DurableConfig.defaultConfig()); + } + + @SuppressWarnings("unchecked") + private static List> records(Map event) { + if (!(event.get("records") instanceof List)) { + throw new IllegalStateException("expected a distributed map processor event carrying a 'records' array (got keys: " + + event.keySet() + "), the function must be triggered by a distributed map, not invoked directly"); + } + return (List>) event.get("records"); + } + + private static List toItems(SerDes serdes, Map event, TypeToken itemType) { + var records = records(event); + var items = new ArrayList(records.size()); + for (var record : records) { + items.add(toItem(serdes, record.get("body"), itemType)); + } + return items; + } + + private static I toItem(SerDes serdes, Object body, TypeToken itemType) { + return serdes.deserialize(JSON.serialize(body), itemType); + } + + private static Object toJsonValue(SerDes serdes, Object value) { + return JSON.deserialize(serdes.serialize(value), OBJECT_TYPE); + } + + private static Map resultEntry(String itemId, Object output) { + var entry = new LinkedHashMap(); + entry.put("itemIdentifier", itemId); + entry.put("output", output); + return entry; + } + + private static Map errorEntry(String itemId, String errorType, String errorMessage) { + var error = new LinkedHashMap(); + error.put("errorType", errorType); + error.put("errorMessage", errorMessage); + var entry = new LinkedHashMap(); + entry.put("itemIdentifier", itemId); + entry.put("error", error); + return entry; + } + + private static Map itemResponse( + ResponseMode report, List> results, List> failures) { + var response = new LinkedHashMap(); + if (report == ResponseMode.REPORT_FAILED_ITEMS) { + response.put("batchItemFailures", failures); + } else { + response.put("batchItemResults", results); + response.put("batchItemFailures", failures); + } + return response; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java index ce6eb70701..79a00cdc8d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java @@ -9,6 +9,9 @@ import java.util.function.Function; import java.util.function.Supplier; import software.amazon.lambda.durable.config.CallbackConfig; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.MapConfig; import software.amazon.lambda.durable.config.ParallelConfig; @@ -18,6 +21,8 @@ import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.config.WithRetryConfig; import software.amazon.lambda.durable.context.BaseContext; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.MapResult; import software.amazon.lambda.durable.model.WaitForConditionResult; @@ -543,6 +548,142 @@ default ParallelDurableFuture parallel(String name) { */ ParallelDurableFuture parallel(String name, ParallelConfig config); + // =============== distributedMap ================ + + /** Runs a distributed map with default config, blocking until the run reaches a terminal state. */ + default DistributedMapSummary distributedMap( + String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { + return distributedMapAsync( + name, source, processor, maxConcurrency, DistributedMapConfig.builder().build()) + .get(); + } + + /** Runs a distributed map, blocking until the run reaches a terminal state. */ + default DistributedMapSummary distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, config) + .get(); + } + + /** Runs a distributed map that collects per-item results with a TypeToken, blocking until the run reaches a terminal state. */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, resultType, config) + .get(); + } + + /** Runs a distributed map that collects per-item results with a result Class, blocking until the run reaches a terminal state. */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config) + .get(); + } + + /** Runs a distributed map that collects per-item results with a result Class and default config, blocking until the run reaches a terminal state. */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + TypeToken.get(resultType), + DistributedMapConfig.builder().build()) + .get(); + } + + /** Runs a distributed map that collects per-item results with a TypeToken and default config, blocking until the run reaches a terminal state. */ + default DistributedMapResult distributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType) { + return distributedMapAsync( + name, source, processor, maxConcurrency, resultType, DistributedMapConfig.builder().build()) + .get(); + } + + /** Asynchronously runs a distributed map with default config. */ + default DurableFuture distributedMapAsync( + String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { + return distributedMapAsync( + name, source, processor, maxConcurrency, DistributedMapConfig.builder().build()); + } + + /** Asynchronously runs a distributed map. */ + DurableFuture distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config); + + /** Asynchronously runs a distributed map that collects per-item results with a TypeToken. */ + DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config); + + /** Asynchronously runs a distributed map that collects per-item results with a result Class. */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType, + DistributedMapConfig config) { + return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config); + } + + /** Asynchronously runs a distributed map that collects per-item results with a result Class and default config. */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + Class resultType) { + return distributedMapAsync( + name, + source, + processor, + maxConcurrency, + TypeToken.get(resultType), + DistributedMapConfig.builder().build()); + } + + /** Asynchronously runs a distributed map that collects per-item results with a TypeToken and default config. */ + default DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType) { + return distributedMapAsync( + name, source, processor, maxConcurrency, resultType, DistributedMapConfig.builder().build()); + } + /** * Executes a submitter function and waits for an external callback, blocking until the callback completes. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java new file mode 100644 index 0000000000..df6d4a3554 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +/** Delimiter for CSV distributed map sources. */ +public enum CsvDelimiter { + COMMA("COMMA"), + PIPE("PIPE"), + SEMICOLON("SEMICOLON"), + SPACE("SPACE"), + TAB("TAB"); + + private final String value; + + CsvDelimiter(String value) { + this.value = value; + } + + /** Returns the wire-format string value. */ + public String getValue() { + return value; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java new file mode 100644 index 0000000000..61d9e2cd34 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.util.List; +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** CSV parsing options for a distributed map S3 source. */ +public record CsvFormat(HeaderLocation headerLocation, List columns, CsvDelimiter delimiter) { + + /** Where the column header row comes from. */ + public enum HeaderLocation { + FIRST_ROW, + GIVEN + } + + public CsvFormat { + columns = columns != null ? List.copyOf(columns) : null; + delimiter = delimiter != null ? delimiter : CsvDelimiter.COMMA; + } + + /** + * File has its own header row. Column names are used only for compile-time typing and are not sent on the wire. + */ + public static CsvFormat expectedColumns(List columns) { + DistributedMapValidation.validateColumns("expectedColumns", columns); + return new CsvFormat(HeaderLocation.FIRST_ROW, columns, CsvDelimiter.COMMA); + } + + /** File has no header row. The given names are the columns, sent on the wire, and the first row is data. */ + public static CsvFormat headers(List headers) { + DistributedMapValidation.validateColumns("headers", headers); + return new CsvFormat(HeaderLocation.GIVEN, headers, CsvDelimiter.COMMA); + } + + /** File has its own header row and no column names are declared. */ + public static CsvFormat firstRow() { + return new CsvFormat(HeaderLocation.FIRST_ROW, null, CsvDelimiter.COMMA); + } + + /** Returns a copy with the given delimiter. */ + public CsvFormat withDelimiter(CsvDelimiter delimiter) { + return new CsvFormat(headerLocation, columns, delimiter); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java new file mode 100644 index 0000000000..ee5bd864bd --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +/** Failure-tolerance configuration for a distributed map run. */ +public record DistributedMapCompletionConfig( + Integer toleratedFailureCount, Double toleratedFailurePercentage, Integer minimumSampleSize) { + + public DistributedMapCompletionConfig { + if (toleratedFailureCount != null && toleratedFailurePercentage != null) { + throw new IllegalArgumentException( + "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive"); + } + if (minimumSampleSize != null && toleratedFailurePercentage == null) { + throw new IllegalArgumentException("minimumSampleSize is only valid with toleratedFailurePercentage"); + } + if (toleratedFailureCount != null && toleratedFailureCount < 0) { + throw new IllegalArgumentException( + "toleratedFailureCount must be non-negative, got: " + toleratedFailureCount); + } + if (toleratedFailurePercentage != null + && (toleratedFailurePercentage < 0 || toleratedFailurePercentage > 100)) { + throw new IllegalArgumentException( + "toleratedFailurePercentage must be between 0 and 100, got: " + toleratedFailurePercentage); + } + if (minimumSampleSize != null && minimumSampleSize < 1) { + throw new IllegalArgumentException("minimumSampleSize must be at least 1, got: " + minimumSampleSize); + } + } + + /** Abort once this many items have permanently failed. */ + public static DistributedMapCompletionConfig failureCount(int count) { + return new DistributedMapCompletionConfig(count, null, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100). */ + public static DistributedMapCompletionConfig failurePercentage(double percentage) { + return new DistributedMapCompletionConfig(null, percentage, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100), gated by a minimum sample size. */ + public static DistributedMapCompletionConfig failurePercentage(double percentage, int minimumSampleSize) { + return new DistributedMapCompletionConfig(null, percentage, minimumSampleSize); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java new file mode 100644 index 0000000000..0ea953c1e2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.time.Duration; +import software.amazon.lambda.durable.serde.SerDes; + +/** Optional configuration for a distributed map run. */ +public class DistributedMapConfig { + private static final long MAX_TIMEOUT_SECONDS = 7776000; // 90 days + + private final DistributedMapDestinationConfig destination; + private final DistributedMapCompletionConfig completionConfig; + private final Duration timeout; + private final SerDes resultSerDes; + + private DistributedMapConfig(Builder builder) { + if (builder.timeout != null + && (builder.timeout.toSeconds() <= 0 || builder.timeout.toSeconds() > MAX_TIMEOUT_SECONDS)) { + throw new IllegalArgumentException( + "timeout must be positive and at most 90 days, got: " + builder.timeout.toSeconds() + "s"); + } + this.destination = builder.destination; + this.completionConfig = builder.completionConfig; + this.timeout = builder.timeout; + this.resultSerDes = builder.resultSerDes; + } + + public DistributedMapDestinationConfig destination() { + return destination; + } + + public DistributedMapCompletionConfig completionConfig() { + return completionConfig; + } + + public Duration timeout() { + return timeout; + } + + public SerDes resultSerDes() { + return resultSerDes; + } + + public static Builder builder() { + return new Builder(); + } + + public Builder toBuilder() { + var builder = new Builder(); + builder.destination = destination; + builder.completionConfig = completionConfig; + builder.timeout = timeout; + builder.resultSerDes = resultSerDes; + return builder; + } + + /** Builder for DistributedMapConfig. */ + public static class Builder { + private DistributedMapDestinationConfig destination; + private DistributedMapCompletionConfig completionConfig; + private Duration timeout; + private SerDes resultSerDes; + + private Builder() {} + + public Builder destination(DistributedMapDestinationConfig destination) { + this.destination = destination; + return this; + } + + public Builder completionConfig(DistributedMapCompletionConfig completionConfig) { + this.completionConfig = completionConfig; + return this; + } + + public Builder timeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + public Builder resultSerDes(SerDes resultSerDes) { + this.resultSerDes = resultSerDes; + return this; + } + + public DistributedMapConfig build() { + return new DistributedMapConfig(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java new file mode 100644 index 0000000000..7a9dd1bb1d --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** Destination factories for a distributed map run. */ +public final class DistributedMapDestination { + private DistributedMapDestination() {} + + /** Route succeeded item records to an S3 prefix (defaults to include output only). */ + public static SuccessDestination successes(String prefixUri) { + return successes(prefixUri, false, true, null); + } + + /** Route succeeded item records to an S3 prefix. */ + public static SuccessDestination successes( + String prefixUri, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return new SuccessDestination( + parsed.bucket(), + parsed.path() != null ? parsed.path() : "", + includeInput, + includeOutput, + expectedBucketOwner); + } + + /** Route permanently-failed item records to an S3 prefix (defaults to include input and error). */ + public static FailureDestination failures(String prefixUri) { + return failures(prefixUri, true, true, null); + } + + /** Route permanently-failed item records to an S3 prefix. */ + public static FailureDestination failures( + String prefixUri, boolean includeInput, boolean includeError, String expectedBucketOwner) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return new FailureDestination( + parsed.bucket(), + parsed.path() != null ? parsed.path() : "", + includeInput, + includeError, + expectedBucketOwner); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java new file mode 100644 index 0000000000..4ce47e60b8 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +/** Destination routing for distributed map run results. */ +public record DistributedMapDestinationConfig(SuccessDestination onSuccess, FailureDestination onFailure) { + + /** Routes only succeeded item records. */ + public static DistributedMapDestinationConfig onSuccess(SuccessDestination onSuccess) { + return new DistributedMapDestinationConfig(onSuccess, null); + } + + /** Routes only failed item records. */ + public static DistributedMapDestinationConfig onFailure(FailureDestination onFailure) { + return new DistributedMapDestinationConfig(null, onFailure); + } + + /** Routes both succeeded and failed item records. */ + public static DistributedMapDestinationConfig of(SuccessDestination onSuccess, FailureDestination onFailure) { + return new DistributedMapDestinationConfig(onSuccess, onFailure); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java new file mode 100644 index 0000000000..db38c7e94c --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java @@ -0,0 +1,123 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** Processor configuration for a distributed map run. */ +public class DistributedMapProcessor { + + /** Per-batch response mode reported by the processor. */ + public enum ResponseMode { + REPORT_BATCH_OUTCOME(null), + REPORT_FAILED_ITEMS("REPORT_BATCH_ITEM_FAILURES"), + REPORT_ITEM_RESULTS("REPORT_BATCH_ITEM_RESULTS"); + + private final String value; + + ResponseMode(String value) { + this.value = value; + } + + /** Returns the wire functionResponseTypes value, or null for the batch-outcome mode. */ + public String getValue() { + return value; + } + } + + private static final int MIN_BATCH_SIZE = 1; + private static final int MAX_BATCH_SIZE = 10000; + private static final int MAX_NAME_PREFIX_LENGTH = 36; + + private final String functionName; + private final ResponseMode responseMode; + private final Integer batchSize; + private final ProcessorRetryConfig retryConfig; + private final String durableExecutionNamePrefix; + + private DistributedMapProcessor(Builder builder) { + DistributedMapValidation.validateFunctionName(builder.functionName); + if (builder.batchSize != null && (builder.batchSize < MIN_BATCH_SIZE || builder.batchSize > MAX_BATCH_SIZE)) { + throw new IllegalArgumentException("batchSize must be between 1 and 10000, got: " + builder.batchSize); + } + if (builder.durableExecutionNamePrefix != null + && (builder.durableExecutionNamePrefix.isEmpty() + || builder.durableExecutionNamePrefix.length() > MAX_NAME_PREFIX_LENGTH)) { + throw new IllegalArgumentException("durableExecutionNamePrefix must be between 1 and 36 characters, got: " + + builder.durableExecutionNamePrefix.length()); + } + this.functionName = builder.functionName; + this.responseMode = builder.responseMode; + this.batchSize = builder.batchSize; + this.retryConfig = builder.retryConfig; + this.durableExecutionNamePrefix = builder.durableExecutionNamePrefix; + } + + public String functionName() { + return functionName; + } + + public ResponseMode responseMode() { + return responseMode; + } + + public Integer batchSize() { + return batchSize; + } + + public ProcessorRetryConfig retryConfig() { + return retryConfig; + } + + public String durableExecutionNamePrefix() { + return durableExecutionNamePrefix; + } + + /** Processor that reports a single pass/fail outcome for the whole batch, with no per-item results. */ + public static Builder reportBatchOutcome(String functionName) { + return new Builder(functionName, ResponseMode.REPORT_BATCH_OUTCOME); + } + + /** Processor that reports the ids of failed items, with all others marked succeeded. */ + public static Builder reportFailedItems(String functionName) { + return new Builder(functionName, ResponseMode.REPORT_FAILED_ITEMS); + } + + /** Processor that reports the results (output or error) for every item. */ + public static Builder reportItemResults(String functionName) { + return new Builder(functionName, ResponseMode.REPORT_ITEM_RESULTS); + } + + /** Builder for DistributedMapProcessor. */ + public static class Builder { + private final String functionName; + private final ResponseMode responseMode; + private Integer batchSize; + private ProcessorRetryConfig retryConfig; + private String durableExecutionNamePrefix; + + private Builder(String functionName, ResponseMode responseMode) { + this.functionName = functionName; + this.responseMode = responseMode; + } + + public Builder batchSize(int batchSize) { + this.batchSize = batchSize; + return this; + } + + public Builder retryConfig(ProcessorRetryConfig retryConfig) { + this.retryConfig = retryConfig; + return this; + } + + public Builder durableExecutionNamePrefix(String durableExecutionNamePrefix) { + this.durableExecutionNamePrefix = durableExecutionNamePrefix; + return this; + } + + public DistributedMapProcessor build() { + return new DistributedMapProcessor(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java new file mode 100644 index 0000000000..3d3607d4c7 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java @@ -0,0 +1,212 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.util.Collection; +import java.util.List; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.util.DistributedMapValidation; +import software.amazon.lambda.durable.util.ParameterValidator; + +/** + * Source of items for a distributed map run. + * + * @param the item type produced by the source + */ +public class DistributedMapSource { + + /** The kind of source backing a distributed map. */ + public enum SourceType { + INLINE, + S3, + READER_FUNCTION + } + + /** How the bytes of an S3 object are split into items. */ + public enum Format { + JSON_LINES, + JSON_ARRAY, + CSV + } + + /** Whether the backend flattens object contents into items or treats each object as one item. */ + public enum Transform { + NONE, + LOAD_AND_FLATTEN + } + + private final SourceType sourceType; + private final Long maxItems; + private final List inlineItems; + private final SerDes inlineSerDes; + private final S3SourceConfig s3; + private final ReaderSourceConfig reader; + + private DistributedMapSource( + SourceType sourceType, + Long maxItems, + List inlineItems, + SerDes inlineSerDes, + S3SourceConfig s3, + ReaderSourceConfig reader) { + if (maxItems != null && maxItems < 1) { + throw new IllegalArgumentException("maxItems must be at least 1, got: " + maxItems); + } + this.sourceType = sourceType; + this.maxItems = maxItems; + this.inlineItems = inlineItems != null ? List.copyOf(inlineItems) : null; + this.inlineSerDes = inlineSerDes; + this.s3 = s3; + this.reader = reader; + } + + public SourceType sourceType() { + return sourceType; + } + + public Long maxItems() { + return maxItems; + } + + public List inlineItems() { + return inlineItems; + } + + public SerDes inlineSerDes() { + return inlineSerDes; + } + + public S3SourceConfig s3() { + return s3; + } + + public ReaderSourceConfig reader() { + return reader; + } + + /** An in-memory list of items embedded in the start checkpoint. */ + public static DistributedMapSource inline(Collection items) { + return inline(items, null); + } + + /** An in-memory list of items embedded in the start checkpoint, serialized with the given SerDes. */ + public static DistributedMapSource inline(Collection items, SerDes serDes) { + ParameterValidator.validateOrderedCollection(items); + return new DistributedMapSource<>(SourceType.INLINE, null, List.copyOf(items), serDes, null, null); + } + + /** Read a single S3 object, treating each line as an item. */ + public static DistributedMapSource s3JsonLines(String uri) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3JsonLines requires an S3 object key"); + } + return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.JSON_LINES, null, null)); + } + + /** Read a single S3 object holding a JSON array, treating each element as an item. */ + public static DistributedMapSource s3JsonArray(String uri) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3JsonArray requires an S3 object key"); + } + return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.JSON_ARRAY, null, null)); + } + + /** Read a single S3 object, treating each record as an item. */ + public static DistributedMapSource s3Csv(String uri, CsvFormat format) { + var parsed = DistributedMapValidation.parseS3Uri(uri); + if (parsed.path() == null) { + throw new IllegalArgumentException("s3Csv requires an S3 object key"); + } + return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.CSV, format, null)); + } + + /** Read each object under a prefix as one item (object contents are not read). */ + public static DistributedMapSource s3Objects(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(new S3SourceConfig(parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.NONE, null, null, null)); + } + + /** Read a prefix, flattening each object's lines into items. */ + public static DistributedMapSource s3FlattenedJsonLines(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(new S3SourceConfig( + parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.JSON_LINES, null, null)); + } + + /** Read a prefix, flattening each object's JSON array elements into items. */ + public static DistributedMapSource s3FlattenedJsonArray(String prefixUri) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(new S3SourceConfig( + parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.JSON_ARRAY, null, null)); + } + + /** Read a prefix, flattening each object's records into items. */ + public static DistributedMapSource s3FlattenedCsv(String prefixUri, CsvFormat format) { + var parsed = DistributedMapValidation.parseS3Uri(prefixUri); + return s3(new S3SourceConfig( + parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.CSV, format, null)); + } + + /** Page items from a customer-supplied reader Lambda function. */ + public static DistributedMapSource reader(String functionName) { + return reader(functionName, null, null); + } + + /** Page items from a customer-supplied reader Lambda function with an initial state. */ + public static DistributedMapSource reader(String functionName, S initialState, SerDes stateSerDes) { + return new DistributedMapSource<>( + SourceType.READER_FUNCTION, + null, + null, + null, + null, + new ReaderSourceConfig(functionName, initialState, stateSerDes)); + } + + /** Returns a copy limiting the total number of items read. */ + public DistributedMapSource maxItemsToRead(long maxItems) { + return new DistributedMapSource<>(sourceType, maxItems, inlineItems, inlineSerDes, s3, reader); + } + + /** Returns a copy asserting the expected S3 bucket owner (S3 sources only). */ + public DistributedMapSource expectedBucketOwner(String accountId) { + if (s3 == null) { + throw new IllegalStateException("expectedBucketOwner only applies to S3 sources"); + } + DistributedMapValidation.validateBucketOwner(accountId); + return new DistributedMapSource<>( + sourceType, maxItems, inlineItems, inlineSerDes, s3.withExpectedBucketOwner(accountId), reader); + } + + private static DistributedMapSource s3(S3SourceConfig s3) { + return new DistributedMapSource<>(SourceType.S3, null, null, null, s3, null); + } + + private static String prefixOrEmpty(String path) { + return path != null ? path : ""; + } + + /** Resolved S3 source configuration. */ + public record S3SourceConfig( + String bucket, + String key, + String prefix, + Transform transform, + Format format, + CsvFormat csvFormat, + String expectedBucketOwner) { + + S3SourceConfig withExpectedBucketOwner(String owner) { + return new S3SourceConfig(bucket, key, prefix, transform, format, csvFormat, owner); + } + } + + /** Resolved reader-function source configuration. */ + public record ReaderSourceConfig(String functionName, Object initialState, SerDes stateSerDes) { + public ReaderSourceConfig { + DistributedMapValidation.validateFunctionName(functionName); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java new file mode 100644 index 0000000000..87a9aa822c --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** S3 destination for permanently-failed distributed map items. */ +public record FailureDestination( + String bucket, String prefix, boolean includeInput, boolean includeError, String expectedBucketOwner) { + + public FailureDestination { + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + if (!includeInput && !includeError) { + throw new IllegalArgumentException("failure destination must include at least one of input or error"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java new file mode 100644 index 0000000000..4fa361fce1 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import java.time.Duration; + +/** Retry configuration for a distributed map processor. */ +public class ProcessorRetryConfig { + /** Sentinel for unbounded retry attempts (wire value -1). */ + public static final int UNLIMITED = -1; + + private static final long MIN_RETRY_DURATION_SECONDS = 60; + private static final long MAX_RETRY_DURATION_SECONDS = 21600; + + private final Integer maxRetryAttempts; + private final Duration maxRetryDuration; + + private ProcessorRetryConfig(Builder builder) { + this.maxRetryAttempts = builder.maxRetryAttempts; + this.maxRetryDuration = builder.maxRetryDuration; + } + + /** Returns the max retry attempts, UNLIMITED for unbounded, or null for the default. */ + public Integer maxRetryAttempts() { + return maxRetryAttempts; + } + + /** Returns the cumulative retry duration budget, or null for the default. */ + public Duration maxRetryDuration() { + return maxRetryDuration; + } + + public static Builder builder() { + return new Builder(); + } + + public Builder toBuilder() { + var builder = new Builder(); + builder.maxRetryAttempts = maxRetryAttempts; + builder.maxRetryDuration = maxRetryDuration; + return builder; + } + + /** Builder for ProcessorRetryConfig. */ + public static class Builder { + private Integer maxRetryAttempts; + private Duration maxRetryDuration; + + private Builder() {} + + public Builder maxRetryAttempts(int maxRetryAttempts) { + if (maxRetryAttempts < 0 && maxRetryAttempts != UNLIMITED) { + throw new IllegalArgumentException( + "maxRetryAttempts must be non-negative or ProcessorRetryConfig.UNLIMITED, got: " + + maxRetryAttempts); + } + this.maxRetryAttempts = maxRetryAttempts; + return this; + } + + public Builder maxRetryDuration(Duration maxRetryDuration) { + if (maxRetryDuration != null + && (maxRetryDuration.toSeconds() < MIN_RETRY_DURATION_SECONDS + || maxRetryDuration.toSeconds() > MAX_RETRY_DURATION_SECONDS)) { + throw new IllegalArgumentException("maxRetryDuration must be between 1 minute and 6 hours, got: " + + maxRetryDuration.toSeconds() + "s"); + } + this.maxRetryDuration = maxRetryDuration; + return this; + } + + public ProcessorRetryConfig build() { + return new ProcessorRetryConfig(this); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java new file mode 100644 index 0000000000..42a31cd0fe --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import software.amazon.lambda.durable.util.DistributedMapValidation; + +/** S3 destination for succeeded distributed map items. */ +public record SuccessDestination( + String bucket, String prefix, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { + + public SuccessDestination { + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + if (!includeInput && !includeOutput) { + throw new IllegalArgumentException("success destination must include at least one of input or output"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java index 0c79165ec3..69a45cc882 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.context; import com.amazonaws.services.lambda.runtime.Context; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; import java.time.Duration; import java.util.Collection; import java.util.List; @@ -18,6 +19,9 @@ import software.amazon.lambda.durable.StepContext; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.CallbackConfig; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.MapConfig; import software.amazon.lambda.durable.config.ParallelConfig; @@ -31,12 +35,16 @@ import software.amazon.lambda.durable.execution.OperationIdGenerator; import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.MapResult; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.operation.CallbackOperation; import software.amazon.lambda.durable.operation.ChildContextOperation; +import software.amazon.lambda.durable.operation.DistributedMapOperation; +import software.amazon.lambda.durable.operation.DistributedMapWire; import software.amazon.lambda.durable.operation.InvokeOperation; import software.amazon.lambda.durable.operation.MapOperation; import software.amazon.lambda.durable.operation.ParallelOperation; @@ -371,6 +379,85 @@ public DurableFuture waitForConditionAsync( return operation; } + // =============== distributedMap ================ + + @Override + public DurableFuture distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config) { + return startDistributedMap( + name, + source, + processor, + maxConcurrency, + config, + false, + TypeToken.get(DistributedMapSummary.class), + DistributedMapWire::toSummary); + } + + @Override + public DurableFuture> distributedMapAsync( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + TypeToken resultType, + DistributedMapConfig config) { + Objects.requireNonNull(resultType, "resultType cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + var resultSerDes = + config.resultSerDes() != null ? config.resultSerDes() : getDurableConfig().getSerDes(); + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeToken> token = (TypeToken) TypeToken.get(DistributedMapResult.class); + return startDistributedMap( + name, + source, + processor, + maxConcurrency, + config, + true, + token, + details -> DistributedMapWire.toResult(details, resultType, resultSerDes)); + } + + private DurableFuture startDistributedMap( + String name, + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config, + boolean collectResults, + TypeToken resultType, + Function resultBuilder) { + Objects.requireNonNull(source, "source cannot be null"); + Objects.requireNonNull(processor, "processor cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + ParameterValidator.validateOperationName(name); + if (maxConcurrency <= 0) { + throw new IllegalArgumentException("maxConcurrency must be greater than zero, got: " + maxConcurrency); + } + if (!collectResults && config.resultSerDes() != null) { + throw new IllegalArgumentException( + "resultSerDes is set but this distributedMap call does not collect results, use a resultType overload"); + } + var options = DistributedMapWire.toOptions( + source, processor, maxConcurrency, config, getDurableConfig().getSerDes(), collectResults); + var operationId = nextOperationId(); + var operation = new DistributedMapOperation<>( + OperationIdentifier.of(operationId, name, OperationSubType.DISTRIBUTED_MAP), + options, + resultType, + getDurableConfig().getSerDes(), + resultBuilder, + this); + operation.execute(); + return operation; + } + // =============== withRetry ================ private static final Duration DEFAULT_BACKOFF_DELAY = Duration.ofSeconds(1); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java new file mode 100644 index 0000000000..49d90359c2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java @@ -0,0 +1,93 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.exception; + +import software.amazon.lambda.durable.model.DistributedMapCompletionReason; +import software.amazon.lambda.durable.model.DistributedMapItemError; +import software.amazon.lambda.durable.model.DistributedMapStatus; + +/** + * Run outcome error, thrown by throwIfError when a distributed map run did not fully succeed. + * It reports an aggregate run result and is not a single operation failure, so it stays out of the DurableOperationException hierarchy. + * See DistributedMapException for a failure of the operation itself. + */ +public class DistributedMapError extends DurableExecutionException { + private final DistributedMapStatus status; + private final DistributedMapCompletionReason completionReason; + private final long failureCount; + + private DistributedMapError( + DistributedMapStatus status, + DistributedMapCompletionReason completionReason, + long failureCount, + String message) { + super(message); + this.status = status; + this.completionReason = completionReason; + this.failureCount = failureCount; + } + + /** Run-level failure describing the run status, reason, and failure count. */ + public static DistributedMapError runLevel( + DistributedMapStatus status, + DistributedMapCompletionReason completionReason, + long failureCount, + String completionDetails) { + return new DistributedMapError( + status, + completionReason, + failureCount, + runMessage(status, completionReason, failureCount, completionDetails)); + } + + /** Item-level failure surfacing the first failed item's error. */ + public static DistributedMapError itemLevel( + DistributedMapStatus status, + DistributedMapCompletionReason completionReason, + long failureCount, + DistributedMapItemError error, + String itemId) { + return new DistributedMapError(status, completionReason, failureCount, itemMessage(error, itemId)); + } + + /** Returns the terminal run status. */ + public DistributedMapStatus status() { + return status; + } + + /** Returns the completion reason. */ + public DistributedMapCompletionReason completionReason() { + return completionReason; + } + + /** Returns the count of failed items. */ + public long failureCount() { + return failureCount; + } + + private static String runMessage( + DistributedMapStatus status, + DistributedMapCompletionReason completionReason, + long failureCount, + String completionDetails) { + var message = new StringBuilder("Distributed map run ") + .append(status) + .append(" (") + .append(completionReason) + .append(")"); + if (failureCount > 0) { + message.append(", ").append(failureCount).append(" item(s) failed"); + } + if (completionDetails != null) { + message.append(": ").append(completionDetails); + } + return message.toString(); + } + + private static String itemMessage(DistributedMapItemError error, String itemId) { + if (error != null) { + return error.errorType() + ": " + error.errorMessage(); + } + return "Distributed map item " + itemId + " failed"; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java new file mode 100644 index 0000000000..d8ee6d20d8 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.exception; + +import software.amazon.awssdk.services.lambda.model.Operation; + +/** + * Operation failure, thrown when a distributed map operation itself terminal-fails (FAILED, TIMED_OUT, or STOPPED). + * Extends DurableOperationException. See DistributedMapError for the run outcome, which is an aggregate result rather than an operation failure. + */ +public class DistributedMapException extends DurableOperationException { + public DistributedMapException(Operation operation) { + super( + operation, + null, + "Distributed map operation " + operation.id() + " ended with status " + operation.statusAsString()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java new file mode 100644 index 0000000000..17377cbf58 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java @@ -0,0 +1,47 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +/** Reason a distributed map run reached its terminal status. */ +public enum DistributedMapCompletionReason { + ALL_COMPLETED("ALL_COMPLETED"), + ITEM_LIMIT_REACHED("ITEM_LIMIT_REACHED"), + STOPPED("STOPPED"), + TIMED_OUT("TIMED_OUT"), + FAILURE_TOLERANCE_EXCEEDED("FAILURE_TOLERANCE_EXCEEDED"), + SOURCE_FAILED("SOURCE_FAILED"), + DESTINATION_FAILED("DESTINATION_FAILED"), + INLINE_RESULT_LIMIT_EXCEEDED("INLINE_RESULT_LIMIT_EXCEEDED"), + INVALID_CONFIGURATION("INVALID_CONFIGURATION"), + QUOTA_EXCEEDED("QUOTA_EXCEEDED"), + KMS_ACCESS_DENIED("KMS_ACCESS_DENIED"), + INTERNAL_ERROR("INTERNAL_ERROR"), + /** A value the backend returned that this SDK version does not recognize. */ + UNKNOWN_TO_SDK_VERSION("UNKNOWN_TO_SDK_VERSION"); + + private final String value; + + DistributedMapCompletionReason(String value) { + this.value = value; + } + + /** Returns the wire-format string value. */ + public String getValue() { + return value; + } + + /** Returns the reason matching a wire-format string value, or UNKNOWN_TO_SDK_VERSION if unrecognized. */ + public static DistributedMapCompletionReason fromValue(String value) { + for (var reason : values()) { + if (reason.value.equals(value)) { + return reason; + } + } + return UNKNOWN_TO_SDK_VERSION; + } + + @Override + public String toString() { + return value; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java new file mode 100644 index 0000000000..4af34e1662 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +/** Error details for a failed distributed map item. */ +public record DistributedMapItemError(String errorType, String errorMessage) { + + /** Creates an item error from a throwable. */ + public static DistributedMapItemError of(Throwable e) { + return new DistributedMapItemError(e.getClass().getName(), e.getMessage()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java new file mode 100644 index 0000000000..7a079eb5d2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java @@ -0,0 +1,112 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import software.amazon.lambda.durable.exception.DistributedMapError; + +/** Outcome of a distributed map run with collected per-item results. */ +public record DistributedMapResult(DistributedMapSummary summary, List> items) { + + /** Applies a defensive copy and defaults. */ + public DistributedMapResult { + items = items != null ? List.copyOf(items) : Collections.emptyList(); + } + + /** Returns items that succeeded. */ + public List> succeeded() { + return items.stream() + .filter(item -> item.status() == DistributedMapResultItem.Status.SUCCEEDED) + .toList(); + } + + /** Returns items that failed. */ + public List> failed() { + return items.stream() + .filter(item -> item.status() == DistributedMapResultItem.Status.FAILED) + .toList(); + } + + /** Returns the outputs of succeeded items. */ + public List getResults() { + return succeeded().stream().map(DistributedMapResultItem::output).filter(Objects::nonNull).toList(); + } + + /** Returns the errors of failed items. */ + public List getErrors() { + return failed().stream().map(DistributedMapResultItem::error).filter(Objects::nonNull).toList(); + } + + /** Returns the terminal run status. */ + public DistributedMapStatus status() { + return summary.status(); + } + + /** Returns the completion reason. */ + public DistributedMapCompletionReason completionReason() { + return summary.completionReason(); + } + + /** Returns the count of succeeded items. */ + public long successCount() { + return summary.successCount(); + } + + /** Returns the count of failed items. */ + public long failureCount() { + return summary.failureCount(); + } + + /** Returns the count of unprocessed items. */ + public long unprocessedCount() { + return summary.unprocessedCount(); + } + + /** Returns the total item count, or null until known. */ + public Long totalCount() { + return summary.totalCount(); + } + + /** Returns the run ARN, or null when the run never started. */ + public String distributedMapRunArn() { + return summary.distributedMapRunArn(); + } + + /** Returns the run id derived from the ARN, or null when the run never started. */ + public String distributedMapId() { + return summary.distributedMapId(); + } + + /** Returns the completion details, or null when absent. */ + public String completionDetails() { + return summary.completionDetails(); + } + + /** Returns true when any item failed. */ + public boolean hasFailure() { + return summary.hasFailure(); + } + + /** Throws DistributedMapError when the run did not fully succeed, surfacing the first failed item. */ + public void throwIfError() { + if (summary.status() != DistributedMapStatus.SUCCEEDED) { + summary.throwIfError(); + return; + } + var failedItems = failed(); + if (!failedItems.isEmpty()) { + var first = failedItems.get(0); + throw DistributedMapError.itemLevel( + summary.status(), + summary.completionReason(), + summary.failureCount(), + first.error(), + first.itemId()); + } + if (summary.hasFailure()) { + summary.throwIfError(); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java new file mode 100644 index 0000000000..864cae769b --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java @@ -0,0 +1,23 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +/** Outcome of a single distributed map item. */ +public record DistributedMapResultItem(String itemId, Status status, O output, DistributedMapItemError error) { + + /** Status of an individual distributed map item. */ + public enum Status { + SUCCEEDED, + FAILED + } + + /** Creates a succeeded item. */ + public static DistributedMapResultItem succeeded(String itemId, O output) { + return new DistributedMapResultItem<>(itemId, Status.SUCCEEDED, output, null); + } + + /** Creates a failed item. */ + public static DistributedMapResultItem failed(String itemId, DistributedMapItemError error) { + return new DistributedMapResultItem<>(itemId, Status.FAILED, null, error); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java new file mode 100644 index 0000000000..f704ca3536 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +/** Terminal status of a distributed map run. */ +public enum DistributedMapStatus { + SUCCEEDED("SUCCEEDED"), + FAILED("FAILED"), + STOPPED("STOPPED"), + TIMED_OUT("TIMED_OUT"); + + private final String value; + + DistributedMapStatus(String value) { + this.value = value; + } + + /** Returns the wire-format string value. */ + public String getValue() { + return value; + } + + /** Returns the status matching a wire-format string value. */ + public static DistributedMapStatus fromValue(String value) { + for (var status : values()) { + if (status.value.equals(value)) { + return status; + } + } + throw new IllegalArgumentException("Unknown DistributedMapStatus: " + value); + } + + @Override + public String toString() { + return value; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java new file mode 100644 index 0000000000..8f26f03f02 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import software.amazon.lambda.durable.exception.DistributedMapError; + +/** Outcome of a distributed map run, without per-item results. */ +public record DistributedMapSummary( + DistributedMapStatus status, + DistributedMapCompletionReason completionReason, + long successCount, + long failureCount, + long unprocessedCount, + String distributedMapRunArn, + String completionDetails, + Long totalCount) { + + /** Returns the run id derived from the ARN, or null when the run never started. */ + public String distributedMapId() { + return distributedMapRunArn != null + ? distributedMapRunArn.substring(distributedMapRunArn.lastIndexOf(':') + 1) + : null; + } + + /** Returns true when any item failed. */ + public boolean hasFailure() { + return failureCount > 0; + } + + /** Throws DistributedMapError when the run did not fully succeed. */ + public void throwIfError() { + if (status != DistributedMapStatus.SUCCEEDED || hasFailure()) { + throw DistributedMapError.runLevel(status, completionReason, failureCount, completionDetails); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java index c7273b9cde..2228b28f42 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java @@ -22,7 +22,9 @@ public enum OperationSubType { PARALLEL_BRANCH(OperationType.CONTEXT, "ParallelBranch"), WAIT_FOR_CALLBACK(OperationType.CONTEXT, "WaitForCallback"), WAIT_FOR_CONDITION(OperationType.STEP, "WaitForCondition"), - WITH_RETRY(OperationType.CONTEXT, "WithRetry"); + WITH_RETRY(OperationType.CONTEXT, "WithRetry"), + // Requires OperationType.DISTRIBUTED_MAP, generated once the Lambda client ships the distributed-map shapes. + DISTRIBUTED_MAP(OperationType.DISTRIBUTED_MAP, "DistributedMap"); private final OperationType operationType; private final String value; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java b/sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java new file mode 100644 index 0000000000..173bac2d55 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java @@ -0,0 +1,15 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import java.util.Collections; +import java.util.List; + +/** A page of items and the next state returned by a distributed map reader function. */ +public record ReaderPage(List items, S nextState) { + + /** Applies a defensive copy and defaults. */ + public ReaderPage { + items = items != null ? List.copyOf(items) : Collections.emptyList(); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java new file mode 100644 index 0000000000..e912ef9434 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java @@ -0,0 +1,79 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +// Written against DistributedMap* shapes not yet in the generated Lambda client, so this does not compile until they ship. + +import java.util.function.Function; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DistributedMapException; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.serde.SerDes; + +/** + * Durable operation that starts a distributed map run and waits for its terminal outcome. + * + * @param the resolved result type (a summary, or a result with collected items) + */ +public class DistributedMapOperation extends SerializableDurableOperation { + private final DistributedMapOptions options; + private final Function resultBuilder; + + public DistributedMapOperation( + OperationIdentifier operationIdentifier, + DistributedMapOptions options, + TypeToken resultType, + SerDes serDes, + Function resultBuilder, + DurableContextImpl durableContext) { + super(operationIdentifier, resultType, serDes, durableContext); + this.options = options; + this.resultBuilder = resultBuilder; + } + + /** Starts the operation. */ + @Override + protected void start() { + sendOperationUpdate(OperationUpdate.builder().action(OperationAction.START).distributedMapOptions(options)); + pollForOperationUpdates(); + } + + /** Replays the operation. */ + @Override + protected void replay(Operation existing) { + switch (existing.status()) { + case STARTED -> pollForOperationUpdates(); + case SUCCEEDED, FAILED, TIMED_OUT, STOPPED -> markAlreadyCompleted(); + default -> + throw terminateExecutionWithIllegalDurableOperationException( + "Unexpected distributed map status: " + existing.statusAsString()); + } + } + + /** Blocks until the operation completes and returns the resolved outcome. */ + @Override + public R get() { + var op = waitForOperationCompletion(); + return switch (op.status()) { + case SUCCEEDED -> { + var details = op.distributedMapDetails(); + if (details == null) { + throw terminateExecutionWithIllegalDurableOperationException( + "DISTRIBUTED_MAP operation succeeded but carried no DistributedMapDetails"); + } + yield resultBuilder.apply(details); + } + // Reaching here means the operation itself terminal-failed, so raise instead of hanging. + case FAILED, TIMED_OUT, STOPPED -> throw new DistributedMapException(op); + default -> + throw terminateExecutionWithIllegalDurableOperationException( + "Unexpected distributed map operation status: " + op.statusAsString()); + }; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java new file mode 100644 index 0000000000..efcd1ff9d3 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java @@ -0,0 +1,318 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +// Written against DistributedMap* shapes not yet in the generated Lambda client, so this does not compile until they ship. + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvDelimiter; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvHeaderLocation; +import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationInclude; +import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationType; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapFunctionResponseType; +import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapProcessorConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionMode; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3DestinationConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceTransform; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CsvFormat; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.config.FailureDestination; +import software.amazon.lambda.durable.config.ProcessorRetryConfig; +import software.amazon.lambda.durable.config.SuccessDestination; +import software.amazon.lambda.durable.exception.DurableExecutionException; +import software.amazon.lambda.durable.model.DistributedMapCompletionReason; +import software.amazon.lambda.durable.model.DistributedMapItemError; +import software.amazon.lambda.durable.model.DistributedMapResultItem; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Translates distributed map config to the checkpoint options and the operation details to result types. */ +public final class DistributedMapWire { + private static final SerDes JSON = new JacksonSerDes(); + private static final long INLINE_SIZE_LIMIT = 1024L * 1024L; + private static final long READER_STATE_LIMIT = 32L * 1024L; + + private DistributedMapWire() {} + + public static DistributedMapOptions toOptions( + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config, + SerDes defaultSerDes, + boolean collectResults) { + var builder = DistributedMapOptions.builder() + .maxConcurrency(maxConcurrency) + .source(sourceConfig(source, defaultSerDes)) + .processor(processorConfig(processor)); + if (config.destination() != null) { + var destination = destinationConfig(config.destination()); + if (destination != null) { + builder.destination(destination); + } + } + var completion = completionConfig(config); + if (completion != null) { + builder.completionConfig(completion); + } + if (collectResults) { + builder.resultCollection(DistributedMapResultCollectionConfig.builder() + .mode(DistributedMapResultCollectionMode.INLINE) + .build()); + } + if (config.timeout() != null) { + builder.timeoutSeconds((int) config.timeout().toSeconds()); + } + return builder.build(); + } + + private static DistributedMapSourceConfig sourceConfig(DistributedMapSource source, SerDes defaultSerDes) { + var builder = DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.fromValue(source.sourceType().name())); + if (source.maxItems() != null) { + builder.maxItemsToRead(source.maxItems().longValue()); + } + switch (source.sourceType()) { + case INLINE -> builder.inlineSourceConfig(inlineSourceConfig(source, defaultSerDes)); + case S3 -> builder.s3SourceConfig(s3SourceConfig(source.s3())); + case READER_FUNCTION -> builder.readerFunctionSourceConfig(readerConfig(source.reader(), defaultSerDes)); + } + return builder.build(); + } + + private static DistributedMapInlineSourceConfig inlineSourceConfig( + DistributedMapSource source, SerDes defaultSerDes) { + var serdes = source.inlineSerDes() != null ? source.inlineSerDes() : defaultSerDes; + var bodies = new ArrayList(source.inlineItems().size()); + for (Object item : source.inlineItems()) { + bodies.add(serdes.serialize(item)); + } + var arrayBytes = JSON.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; + if (arrayBytes > INLINE_SIZE_LIMIT) { + throw new IllegalArgumentException( + "inline source exceeds the 1 MB limit (serialized size: " + arrayBytes + " bytes)"); + } + return DistributedMapInlineSourceConfig.builder().items(bodies).build(); + } + + private static DistributedMapS3SourceConfig s3SourceConfig(DistributedMapSource.S3SourceConfig s3) { + var builder = DistributedMapS3SourceConfig.builder().bucket(s3.bucket()); + if (s3.key() != null) { + builder.key(s3.key()); + } + if (s3.prefix() != null) { + builder.keyPrefix(s3.prefix()); + } + if (s3.transform() != null) { + builder.transform(DistributedMapS3SourceTransform.fromValue(s3.transform().name())); + } + if (s3.format() != null) { + builder.format(DistributedMapSourceFormat.fromValue(s3.format().name())); + } + if (s3.expectedBucketOwner() != null) { + builder.expectedBucketOwner(s3.expectedBucketOwner()); + } + if (s3.csvFormat() != null) { + builder.csvFormatOptions(csvFormatOptions(s3.csvFormat())); + } + return builder.build(); + } + + private static DistributedMapCsvFormatOptions csvFormatOptions(CsvFormat format) { + var builder = DistributedMapCsvFormatOptions.builder() + .headerLocation(DistributedMapCsvHeaderLocation.fromValue(format.headerLocation().name())) + .delimiter(DistributedMapCsvDelimiter.fromValue(format.delimiter().getValue())); + // Column names go on the wire only for headerless files (GIVEN), expectedColumns (FIRST_ROW) is compile-time only. + if (format.headerLocation() == CsvFormat.HeaderLocation.GIVEN && format.columns() != null) { + builder.headers(format.columns()); + } + return builder.build(); + } + + private static DistributedMapReaderFunctionSourceConfig readerConfig( + DistributedMapSource.ReaderSourceConfig reader, SerDes defaultSerDes) { + var builder = DistributedMapReaderFunctionSourceConfig.builder().functionName(reader.functionName()); + if (reader.initialState() != null) { + var serdes = reader.stateSerDes() != null ? reader.stateSerDes() : defaultSerDes; + var state = serdes.serialize(reader.initialState()); + if (state != null && state.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalArgumentException("reader initialState exceeds the 32 KB limit"); + } + builder.initialState(state); + } + return builder.build(); + } + + private static DistributedMapProcessorConfig processorConfig(DistributedMapProcessor processor) { + var builder = DistributedMapProcessorConfig.builder().functionName(processor.functionName()); + if (processor.durableExecutionNamePrefix() != null) { + builder.durableExecutionNamePrefix(processor.durableExecutionNamePrefix()); + } + var responseType = processor.responseMode().getValue(); + if (responseType != null) { + builder.functionResponseTypes(List.of(DistributedMapFunctionResponseType.fromValue(responseType))); + } + if (processor.batchSize() != null) { + builder.batchSize(processor.batchSize()); + } + ProcessorRetryConfig retry = processor.retryConfig(); + if (retry != null) { + if (retry.maxRetryAttempts() != null) { + builder.maxRetryAttempts(retry.maxRetryAttempts()); + } + if (retry.maxRetryDuration() != null) { + builder.maxRetryDurationSeconds((int) retry.maxRetryDuration().toSeconds()); + } + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig destinationConfig( + software.amazon.lambda.durable.config.DistributedMapDestinationConfig destination) { + if (destination.onSuccess() == null && destination.onFailure() == null) { + return null; // omit an empty destination config from the wire + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig.builder(); + SuccessDestination onSuccess = destination.onSuccess(); + if (onSuccess != null) { + var include = new ArrayList(); + if (onSuccess.includeInput()) { + include.add(DistributedMapDestinationInclude.INPUT); + } + if (onSuccess.includeOutput()) { + include.add(DistributedMapDestinationInclude.OUTPUT); + } + builder.onSuccess(DistributedMapOnSuccessConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig(s3Destination( + onSuccess.bucket(), onSuccess.prefix(), onSuccess.expectedBucketOwner())) + .build()); + } + FailureDestination onFailure = destination.onFailure(); + if (onFailure != null) { + var include = new ArrayList(); + if (onFailure.includeInput()) { + include.add(DistributedMapDestinationInclude.INPUT); + } + if (onFailure.includeError()) { + include.add(DistributedMapDestinationInclude.ERROR); + } + builder.onFailure(DistributedMapOnFailureConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig(s3Destination( + onFailure.bucket(), onFailure.prefix(), onFailure.expectedBucketOwner())) + .build()); + } + return builder.build(); + } + + private static DistributedMapS3DestinationConfig s3Destination( + String bucket, String keyPrefix, String expectedBucketOwner) { + var builder = DistributedMapS3DestinationConfig.builder().bucket(bucket).keyPrefix(keyPrefix); + if (expectedBucketOwner != null) { + builder.expectedBucketOwner(expectedBucketOwner); + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig completionConfig( + DistributedMapConfig config) { + var completion = config.completionConfig(); + if (completion == null + || (completion.toleratedFailureCount() == null + && completion.toleratedFailurePercentage() == null + && completion.minimumSampleSize() == null)) { + return null; // omit an empty completion config from the wire + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig.builder(); + if (completion.toleratedFailureCount() != null) { + builder.toleratedFailureCount(completion.toleratedFailureCount()); + } + if (completion.toleratedFailurePercentage() != null) { + builder.toleratedFailurePercentage(completion.toleratedFailurePercentage().floatValue()); + } + if (completion.minimumSampleSize() != null) { + builder.minimumSampleSize(completion.minimumSampleSize()); + } + return builder.build(); + } + + public static DistributedMapSummary toSummary(DistributedMapDetails details) { + return new DistributedMapSummary( + parseStatus(details), + parseReason(details), + details.successCount(), + details.failureCount(), + details.unprocessedCount(), + details.distributedMapRunArn(), + details.completionDetails(), + details.totalCount()); + } + + public static DistributedMapResult toResult( + DistributedMapDetails details, TypeToken resultType, SerDes serdes) { + var summary = toSummary(details); + var items = new ArrayList>(); + if (details.results() != null) { + for (var wire : details.results()) { + var status = wire.statusAsString(); + if (DistributedMapResultItem.Status.SUCCEEDED.name().equals(status)) { + O output = wire.output() != null ? serdes.deserialize(wire.output(), resultType) : null; + items.add(DistributedMapResultItem.succeeded(wire.itemId(), output)); + } else if (DistributedMapResultItem.Status.FAILED.name().equals(status)) { + var error = wire.error() != null + ? new DistributedMapItemError( + wire.error().errorType() != null ? wire.error().errorType() : "", + wire.error().errorMessage() != null ? wire.error().errorMessage() : "") + : null; + items.add(DistributedMapResultItem.failed(wire.itemId(), error)); + } else { + throw new DurableExecutionException("unrecognized distributed map item status from the backend (" + + status + ") for item " + wire.itemId()); + } + } + } + return new DistributedMapResult<>(summary, items); + } + + private static DistributedMapStatus parseStatus(DistributedMapDetails details) { + if (details.statusAsString() == null) { + throw new DurableExecutionException("distributed map details missing the required Status field"); + } + try { + return DistributedMapStatus.fromValue(details.statusAsString()); + } catch (IllegalArgumentException e) { + throw new DurableExecutionException( + "unrecognized distributed map status from the backend (" + details.statusAsString() + ")", e); + } + } + + private static DistributedMapCompletionReason parseReason(DistributedMapDetails details) { + if (details.completionReasonAsString() == null) { + throw new DurableExecutionException("distributed map details missing the required CompletionReason field"); + } + return DistributedMapCompletionReason.fromValue(details.completionReasonAsString()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java new file mode 100644 index 0000000000..f5dbba1124 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java @@ -0,0 +1,62 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.util; + +import java.util.HashSet; +import java.util.List; +import java.util.regex.Pattern; + +/** Shared validation and S3 URI parsing for distributed map configuration. */ +public final class DistributedMapValidation { + private static final Pattern ACCOUNT_ID = Pattern.compile("\\d{12}"); + private static final String S3_SCHEME = "s3://"; + private static final int MAX_FUNCTION_NAME_LENGTH = 170; + + private DistributedMapValidation() {} + + public static void validateFunctionName(String functionName) { + if (functionName == null || functionName.isBlank()) { + throw new IllegalArgumentException("function name cannot be empty"); + } + if (functionName.length() > MAX_FUNCTION_NAME_LENGTH) { + throw new IllegalArgumentException( + "function name must be at most 170 characters, got: " + functionName.length()); + } + } + + public static void validateBucketOwner(String expectedBucketOwner) { + if (expectedBucketOwner != null + && !ACCOUNT_ID.matcher(expectedBucketOwner).matches()) { + throw new IllegalArgumentException( + "expectedBucketOwner must be a 12-digit account id, got: " + expectedBucketOwner); + } + } + + public static void validateColumns(String parameterName, List columns) { + if (columns == null) { + return; + } + if (columns.isEmpty()) { + throw new IllegalArgumentException(parameterName + " cannot be empty"); + } + if (new HashSet<>(columns).size() != columns.size()) { + throw new IllegalArgumentException(parameterName + " must not contain duplicates"); + } + } + + public static ParsedS3Uri parseS3Uri(String uri) { + if (uri == null || !uri.startsWith(S3_SCHEME)) { + throw new IllegalArgumentException("S3 URI must start with s3://, got: " + uri); + } + var remainder = uri.substring(S3_SCHEME.length()); + var slash = remainder.indexOf('/'); + var bucket = slash < 0 ? remainder : remainder.substring(0, slash); + var path = slash < 0 ? null : remainder.substring(slash + 1); + if (bucket.isEmpty()) { + throw new IllegalArgumentException("S3 URI must contain a bucket, got: " + uri); + } + return new ParsedS3Uri(bucket, path == null || path.isEmpty() ? null : path); + } + + public record ParsedS3Uri(String bucket, String path) {} +} From 369fe1d3fe8e6308f460075e3f39fb2dc8a180a1 Mon Sep 17 00:00:00 2001 From: nvasiu Date: Fri, 25 Sep 2026 18:16:04 +0000 Subject: [PATCH 2/2] refactor(sdk): Align distributed map with spec Addresses review feedback on the unreleased distributed map feature. Fixes: - The durable item handler checkpointed item results with the default serdes, so a custom resultSerDes produced different JSON after a suspension. Results are now serialized before being checkpointed. - Output serialization ran after the per-item futures settled, so one unserializable result failed the whole batch. It now runs inside each submitted task, so the failure is reported against its own item. - Non-positive concurrency fell back to one thread per record. It is now rejected, and a new overload defaults to one. - getResults filtered out null outputs, so the list no longer lined up with succeeded(). It now preserves them, matching MapResult. - An item handler that returns nothing reports no output. - A reported item failure carries the error's stack trace. - A replay that finds the operation PENDING waits for the outcome. - An absent terminal count reads as zero rather than throwing. - maxItems is bounded at 100,000,000, and a CSV column list at 1,000 names of at most 256 characters. - The maxConcurrency upper bound is removed, leaving the service to enforce the range. Changes: - Authoring helpers move into a dmap package, keeping feature specific classes off the root package. - Handler signatures use concrete records instead of Map and Object. The seven envelope types are public, and envelope validation moves into their compact constructors. - DistributedMapWire folds into DistributedMapOperation, and ProcessorRetryConfig folds onto DistributedMapProcessor. - DistributedMapError becomes DistributedMapException, matching the other 21 exception types. - Types owned by one aggregate are nested into it, removing 8 top level names. - Adds DISTRIBUTED_MAP to the exhaustive switch in ChildContextOperation. - Processor response mode is BATCH or ITEM_OUTCOMES, sending REPORT_BATCH_ITEM_OUTCOMES. The itemFailures and itemResults factories become itemOutcomes. - ItemHandlerResponse drops its two factories. - The S3 source carries a type with one matching nested config block. Transform is deleted. The seven public factory names are unchanged. - sourceExhausted and error replace completionReason and completionDetails. DistributedMapCompletionReason is deleted. - New DistributedMapError record with the four error members, shared by the summary and each item. It replaces DistributedMapResult.ItemError. - Destination include values come from two enums, one per outcome. - toleratedFailurePercentage is an integer. --- .../durable/DistributedMapHandlers.java | 265 ------ .../amazon/lambda/durable/DurableContext.java | 102 ++- .../lambda/durable/config/CsvDelimiter.java | 23 - .../lambda/durable/config/CsvFormat.java | 45 - .../DistributedMapCompletionConfig.java | 45 - .../durable/config/DistributedMapConfig.java | 63 +- .../config/DistributedMapDestination.java | 68 +- .../DistributedMapDestinationConfig.java | 22 - .../config/DistributedMapProcessor.java | 53 +- ...> DistributedMapProcessorRetryConfig.java} | 49 +- .../durable/config/DistributedMapSource.java | 145 ++- .../durable/config/FailureDestination.java | 17 - .../durable/config/SuccessDestination.java | 17 - .../durable/context/DurableContextImpl.java | 19 +- .../durable/dmap/DistributedMapHandlers.java | 320 +++++++ .../lambda/durable/dmap/ItemFailure.java | 37 + .../durable/dmap/ItemHandlerResponse.java | 17 + .../lambda/durable/dmap/ItemResult.java | 12 + .../lambda/durable/dmap/ProcessorEvent.java | 25 + .../lambda/durable/dmap/ProcessorRecord.java | 22 + .../lambda/durable/dmap/ReaderEvent.java | 22 + .../durable/{model => dmap}/ReaderPage.java | 6 +- .../lambda/durable/dmap/ReaderResponse.java | 16 + .../exception/DistributedMapError.java | 93 -- .../exception/DistributedMapException.java | 77 +- .../durable/execution/CheckpointManager.java | 40 + .../durable/execution/DurableExecutor.java | 12 +- .../model/DistributedMapCompletionReason.java | 47 - .../durable/model/DistributedMapError.java | 21 + .../model/DistributedMapItemError.java | 12 - .../durable/model/DistributedMapResult.java | 89 +- .../model/DistributedMapResultItem.java | 23 - .../durable/model/DistributedMapStatus.java | 34 +- .../durable/model/DistributedMapSummary.java | 24 +- .../durable/model/OperationSubType.java | 1 - .../operation/BaseDurableOperation.java | 4 + .../operation/ChildContextOperation.java | 2 +- .../operation/DistributedMapOperation.java | 399 +++++++- .../durable/operation/DistributedMapWire.java | 318 ------- .../util/DistributedMapValidation.java | 31 + .../config/DistributedMapConfigTest.java | 109 +++ .../config/DistributedMapDestinationTest.java | 216 +++++ ...istributedMapProcessorRetryConfigTest.java | 77 ++ .../config/DistributedMapProcessorTest.java | 176 ++++ .../config/DistributedMapSourceTest.java | 415 +++++++++ .../DurableContextDistributedMapTest.java | 194 ++++ .../DistributedMapDurableHandlersTest.java | 140 +++ .../dmap/DistributedMapHandlersTest.java | 460 ++++++++++ .../execution/CheckpointManagerTest.java | 149 +++ .../model/DistributedMapResultTest.java | 191 ++++ .../model/DistributedMapSummaryTest.java | 69 ++ .../DistributedMapOperationTest.java | 333 +++++++ ...istributedMapOperationTranslationTest.java | 852 ++++++++++++++++++ 53 files changed, 4864 insertions(+), 1154 deletions(-) delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java rename sdk/src/main/java/software/amazon/lambda/durable/config/{ProcessorRetryConfig.java => DistributedMapProcessorRetryConfig.java} (60%) delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java rename sdk/src/main/java/software/amazon/lambda/durable/{model => dmap}/ReaderPage.java (57%) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java b/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java deleted file mode 100644 index 28f8ccca41..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/DistributedMapHandlers.java +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable; - -import com.amazonaws.services.lambda.runtime.RequestHandler; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.Function; -import software.amazon.lambda.durable.config.CompletionConfig; -import software.amazon.lambda.durable.config.DistributedMapProcessor.ResponseMode; -import software.amazon.lambda.durable.config.MapConfig; -import software.amazon.lambda.durable.execution.DurableExecutor; -import software.amazon.lambda.durable.model.DurableExecutionInput; -import software.amazon.lambda.durable.model.DurableExecutionOutput; -import software.amazon.lambda.durable.model.MapResult; -import software.amazon.lambda.durable.model.ReaderPage; -import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.SerDes; - -/** Authoring helpers that wrap a customer function into a distributed map processor or reader Lambda handler. */ -public final class DistributedMapHandlers { - private static final SerDes JSON = new JacksonSerDes(); - private static final TypeToken OBJECT_TYPE = TypeToken.get(Object.class); - private static final TypeToken> EVENT_TYPE = new TypeToken>() {}; - private static final int READER_STATE_LIMIT = 32 * 1024; - private static final String ITEMS_OP_NAME = "distributed-map-items"; - - private DistributedMapHandlers() {} - - private static void validateItemReport(ResponseMode report) { - if (report != ResponseMode.REPORT_ITEM_RESULTS && report != ResponseMode.REPORT_FAILED_ITEMS) { - throw new IllegalArgumentException( - "item handler report mode must be REPORT_ITEM_RESULTS or REPORT_FAILED_ITEMS, got: " + report); - } - } - - /** Wraps a per-item function as a processor handler. Use REPORT_ITEM_RESULTS or REPORT_FAILED_ITEMS. */ - public static RequestHandler, Map> createDistributedMapItemHandler( - Function func, - TypeToken itemType, - SerDes itemSerDes, - SerDes resultSerDes, - ResponseMode report, - int concurrency) { - validateItemReport(report); - var inSerdes = itemSerDes != null ? itemSerDes : JSON; - var outSerdes = resultSerDes != null ? resultSerDes : JSON; - return (event, context) -> { - var records = records(event); - var workers = concurrency > 0 ? concurrency : Math.max(1, records.size()); - var outputs = new Object[records.size()]; - var errors = new Throwable[records.size()]; - var pool = Executors.newFixedThreadPool(workers); - try { - var futures = new ArrayList>(); - for (var i = 0; i < records.size(); i++) { - var index = i; - futures.add(pool.submit(() -> { - I item = toItem(inSerdes, records.get(index).get("body"), itemType); - outputs[index] = func.apply(item); - return null; - })); - } - for (var i = 0; i < futures.size(); i++) { - try { - futures.get(i).get(); - } catch (ExecutionException e) { - errors[i] = e.getCause() != null ? e.getCause() : e; - } - } - } catch (InterruptedException e) { - pool.shutdownNow(); - Thread.currentThread().interrupt(); - throw new IllegalStateException("Distributed map item handler interrupted", e); - } finally { - pool.shutdown(); - } - - var results = new ArrayList>(); - var failures = new ArrayList>(); - for (var i = 0; i < records.size(); i++) { - var itemId = (String) records.get(i).get("itemId"); - if (errors[i] != null) { - failures.add(errorEntry(itemId, errors[i].getClass().getName(), errors[i].getMessage())); - } else if (report == ResponseMode.REPORT_ITEM_RESULTS) { - results.add(resultEntry(itemId, toJsonValue(outSerdes, outputs[i]))); - } - } - return itemResponse(report, results, failures); - }; - } - - /** Wraps a whole-batch function as a processor handler. Returning succeeds every item, throwing fails the batch. */ - public static RequestHandler, Object> createDistributedMapBatchHandler( - Consumer> func, TypeToken itemType, SerDes itemSerDes) { - var serdes = itemSerDes != null ? itemSerDes : JSON; - return (event, context) -> { - func.accept(toItems(serdes, event, itemType)); - return null; - }; - } - - /** Wraps a reader function as a source handler. A null nextState signals the source is exhausted. */ - public static RequestHandler, Map> createDistributedMapReader( - Function> func, TypeToken stateType, SerDes stateSerDes) { - var serdes = stateSerDes != null ? stateSerDes : JSON; - return (event, context) -> { - if (!(event.get("maxItems") instanceof Number maxItemsValue)) { - throw new IllegalStateException("expected a distributed map reader event carrying a numeric 'maxItems' (got keys: " - + event.keySet() + "), the function must be registered as a distributed map reader"); - } - var rawState = (String) event.get("state"); - S state = rawState != null ? serdes.deserialize(rawState, stateType) : null; - var maxItems = maxItemsValue.intValue(); - - var page = func.apply(state); - if (page.items().size() > maxItems) { - throw new IllegalStateException( - "reader returned " + page.items().size() + " items, exceeding maxItems " + maxItems); - } - var response = new LinkedHashMap(); - response.put("items", page.items()); - if (page.nextState() != null) { - var nextState = serdes.serialize(page.nextState()); - if (nextState.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { - throw new IllegalStateException( - "reader nextState exceeds the " + (READER_STATE_LIMIT / 1024) + " KB limit"); - } - response.put("nextState", nextState); - } - return response; - }; - } - - /** Durable variant of the item handler. The function receives the per-item DurableContext. */ - public static - RequestHandler - createDistributedMapItemHandlerWithDurableExecution( - BiFunction func, - TypeToken itemType, - SerDes itemSerDes, - SerDes resultSerDes, - ResponseMode report) { - validateItemReport(report); - var inSerdes = itemSerDes != null ? itemSerDes : JSON; - var outSerdes = resultSerDes != null ? resultSerDes : JSON; - return DurableExecutor.wrap( - EVENT_TYPE, - (event, ctx) -> { - var records = records(event); - var bodies = new ArrayList(records.size()); - for (var record : records) { - bodies.add(record.get("body")); - } - MapResult batch = ctx.map( - ITEMS_OP_NAME, - bodies, - OBJECT_TYPE, - (body, index, mapContext) -> - (Object) func.apply(mapContext, toItem(inSerdes, body, itemType)), - MapConfig.builder() - .completionConfig(CompletionConfig.allCompleted()) - .build()); - - var results = new ArrayList>(); - var failures = new ArrayList>(); - for (var i = 0; i < records.size(); i++) { - var itemId = (String) records.get(i).get("itemId"); - var item = batch.getItem(i); - if (item.status() == MapResult.MapResultItem.Status.SUCCEEDED) { - if (report == ResponseMode.REPORT_ITEM_RESULTS) { - results.add(resultEntry(itemId, toJsonValue(outSerdes, item.result()))); - } - } else { - var error = item.error(); - failures.add(errorEntry( - itemId, - error != null ? error.errorType() : "", - error != null ? error.errorMessage() : "")); - } - } - return itemResponse(report, results, failures); - }, - DurableConfig.defaultConfig()); - } - - /** Durable variant of the batch handler. The function receives the DurableContext. */ - public static - RequestHandler - createDistributedMapBatchHandlerWithDurableExecution( - BiConsumer> func, TypeToken itemType, SerDes itemSerDes) { - var serdes = itemSerDes != null ? itemSerDes : JSON; - return DurableExecutor.wrap( - EVENT_TYPE, - (event, ctx) -> { - func.accept(ctx, toItems(serdes, event, itemType)); - return null; - }, - DurableConfig.defaultConfig()); - } - - @SuppressWarnings("unchecked") - private static List> records(Map event) { - if (!(event.get("records") instanceof List)) { - throw new IllegalStateException("expected a distributed map processor event carrying a 'records' array (got keys: " - + event.keySet() + "), the function must be triggered by a distributed map, not invoked directly"); - } - return (List>) event.get("records"); - } - - private static List toItems(SerDes serdes, Map event, TypeToken itemType) { - var records = records(event); - var items = new ArrayList(records.size()); - for (var record : records) { - items.add(toItem(serdes, record.get("body"), itemType)); - } - return items; - } - - private static I toItem(SerDes serdes, Object body, TypeToken itemType) { - return serdes.deserialize(JSON.serialize(body), itemType); - } - - private static Object toJsonValue(SerDes serdes, Object value) { - return JSON.deserialize(serdes.serialize(value), OBJECT_TYPE); - } - - private static Map resultEntry(String itemId, Object output) { - var entry = new LinkedHashMap(); - entry.put("itemIdentifier", itemId); - entry.put("output", output); - return entry; - } - - private static Map errorEntry(String itemId, String errorType, String errorMessage) { - var error = new LinkedHashMap(); - error.put("errorType", errorType); - error.put("errorMessage", errorMessage); - var entry = new LinkedHashMap(); - entry.put("itemIdentifier", itemId); - entry.put("error", error); - return entry; - } - - private static Map itemResponse( - ResponseMode report, List> results, List> failures) { - var response = new LinkedHashMap(); - if (report == ResponseMode.REPORT_FAILED_ITEMS) { - response.put("batchItemFailures", failures); - } else { - response.put("batchItemResults", results); - response.put("batchItemFailures", failures); - } - return response; - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java index 79a00cdc8d..3b0549ddf8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableContext.java @@ -550,15 +550,27 @@ default ParallelDurableFuture parallel(String name) { // =============== distributedMap ================ - /** Runs a distributed map with default config, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map with default config, blocking until the run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapSummary distributedMap( String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { return distributedMapAsync( - name, source, processor, maxConcurrency, DistributedMapConfig.builder().build()) + name, + source, + processor, + maxConcurrency, + DistributedMapConfig.builder().build()) .get(); } - /** Runs a distributed map, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map, blocking until the run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapSummary distributedMap( String name, DistributedMapSource source, @@ -569,7 +581,12 @@ default DistributedMapSummary distributedMap( .get(); } - /** Runs a distributed map that collects per-item results with a TypeToken, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map that collects per-item results with a TypeToken, blocking until the run reaches a terminal + * state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapResult distributedMap( String name, DistributedMapSource source, @@ -581,7 +598,12 @@ default DistributedMapResult distributedMap( .get(); } - /** Runs a distributed map that collects per-item results with a result Class, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map that collects per-item results with a result Class, blocking until the run reaches a + * terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapResult distributedMap( String name, DistributedMapSource source, @@ -593,7 +615,12 @@ default DistributedMapResult distributedMap( .get(); } - /** Runs a distributed map that collects per-item results with a result Class and default config, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map that collects per-item results with a result Class and default config, blocking until the + * run reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapResult distributedMap( String name, DistributedMapSource source, @@ -610,7 +637,12 @@ default DistributedMapResult distributedMap( .get(); } - /** Runs a distributed map that collects per-item results with a TypeToken and default config, blocking until the run reaches a terminal state. */ + /** + * Runs a distributed map that collects per-item results with a TypeToken and default config, blocking until the run + * reaches a terminal state. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DistributedMapResult distributedMap( String name, DistributedMapSource source, @@ -618,18 +650,35 @@ default DistributedMapResult distributedMap( int maxConcurrency, TypeToken resultType) { return distributedMapAsync( - name, source, processor, maxConcurrency, resultType, DistributedMapConfig.builder().build()) + name, + source, + processor, + maxConcurrency, + resultType, + DistributedMapConfig.builder().build()) .get(); } - /** Asynchronously runs a distributed map with default config. */ + /** + * Asynchronously runs a distributed map with default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DurableFuture distributedMapAsync( String name, DistributedMapSource source, DistributedMapProcessor processor, int maxConcurrency) { return distributedMapAsync( - name, source, processor, maxConcurrency, DistributedMapConfig.builder().build()); + name, + source, + processor, + maxConcurrency, + DistributedMapConfig.builder().build()); } - /** Asynchronously runs a distributed map. */ + /** + * Asynchronously runs a distributed map. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ DurableFuture distributedMapAsync( String name, DistributedMapSource source, @@ -637,7 +686,11 @@ DurableFuture distributedMapAsync( int maxConcurrency, DistributedMapConfig config); - /** Asynchronously runs a distributed map that collects per-item results with a TypeToken. */ + /** + * Asynchronously runs a distributed map that collects per-item results with a TypeToken. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ DurableFuture> distributedMapAsync( String name, DistributedMapSource source, @@ -646,7 +699,11 @@ DurableFuture> distributedMapAsync( TypeToken resultType, DistributedMapConfig config); - /** Asynchronously runs a distributed map that collects per-item results with a result Class. */ + /** + * Asynchronously runs a distributed map that collects per-item results with a result Class. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DurableFuture> distributedMapAsync( String name, DistributedMapSource source, @@ -657,7 +714,11 @@ default DurableFuture> distributedMapAsync( return distributedMapAsync(name, source, processor, maxConcurrency, TypeToken.get(resultType), config); } - /** Asynchronously runs a distributed map that collects per-item results with a result Class and default config. */ + /** + * Asynchronously runs a distributed map that collects per-item results with a result Class and default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DurableFuture> distributedMapAsync( String name, DistributedMapSource source, @@ -673,7 +734,11 @@ default DurableFuture> distributedMapAsync( DistributedMapConfig.builder().build()); } - /** Asynchronously runs a distributed map that collects per-item results with a TypeToken and default config. */ + /** + * Asynchronously runs a distributed map that collects per-item results with a TypeToken and default config. + * + * @param maxConcurrency the maximum number of items processed concurrently. Required, with no service default. + */ default DurableFuture> distributedMapAsync( String name, DistributedMapSource source, @@ -681,7 +746,12 @@ default DurableFuture> distributedMapAsync( int maxConcurrency, TypeToken resultType) { return distributedMapAsync( - name, source, processor, maxConcurrency, resultType, DistributedMapConfig.builder().build()); + name, + source, + processor, + maxConcurrency, + resultType, + DistributedMapConfig.builder().build()); } /** diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java deleted file mode 100644 index df6d4a3554..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvDelimiter.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -/** Delimiter for CSV distributed map sources. */ -public enum CsvDelimiter { - COMMA("COMMA"), - PIPE("PIPE"), - SEMICOLON("SEMICOLON"), - SPACE("SPACE"), - TAB("TAB"); - - private final String value; - - CsvDelimiter(String value) { - this.value = value; - } - - /** Returns the wire-format string value. */ - public String getValue() { - return value; - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java b/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java deleted file mode 100644 index 61d9e2cd34..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/CsvFormat.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -import java.util.List; -import software.amazon.lambda.durable.util.DistributedMapValidation; - -/** CSV parsing options for a distributed map S3 source. */ -public record CsvFormat(HeaderLocation headerLocation, List columns, CsvDelimiter delimiter) { - - /** Where the column header row comes from. */ - public enum HeaderLocation { - FIRST_ROW, - GIVEN - } - - public CsvFormat { - columns = columns != null ? List.copyOf(columns) : null; - delimiter = delimiter != null ? delimiter : CsvDelimiter.COMMA; - } - - /** - * File has its own header row. Column names are used only for compile-time typing and are not sent on the wire. - */ - public static CsvFormat expectedColumns(List columns) { - DistributedMapValidation.validateColumns("expectedColumns", columns); - return new CsvFormat(HeaderLocation.FIRST_ROW, columns, CsvDelimiter.COMMA); - } - - /** File has no header row. The given names are the columns, sent on the wire, and the first row is data. */ - public static CsvFormat headers(List headers) { - DistributedMapValidation.validateColumns("headers", headers); - return new CsvFormat(HeaderLocation.GIVEN, headers, CsvDelimiter.COMMA); - } - - /** File has its own header row and no column names are declared. */ - public static CsvFormat firstRow() { - return new CsvFormat(HeaderLocation.FIRST_ROW, null, CsvDelimiter.COMMA); - } - - /** Returns a copy with the given delimiter. */ - public CsvFormat withDelimiter(CsvDelimiter delimiter) { - return new CsvFormat(headerLocation, columns, delimiter); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java deleted file mode 100644 index ee5bd864bd..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapCompletionConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -/** Failure-tolerance configuration for a distributed map run. */ -public record DistributedMapCompletionConfig( - Integer toleratedFailureCount, Double toleratedFailurePercentage, Integer minimumSampleSize) { - - public DistributedMapCompletionConfig { - if (toleratedFailureCount != null && toleratedFailurePercentage != null) { - throw new IllegalArgumentException( - "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive"); - } - if (minimumSampleSize != null && toleratedFailurePercentage == null) { - throw new IllegalArgumentException("minimumSampleSize is only valid with toleratedFailurePercentage"); - } - if (toleratedFailureCount != null && toleratedFailureCount < 0) { - throw new IllegalArgumentException( - "toleratedFailureCount must be non-negative, got: " + toleratedFailureCount); - } - if (toleratedFailurePercentage != null - && (toleratedFailurePercentage < 0 || toleratedFailurePercentage > 100)) { - throw new IllegalArgumentException( - "toleratedFailurePercentage must be between 0 and 100, got: " + toleratedFailurePercentage); - } - if (minimumSampleSize != null && minimumSampleSize < 1) { - throw new IllegalArgumentException("minimumSampleSize must be at least 1, got: " + minimumSampleSize); - } - } - - /** Abort once this many items have permanently failed. */ - public static DistributedMapCompletionConfig failureCount(int count) { - return new DistributedMapCompletionConfig(count, null, null); - } - - /** Abort once the failure rate exceeds this percentage (0 to 100). */ - public static DistributedMapCompletionConfig failurePercentage(double percentage) { - return new DistributedMapCompletionConfig(null, percentage, null); - } - - /** Abort once the failure rate exceeds this percentage (0 to 100), gated by a minimum sample size. */ - public static DistributedMapCompletionConfig failurePercentage(double percentage, int minimumSampleSize) { - return new DistributedMapCompletionConfig(null, percentage, minimumSampleSize); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java index 0ea953c1e2..736a636e44 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapConfig.java @@ -9,8 +9,8 @@ public class DistributedMapConfig { private static final long MAX_TIMEOUT_SECONDS = 7776000; // 90 days - private final DistributedMapDestinationConfig destination; - private final DistributedMapCompletionConfig completionConfig; + private final DistributedMapDestination destination; + private final CompletionConfig completionConfig; private final Duration timeout; private final SerDes resultSerDes; @@ -26,11 +26,11 @@ private DistributedMapConfig(Builder builder) { this.resultSerDes = builder.resultSerDes; } - public DistributedMapDestinationConfig destination() { + public DistributedMapDestination destination() { return destination; } - public DistributedMapCompletionConfig completionConfig() { + public CompletionConfig completionConfig() { return completionConfig; } @@ -57,19 +57,19 @@ public Builder toBuilder() { /** Builder for DistributedMapConfig. */ public static class Builder { - private DistributedMapDestinationConfig destination; - private DistributedMapCompletionConfig completionConfig; + private DistributedMapDestination destination; + private CompletionConfig completionConfig; private Duration timeout; private SerDes resultSerDes; private Builder() {} - public Builder destination(DistributedMapDestinationConfig destination) { + public Builder destination(DistributedMapDestination destination) { this.destination = destination; return this; } - public Builder completionConfig(DistributedMapCompletionConfig completionConfig) { + public Builder completionConfig(CompletionConfig completionConfig) { this.completionConfig = completionConfig; return this; } @@ -88,4 +88,51 @@ public DistributedMapConfig build() { return new DistributedMapConfig(this); } } + + /** Failure-tolerance configuration for a distributed map run. */ + public record CompletionConfig( + Integer toleratedFailureCount, Integer toleratedFailurePercentage, Integer minimumSampleSize) { + + public CompletionConfig { + if (toleratedFailureCount != null && toleratedFailurePercentage != null) { + throw new IllegalArgumentException( + "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive"); + } + if (minimumSampleSize != null && toleratedFailurePercentage == null) { + throw new IllegalArgumentException("minimumSampleSize is only valid with toleratedFailurePercentage"); + } + // Omitting the config means zero tolerated failures, so an empty one is rejected rather than omitted. + if (toleratedFailureCount == null && toleratedFailurePercentage == null) { + throw new IllegalArgumentException( + "a completion config must set toleratedFailureCount or toleratedFailurePercentage"); + } + if (toleratedFailureCount != null && toleratedFailureCount < 0) { + throw new IllegalArgumentException( + "toleratedFailureCount must be non-negative, got: " + toleratedFailureCount); + } + if (toleratedFailurePercentage != null + && (toleratedFailurePercentage < 0 || toleratedFailurePercentage > 100)) { + throw new IllegalArgumentException( + "toleratedFailurePercentage must be between 0 and 100, got: " + toleratedFailurePercentage); + } + if (minimumSampleSize != null && minimumSampleSize < 1) { + throw new IllegalArgumentException("minimumSampleSize must be at least 1, got: " + minimumSampleSize); + } + } + + /** Stops the run after more than this many items permanently fail. */ + public static CompletionConfig failureCount(int count) { + return new CompletionConfig(count, null, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100). */ + public static CompletionConfig failurePercentage(int percentage) { + return new CompletionConfig(null, percentage, null); + } + + /** Abort once the failure rate exceeds this percentage (0 to 100), gated by a minimum sample size. */ + public static CompletionConfig failurePercentage(int percentage, int minimumSampleSize) { + return new CompletionConfig(null, percentage, minimumSampleSize); + } + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java index 7a9dd1bb1d..400f777b62 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestination.java @@ -4,20 +4,34 @@ import software.amazon.lambda.durable.util.DistributedMapValidation; -/** Destination factories for a distributed map run. */ -public final class DistributedMapDestination { - private DistributedMapDestination() {} +/** Destination routing for distributed map run results. */ +public record DistributedMapDestination(Success onSuccess, Failure onFailure) { + + /** Routes only succeeded item records. */ + public static DistributedMapDestination of(Success onSuccess) { + return new DistributedMapDestination(onSuccess, null); + } + + /** Routes only failed item records. */ + public static DistributedMapDestination of(Failure onFailure) { + return new DistributedMapDestination(null, onFailure); + } + + /** Routes both succeeded and failed item records. */ + public static DistributedMapDestination of(Success onSuccess, Failure onFailure) { + return new DistributedMapDestination(onSuccess, onFailure); + } /** Route succeeded item records to an S3 prefix (defaults to include output only). */ - public static SuccessDestination successes(String prefixUri) { - return successes(prefixUri, false, true, null); + public static Success s3Successes(String prefixUri) { + return s3Successes(prefixUri, false, true, null); } /** Route succeeded item records to an S3 prefix. */ - public static SuccessDestination successes( + public static Success s3Successes( String prefixUri, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return new SuccessDestination( + return new Success( parsed.bucket(), parsed.path() != null ? parsed.path() : "", includeInput, @@ -26,19 +40,51 @@ public static SuccessDestination successes( } /** Route permanently-failed item records to an S3 prefix (defaults to include input and error). */ - public static FailureDestination failures(String prefixUri) { - return failures(prefixUri, true, true, null); + public static Failure s3Failures(String prefixUri) { + return s3Failures(prefixUri, true, true, null); } /** Route permanently-failed item records to an S3 prefix. */ - public static FailureDestination failures( + public static Failure s3Failures( String prefixUri, boolean includeInput, boolean includeError, String expectedBucketOwner) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return new FailureDestination( + return new Failure( parsed.bucket(), parsed.path() != null ? parsed.path() : "", includeInput, includeError, expectedBucketOwner); } + + /** S3 destination for succeeded distributed map items. */ + public record Success( + String bucket, String prefix, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { + + public Success { + if (bucket == null || bucket.isEmpty()) { + throw new IllegalArgumentException("success destination bucket cannot be empty"); + } + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + DistributedMapValidation.validateDestinationPrefix(prefix); + if (!includeInput && !includeOutput) { + throw new IllegalArgumentException("success destination must include at least one of input or output"); + } + } + } + + /** S3 destination for permanently-failed distributed map items. */ + public record Failure( + String bucket, String prefix, boolean includeInput, boolean includeError, String expectedBucketOwner) { + + public Failure { + if (bucket == null || bucket.isEmpty()) { + throw new IllegalArgumentException("failure destination bucket cannot be empty"); + } + DistributedMapValidation.validateBucketOwner(expectedBucketOwner); + DistributedMapValidation.validateDestinationPrefix(prefix); + if (!includeInput && !includeError) { + throw new IllegalArgumentException("failure destination must include at least one of input or error"); + } + } + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java deleted file mode 100644 index 4ce47e60b8..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapDestinationConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -/** Destination routing for distributed map run results. */ -public record DistributedMapDestinationConfig(SuccessDestination onSuccess, FailureDestination onFailure) { - - /** Routes only succeeded item records. */ - public static DistributedMapDestinationConfig onSuccess(SuccessDestination onSuccess) { - return new DistributedMapDestinationConfig(onSuccess, null); - } - - /** Routes only failed item records. */ - public static DistributedMapDestinationConfig onFailure(FailureDestination onFailure) { - return new DistributedMapDestinationConfig(null, onFailure); - } - - /** Routes both succeeded and failed item records. */ - public static DistributedMapDestinationConfig of(SuccessDestination onSuccess, FailureDestination onFailure) { - return new DistributedMapDestinationConfig(onSuccess, onFailure); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java index db38c7e94c..6c70ee05e1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessor.java @@ -9,20 +9,8 @@ public class DistributedMapProcessor { /** Per-batch response mode reported by the processor. */ public enum ResponseMode { - REPORT_BATCH_OUTCOME(null), - REPORT_FAILED_ITEMS("REPORT_BATCH_ITEM_FAILURES"), - REPORT_ITEM_RESULTS("REPORT_BATCH_ITEM_RESULTS"); - - private final String value; - - ResponseMode(String value) { - this.value = value; - } - - /** Returns the wire functionResponseTypes value, or null for the batch-outcome mode. */ - public String getValue() { - return value; - } + BATCH, + ITEM_OUTCOMES } private static final int MIN_BATCH_SIZE = 1; @@ -32,7 +20,7 @@ public String getValue() { private final String functionName; private final ResponseMode responseMode; private final Integer batchSize; - private final ProcessorRetryConfig retryConfig; + private final DistributedMapProcessorRetryConfig retry; private final String durableExecutionNamePrefix; private DistributedMapProcessor(Builder builder) { @@ -49,7 +37,7 @@ private DistributedMapProcessor(Builder builder) { this.functionName = builder.functionName; this.responseMode = builder.responseMode; this.batchSize = builder.batchSize; - this.retryConfig = builder.retryConfig; + this.retry = builder.retry; this.durableExecutionNamePrefix = builder.durableExecutionNamePrefix; } @@ -65,8 +53,9 @@ public Integer batchSize() { return batchSize; } - public ProcessorRetryConfig retryConfig() { - return retryConfig; + /** Returns the retry settings, or null for the service defaults. */ + public DistributedMapProcessorRetryConfig retry() { + return retry; } public String durableExecutionNamePrefix() { @@ -75,17 +64,15 @@ public String durableExecutionNamePrefix() { /** Processor that reports a single pass/fail outcome for the whole batch, with no per-item results. */ public static Builder reportBatchOutcome(String functionName) { - return new Builder(functionName, ResponseMode.REPORT_BATCH_OUTCOME); - } - - /** Processor that reports the ids of failed items, with all others marked succeeded. */ - public static Builder reportFailedItems(String functionName) { - return new Builder(functionName, ResponseMode.REPORT_FAILED_ITEMS); + return new Builder(functionName, ResponseMode.BATCH); } - /** Processor that reports the results (output or error) for every item. */ - public static Builder reportItemResults(String functionName) { - return new Builder(functionName, ResponseMode.REPORT_ITEM_RESULTS); + /** + * Processor that may report a failure or an output for each item. Every item it does not list succeeds with no + * output. + */ + public static Builder reportItemOutcomes(String functionName) { + return new Builder(functionName, ResponseMode.ITEM_OUTCOMES); } /** Builder for DistributedMapProcessor. */ @@ -93,7 +80,7 @@ public static class Builder { private final String functionName; private final ResponseMode responseMode; private Integer batchSize; - private ProcessorRetryConfig retryConfig; + private DistributedMapProcessorRetryConfig retry; private String durableExecutionNamePrefix; private Builder(String functionName, ResponseMode responseMode) { @@ -106,8 +93,14 @@ public Builder batchSize(int batchSize) { return this; } - public Builder retryConfig(ProcessorRetryConfig retryConfig) { - this.retryConfig = retryConfig; + /** + * Sets the per-item retry policy. Omit it to take the service defaults of 2 attempts within 6 hours. + * + * @param retry the retry settings + * @return this builder + */ + public Builder retry(DistributedMapProcessorRetryConfig retry) { + this.retry = retry; return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java similarity index 60% rename from sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java rename to sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java index 4fa361fce1..bda75df6a5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/ProcessorRetryConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfig.java @@ -4,8 +4,14 @@ import java.time.Duration; -/** Retry configuration for a distributed map processor. */ -public class ProcessorRetryConfig { +/** + * Retry settings for a distributed map processor. + * + *

Retry policy belongs to the processor reference rather than the run, because it defines how the service invokes + * that function. One Lambda function can back several processor references with different policies. + */ +public class DistributedMapProcessorRetryConfig { + /** Sentinel for unbounded retry attempts (wire value -1). */ public static final int UNLIMITED = -1; @@ -15,49 +21,60 @@ public class ProcessorRetryConfig { private final Integer maxRetryAttempts; private final Duration maxRetryDuration; - private ProcessorRetryConfig(Builder builder) { + private DistributedMapProcessorRetryConfig(Builder builder) { this.maxRetryAttempts = builder.maxRetryAttempts; this.maxRetryDuration = builder.maxRetryDuration; } - /** Returns the max retry attempts, UNLIMITED for unbounded, or null for the default. */ + /** Returns the max retry attempts, UNLIMITED for unbounded, or null for the service default of 2. */ public Integer maxRetryAttempts() { return maxRetryAttempts; } - /** Returns the cumulative retry duration budget, or null for the default. */ + /** Returns the cumulative retry duration budget, or null for the service default of 6 hours. */ public Duration maxRetryDuration() { return maxRetryDuration; } + /** + * Creates a new builder for {@code DistributedMapProcessorRetryConfig}. + * + * @return a new builder instance + */ public static Builder builder() { return new Builder(); } - public Builder toBuilder() { - var builder = new Builder(); - builder.maxRetryAttempts = maxRetryAttempts; - builder.maxRetryDuration = maxRetryDuration; - return builder; - } - - /** Builder for ProcessorRetryConfig. */ + /** Builder for DistributedMapProcessorRetryConfig. */ public static class Builder { private Integer maxRetryAttempts; private Duration maxRetryDuration; private Builder() {} + /** + * Sets the number of function-error retries before an item permanently fails. Pass {@link #UNLIMITED} for no + * attempt limit within the duration budget, or 0 to disable retries. + * + * @param maxRetryAttempts a non-negative count, or UNLIMITED + * @return this builder + */ public Builder maxRetryAttempts(int maxRetryAttempts) { if (maxRetryAttempts < 0 && maxRetryAttempts != UNLIMITED) { throw new IllegalArgumentException( - "maxRetryAttempts must be non-negative or ProcessorRetryConfig.UNLIMITED, got: " + "maxRetryAttempts must be non-negative or DistributedMapProcessorRetryConfig.UNLIMITED, got: " + maxRetryAttempts); } this.maxRetryAttempts = maxRetryAttempts; return this; } + /** + * Sets the cumulative backoff budget across all of an item's retries, from 1 minute through 6 hours. + * + * @param maxRetryDuration the budget + * @return this builder + */ public Builder maxRetryDuration(Duration maxRetryDuration) { if (maxRetryDuration != null && (maxRetryDuration.toSeconds() < MIN_RETRY_DURATION_SECONDS @@ -69,8 +86,8 @@ public Builder maxRetryDuration(Duration maxRetryDuration) { return this; } - public ProcessorRetryConfig build() { - return new ProcessorRetryConfig(this); + public DistributedMapProcessorRetryConfig build() { + return new DistributedMapProcessorRetryConfig(this); } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java index 3d3607d4c7..620e460567 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/DistributedMapSource.java @@ -2,7 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.config; +import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.util.DistributedMapValidation; @@ -15,6 +17,8 @@ */ public class DistributedMapSource { + private static final long MAX_ITEMS_TO_READ = 100_000_000L; + /** The kind of source backing a distributed map. */ public enum SourceType { INLINE, @@ -29,10 +33,11 @@ public enum Format { CSV } - /** Whether the backend flattens object contents into items or treats each object as one item. */ - public enum Transform { - NONE, - LOAD_AND_FLATTEN + /** What an S3 source reads. */ + public enum S3SourceType { + OBJECT, + LISTED_OBJECT_METADATA, + LISTED_OBJECTS } private final SourceType sourceType; @@ -49,12 +54,14 @@ private DistributedMapSource( SerDes inlineSerDes, S3SourceConfig s3, ReaderSourceConfig reader) { - if (maxItems != null && maxItems < 1) { - throw new IllegalArgumentException("maxItems must be at least 1, got: " + maxItems); + if (maxItems != null && (maxItems < 1 || maxItems > MAX_ITEMS_TO_READ)) { + throw new IllegalArgumentException( + "maxItems must be between 1 and " + MAX_ITEMS_TO_READ + ", got: " + maxItems); } this.sourceType = sourceType; this.maxItems = maxItems; - this.inlineItems = inlineItems != null ? List.copyOf(inlineItems) : null; + // A null item is allowed and is sent as JSON null, so the copy must accept null elements. + this.inlineItems = inlineItems != null ? Collections.unmodifiableList(new ArrayList<>(inlineItems)) : null; this.inlineSerDes = inlineSerDes; this.s3 = s3; this.reader = reader; @@ -92,7 +99,7 @@ public static DistributedMapSource inline(Collection items) { /** An in-memory list of items embedded in the start checkpoint, serialized with the given SerDes. */ public static DistributedMapSource inline(Collection items, SerDes serDes) { ParameterValidator.validateOrderedCollection(items); - return new DistributedMapSource<>(SourceType.INLINE, null, List.copyOf(items), serDes, null, null); + return new DistributedMapSource<>(SourceType.INLINE, null, new ArrayList<>(items), serDes, null, null); } /** Read a single S3 object, treating each line as an item. */ @@ -101,7 +108,7 @@ public static DistributedMapSource s3JsonLines(String uri) { if (parsed.path() == null) { throw new IllegalArgumentException("s3JsonLines requires an S3 object key"); } - return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.JSON_LINES, null, null)); + return s3(object(parsed.bucket(), parsed.path(), Format.JSON_LINES, null)); } /** Read a single S3 object holding a JSON array, treating each element as an item. */ @@ -110,7 +117,7 @@ public static DistributedMapSource s3JsonArray(String uri) { if (parsed.path() == null) { throw new IllegalArgumentException("s3JsonArray requires an S3 object key"); } - return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.JSON_ARRAY, null, null)); + return s3(object(parsed.bucket(), parsed.path(), Format.JSON_ARRAY, null)); } /** Read a single S3 object, treating each record as an item. */ @@ -119,34 +126,52 @@ public static DistributedMapSource s3Csv(String uri, CsvFormat format) { if (parsed.path() == null) { throw new IllegalArgumentException("s3Csv requires an S3 object key"); } - return s3(new S3SourceConfig(parsed.bucket(), parsed.path(), null, null, Format.CSV, format, null)); + return s3(object(parsed.bucket(), parsed.path(), Format.CSV, format)); } /** Read each object under a prefix as one item (object contents are not read). */ public static DistributedMapSource s3Objects(String prefixUri) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return s3(new S3SourceConfig(parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.NONE, null, null, null)); + return s3(new S3SourceConfig( + S3SourceType.LISTED_OBJECT_METADATA, + parsed.bucket(), + null, + null, + new ListedObjectMetadataConfig(prefixOrEmpty(parsed.path())), + null)); } /** Read a prefix, flattening each object's lines into items. */ public static DistributedMapSource s3FlattenedJsonLines(String prefixUri) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return s3(new S3SourceConfig( - parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.JSON_LINES, null, null)); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.JSON_LINES, null)); } /** Read a prefix, flattening each object's JSON array elements into items. */ public static DistributedMapSource s3FlattenedJsonArray(String prefixUri) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return s3(new S3SourceConfig( - parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.JSON_ARRAY, null, null)); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.JSON_ARRAY, null)); } /** Read a prefix, flattening each object's records into items. */ public static DistributedMapSource s3FlattenedCsv(String prefixUri, CsvFormat format) { var parsed = DistributedMapValidation.parseS3Uri(prefixUri); - return s3(new S3SourceConfig( - parsed.bucket(), null, prefixOrEmpty(parsed.path()), Transform.LOAD_AND_FLATTEN, Format.CSV, format, null)); + return s3(listedObjects(parsed.bucket(), prefixOrEmpty(parsed.path()), Format.CSV, format)); + } + + private static S3SourceConfig object(String bucket, String key, Format format, CsvFormat csvFormat) { + return new S3SourceConfig( + S3SourceType.OBJECT, bucket, null, new ObjectConfig(key, format, csvFormat), null, null); + } + + private static S3SourceConfig listedObjects(String bucket, String keyPrefix, Format format, CsvFormat csvFormat) { + return new S3SourceConfig( + S3SourceType.LISTED_OBJECTS, + bucket, + null, + null, + null, + new ListedObjectsConfig(keyPrefix, format, csvFormat)); } /** Page items from a customer-supplied reader Lambda function. */ @@ -166,7 +191,7 @@ public static DistributedMapSource reader(String functionName, S initi } /** Returns a copy limiting the total number of items read. */ - public DistributedMapSource maxItemsToRead(long maxItems) { + public DistributedMapSource maxItems(long maxItems) { return new DistributedMapSource<>(sourceType, maxItems, inlineItems, inlineSerDes, s3, reader); } @@ -188,25 +213,91 @@ private static String prefixOrEmpty(String path) { return path != null ? path : ""; } - /** Resolved S3 source configuration. */ + /** Resolved S3 source configuration. Exactly one config block matching {@code type} is set. */ public record S3SourceConfig( + S3SourceType type, String bucket, - String key, - String prefix, - Transform transform, - Format format, - CsvFormat csvFormat, - String expectedBucketOwner) { + String expectedBucketOwner, + ObjectConfig objectConfig, + ListedObjectMetadataConfig listedObjectMetadataConfig, + ListedObjectsConfig listedObjectsConfig) { S3SourceConfig withExpectedBucketOwner(String owner) { - return new S3SourceConfig(bucket, key, prefix, transform, format, csvFormat, owner); + return new S3SourceConfig( + type, bucket, owner, objectConfig, listedObjectMetadataConfig, listedObjectsConfig); } } + /** Reads the single object at {@code key}. */ + public record ObjectConfig(String key, Format format, CsvFormat csvFormat) {} + + /** Lists the objects under {@code keyPrefix} without reading their contents. */ + public record ListedObjectMetadataConfig(String keyPrefix) {} + + /** Lists the objects under {@code keyPrefix} and reads every listed object as {@code format}. */ + public record ListedObjectsConfig(String keyPrefix, Format format, CsvFormat csvFormat) {} + /** Resolved reader-function source configuration. */ public record ReaderSourceConfig(String functionName, Object initialState, SerDes stateSerDes) { public ReaderSourceConfig { DistributedMapValidation.validateFunctionName(functionName); } } + + /** CSV parsing options for a distributed map S3 source. */ + public record CsvFormat(HeaderLocation headerLocation, List columns, CsvDelimiter delimiter) { + + /** Where the column header row comes from. */ + public enum HeaderLocation { + FIRST_ROW, + GIVEN + } + + public CsvFormat { + if (headerLocation == null) { + throw new IllegalArgumentException("CsvFormat requires a headerLocation"); + } + if (headerLocation == HeaderLocation.GIVEN) { + if (columns == null) { + throw new IllegalArgumentException("CsvFormat with headerLocation GIVEN requires headers"); + } + DistributedMapValidation.validateColumns("headers", columns); + } else { + DistributedMapValidation.validateColumns("expectedColumns", columns); + } + columns = columns != null ? List.copyOf(columns) : null; + delimiter = delimiter != null ? delimiter : CsvDelimiter.COMMA; + } + + /** + * File has its own header row. Column names are used only for compile-time typing and are not sent on the wire. + */ + public static CsvFormat expectedColumns(List columns) { + return new CsvFormat(HeaderLocation.FIRST_ROW, columns, CsvDelimiter.COMMA); + } + + /** File has no header row. The given names are the columns, sent on the wire, and the first row is data. */ + public static CsvFormat headers(List headers) { + return new CsvFormat(HeaderLocation.GIVEN, headers, CsvDelimiter.COMMA); + } + + /** File has its own header row and no column names are declared. */ + public static CsvFormat firstRow() { + return new CsvFormat(HeaderLocation.FIRST_ROW, null, CsvDelimiter.COMMA); + } + + /** Returns a copy with the given delimiter. */ + public CsvFormat withDelimiter(CsvDelimiter delimiter) { + return new CsvFormat(headerLocation, columns, delimiter); + } + } + + /** Delimiter for CSV distributed map sources. */ + public enum CsvDelimiter { + COMMA, + PIPE, + SEMICOLON, + SPACE, + TAB + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java deleted file mode 100644 index 87a9aa822c..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/FailureDestination.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -import software.amazon.lambda.durable.util.DistributedMapValidation; - -/** S3 destination for permanently-failed distributed map items. */ -public record FailureDestination( - String bucket, String prefix, boolean includeInput, boolean includeError, String expectedBucketOwner) { - - public FailureDestination { - DistributedMapValidation.validateBucketOwner(expectedBucketOwner); - if (!includeInput && !includeError) { - throw new IllegalArgumentException("failure destination must include at least one of input or error"); - } - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java b/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java deleted file mode 100644 index 42a31cd0fe..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/SuccessDestination.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.config; - -import software.amazon.lambda.durable.util.DistributedMapValidation; - -/** S3 destination for succeeded distributed map items. */ -public record SuccessDestination( - String bucket, String prefix, boolean includeInput, boolean includeOutput, String expectedBucketOwner) { - - public SuccessDestination { - DistributedMapValidation.validateBucketOwner(expectedBucketOwner); - if (!includeInput && !includeOutput) { - throw new IllegalArgumentException("success destination must include at least one of input or output"); - } - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java index 69a45cc882..d755231fa9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java @@ -3,7 +3,6 @@ package software.amazon.lambda.durable.context; import com.amazonaws.services.lambda.runtime.Context; -import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; import java.time.Duration; import java.util.Collection; import java.util.List; @@ -11,6 +10,7 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; import software.amazon.lambda.durable.DurableCallbackFuture; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.DurableContext; @@ -36,6 +36,7 @@ import software.amazon.lambda.durable.execution.SuspendExecutionException; import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.MapResult; import software.amazon.lambda.durable.model.OperationIdentifier; @@ -44,7 +45,6 @@ import software.amazon.lambda.durable.operation.CallbackOperation; import software.amazon.lambda.durable.operation.ChildContextOperation; import software.amazon.lambda.durable.operation.DistributedMapOperation; -import software.amazon.lambda.durable.operation.DistributedMapWire; import software.amazon.lambda.durable.operation.InvokeOperation; import software.amazon.lambda.durable.operation.MapOperation; import software.amazon.lambda.durable.operation.ParallelOperation; @@ -396,7 +396,7 @@ public DurableFuture distributedMapAsync( config, false, TypeToken.get(DistributedMapSummary.class), - DistributedMapWire::toSummary); + DistributedMapOperation::toSummary); } @Override @@ -409,8 +409,9 @@ public DurableFuture> distributedMapAsync( DistributedMapConfig config) { Objects.requireNonNull(resultType, "resultType cannot be null"); Objects.requireNonNull(config, "config cannot be null"); - var resultSerDes = - config.resultSerDes() != null ? config.resultSerDes() : getDurableConfig().getSerDes(); + var resultSerDes = config.resultSerDes() != null + ? config.resultSerDes() + : getDurableConfig().getSerDes(); @SuppressWarnings({"unchecked", "rawtypes"}) TypeToken> token = (TypeToken) TypeToken.get(DistributedMapResult.class); return startDistributedMap( @@ -421,7 +422,7 @@ public DurableFuture> distributedMapAsync( config, true, token, - details -> DistributedMapWire.toResult(details, resultType, resultSerDes)); + (status, details) -> DistributedMapOperation.toResult(status, details, resultType, resultSerDes)); } private DurableFuture startDistributedMap( @@ -432,19 +433,19 @@ private DurableFuture startDistributedMap( DistributedMapConfig config, boolean collectResults, TypeToken resultType, - Function resultBuilder) { + BiFunction resultBuilder) { Objects.requireNonNull(source, "source cannot be null"); Objects.requireNonNull(processor, "processor cannot be null"); Objects.requireNonNull(config, "config cannot be null"); ParameterValidator.validateOperationName(name); - if (maxConcurrency <= 0) { + if (maxConcurrency < 1) { throw new IllegalArgumentException("maxConcurrency must be greater than zero, got: " + maxConcurrency); } if (!collectResults && config.resultSerDes() != null) { throw new IllegalArgumentException( "resultSerDes is set but this distributedMap call does not collect results, use a resultType overload"); } - var options = DistributedMapWire.toOptions( + var options = DistributedMapOperation.toOptions( source, processor, maxConcurrency, config, getDurableConfig().getSerDes(), collectResults); var operationId = nextOperationId(); var operation = new DistributedMapOperation<>( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java new file mode 100644 index 0000000000..eb08d7ecc2 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/DistributedMapHandlers.java @@ -0,0 +1,320 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import com.amazonaws.services.lambda.runtime.RequestHandler; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.CompletionConfig; +import software.amazon.lambda.durable.config.MapConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.DurableExecutionOutput; +import software.amazon.lambda.durable.model.MapResult; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Authoring helpers that wrap a customer function into a distributed map processor or reader Lambda handler. */ +public final class DistributedMapHandlers { + private static final SerDes JSON = new JacksonSerDes(); + private static final TypeToken OUTPUT_TYPE = TypeToken.get(String.class); + private static final int MIN_CONCURRENCY = 1; + private static final int DEFAULT_CONCURRENCY = 1; + private static final TypeToken EVENT_TYPE = TypeToken.get(ProcessorEvent.class); + private static final int READER_STATE_LIMIT = 32 * 1024; + private static final String ITEMS_OP_NAME = "distributed-map-items"; + + private DistributedMapHandlers() {} + + private static void validateConcurrency(int concurrency) { + if (concurrency < MIN_CONCURRENCY) { + throw new IllegalArgumentException( + "concurrency must be at least " + MIN_CONCURRENCY + ", got: " + concurrency); + } + } + + /** Wraps a per-item function as a processor handler, running one item at a time. */ + public static RequestHandler createDistributedMapItemHandler( + Function func, TypeToken itemType, SerDes itemSerDes, SerDes resultSerDes) { + return createDistributedMapItemHandler(func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY); + } + + /** Wraps a per-item function as a processor handler. Pair it with DistributedMapProcessor.reportItemOutcomes. */ + public static RequestHandler createDistributedMapItemHandler( + Function func, TypeToken itemType, SerDes itemSerDes, SerDes resultSerDes, int concurrency) { + validateConcurrency(concurrency); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return (event, context) -> { + var records = event.records(); + var outputs = new String[records.size()]; + var errors = new Throwable[records.size()]; + var pool = Executors.newFixedThreadPool(concurrency); + try { + var futures = new ArrayList>(); + for (var i = 0; i < records.size(); i++) { + var index = i; + futures.add(pool.submit(() -> { + I item = toItem(inSerdes, records.get(index).body(), itemType); + O output = func.apply(item); + outputs[index] = toOutput(outSerdes, output); + return null; + })); + } + for (var i = 0; i < futures.size(); i++) { + try { + futures.get(i).get(); + } catch (ExecutionException e) { + errors[i] = e.getCause() != null ? e.getCause() : e; + } + } + } catch (InterruptedException e) { + pool.shutdownNow(); + Thread.currentThread().interrupt(); + throw new IllegalStateException("Distributed map item handler interrupted", e); + } finally { + pool.shutdown(); + } + + var results = new ArrayList(); + var failures = new ArrayList(); + for (var i = 0; i < records.size(); i++) { + var itemId = records.get(i).itemId(); + if (errors[i] != null) { + failures.add(new ItemFailure(itemId, ItemFailure.ErrorDetail.of(errors[i]))); + } else { + results.add(new ItemResult(itemId, outputs[i])); + } + } + return new ItemHandlerResponse(results, failures); + }; + } + + /** + * Wraps a whole-batch function as a processor handler. Returning succeeds every item, throwing fails the batch. + * Pair it with DistributedMapProcessor.reportBatchOutcome. + */ + public static RequestHandler createDistributedMapBatchHandler( + Consumer> func, TypeToken itemType, SerDes itemSerDes) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return (event, context) -> { + func.accept(toItems(serdes, event, itemType)); + return null; + }; + } + + /** + * Wraps a reader function as a source handler. A null nextState signals the source is exhausted. + * + *

itemSerDes must be the serdes the processor is configured with, since the processor deserializes every item + * body with its own item serdes. The reader and the processor are separate Lambdas, so nothing checks that they + * match. + */ + public static RequestHandler createDistributedMapReader( + BiFunction> func, + TypeToken stateType, + SerDes stateSerDes, + SerDes itemSerDes) { + var serdes = stateSerDes != null ? stateSerDes : JSON; + var itemSerdes = itemSerDes != null ? itemSerDes : JSON; + return (event, context) -> { + S state = event.state() != null ? serdes.deserialize(event.state(), stateType) : null; + + var page = func.apply(state, event.maxItemsPerPage()); + if (page.items().size() > event.maxItemsPerPage()) { + throw new IllegalStateException("reader returned " + + page.items().size() + " items, exceeding maxItemsPerPage " + event.maxItemsPerPage()); + } + var items = new ArrayList(page.items().size()); + var index = 0; + for (I item : page.items()) { + var body = itemSerdes.serialize(item); + if (body == null) { + // The service requires every item to be a string, so a null item is sent as the JSON literal null. + if (item != null) { + throw new IllegalStateException( + "serialization of reader item " + index + " produced no string"); + } + body = "null"; + } + items.add(body); + index++; + } + String nextState = null; + if (page.nextState() != null) { + nextState = serdes.serialize(page.nextState()); + if (nextState == null) { + throw new IllegalStateException("serialization of reader nextState produced no string"); + } + if (nextState.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalStateException( + "reader nextState exceeds the " + (READER_STATE_LIMIT / 1024) + " KB limit"); + } + } + return new ReaderResponse(items, nextState); + }; + } + + /** Durable variant of the item handler, running one item at a time. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY, null); + } + + /** Durable variant of the item handler, running one item at a time, with a durable execution config. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + DurableConfig durableConfig) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, DEFAULT_CONCURRENCY, durableConfig); + } + + /** Durable variant of the item handler. The function receives the per-item DurableContext. */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + int concurrency) { + return createDistributedMapItemHandlerWithDurableExecution( + func, itemType, itemSerDes, resultSerDes, concurrency, null); + } + + /** + * Durable variant of the item handler. The function receives the per-item DurableContext. + * + * @param durableConfig the config the wrapped durable execution runs under, for example to register plugins or + * supply a client. A null uses DurableConfig.defaultConfig. + */ + public static + RequestHandler + createDistributedMapItemHandlerWithDurableExecution( + BiFunction func, + TypeToken itemType, + SerDes itemSerDes, + SerDes resultSerDes, + int concurrency, + DurableConfig durableConfig) { + validateConcurrency(concurrency); + var inSerdes = itemSerDes != null ? itemSerDes : JSON; + var outSerdes = resultSerDes != null ? resultSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + var records = event.records(); + var bodies = new ArrayList(records.size()); + for (var record : records) { + bodies.add(record.body()); + } + MapResult batch = ctx.map( + ITEMS_OP_NAME, + bodies, + OUTPUT_TYPE, + (body, index, mapContext) -> { + O output = func.apply(mapContext, toItem(inSerdes, body, itemType)); + return toOutput(outSerdes, output); + }, + MapConfig.builder() + .completionConfig(CompletionConfig.allCompleted()) + .maxConcurrency(concurrency) + .build()); + + var results = new ArrayList(); + var failures = new ArrayList(); + for (var i = 0; i < records.size(); i++) { + var itemId = records.get(i).itemId(); + var item = batch.getItem(i); + if (item.status() == MapResult.MapResultItem.Status.SUCCEEDED) { + results.add(new ItemResult(itemId, item.result())); + } else { + var error = item.error(); + failures.add(new ItemFailure( + itemId, + error != null + ? new ItemFailure.ErrorDetail( + error.errorType(), error.errorMessage(), null, error.stackTrace()) + : null)); + } + } + return new ItemHandlerResponse(results, failures); + }, + configOrDefault(durableConfig)); + } + + /** Durable variant of the batch handler. The function receives the DurableContext. */ + public static + RequestHandler + createDistributedMapBatchHandlerWithDurableExecution( + BiConsumer> func, TypeToken itemType, SerDes itemSerDes) { + return createDistributedMapBatchHandlerWithDurableExecution(func, itemType, itemSerDes, null); + } + + /** + * Durable variant of the batch handler. The function receives the DurableContext. + * + * @param durableConfig the config the wrapped durable execution runs under, for example to register plugins or + * supply a client. A null uses DurableConfig.defaultConfig. + */ + public static + RequestHandler + createDistributedMapBatchHandlerWithDurableExecution( + BiConsumer> func, + TypeToken itemType, + SerDes itemSerDes, + DurableConfig durableConfig) { + var serdes = itemSerDes != null ? itemSerDes : JSON; + return DurableExecutor.wrap( + EVENT_TYPE, + (event, ctx) -> { + func.accept(ctx, toItems(serdes, event, itemType)); + return null; + }, + configOrDefault(durableConfig)); + } + + private static DurableConfig configOrDefault(DurableConfig durableConfig) { + return durableConfig != null ? durableConfig : DurableConfig.defaultConfig(); + } + + private static List toItems(SerDes serdes, ProcessorEvent event, TypeToken itemType) { + var records = event.records(); + var items = new ArrayList(records.size()); + for (var record : records) { + items.add(toItem(serdes, record.body(), itemType)); + } + return items; + } + + private static I toItem(SerDes serdes, String body, TypeToken itemType) { + return serdes.deserialize(body, itemType); + } + + /** Serializes an item's output, or returns null when the handler returned nothing. */ + private static String toOutput(SerDes serdes, Object output) { + return output != null ? serdes.serialize(output) : null; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java new file mode 100644 index 0000000000..8620aaffa7 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemFailure.java @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * A failed item in a processor response. The error is a nested object on the wire, so it is a nested type here. + * + * @param itemIdentifier the item's identifier, copied from the input record + * @param error why the item failed, or null to report a failure with no detail + */ +public record ItemFailure(String itemIdentifier, ErrorDetail error) { + + /** + * Why an item failed. + * + * @param errorType the error's type, conventionally a fully qualified class name + * @param errorMessage the error's message, or null when it has none + * @param errorData machine-readable error detail, or null when there is none + * @param stackTrace the frames where the error was thrown, or null when they were not captured + */ + public record ErrorDetail(String errorType, String errorMessage, String errorData, List stackTrace) { + + /** Builds an error detail from a throwable. */ + public static ErrorDetail of(Throwable e) { + // errorData stays null. A throwable carries no machine-readable error data, so there is nothing + // to put here. + return new ErrorDetail( + e.getClass().getName(), + e.getMessage(), + null, + ExceptionHelper.serializeStackTrace(e.getStackTrace())); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java new file mode 100644 index 0000000000..acb9d80a28 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemHandlerResponse.java @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The reply an item handler returns. + * + *

A null list is omitted from the JSON rather than sent as null, because the Lambda runtime's serializer excludes + * null fields. The service treats an absent, null, or empty list identically, so this is a shape choice rather than a + * behavioural one. + * + * @param batchItemResults per-item outputs, or null to leave the key off the wire + * @param batchItemFailures the items that failed + */ +public record ItemHandlerResponse(List batchItemResults, List batchItemFailures) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java new file mode 100644 index 0000000000..82d50f25f0 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ItemResult.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * A succeeded item in a processor response. + * + * @param itemIdentifier the item's identifier, copied from the input record + * @param output the item's output as the result serdes produced it, or null when the handler returned nothing. A null + * output means the item succeeded without an output. + */ +public record ItemResult(String itemIdentifier, String output) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java new file mode 100644 index 0000000000..14046b9cfd --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorEvent.java @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The batch the fan-out service sends to a processor function. + * + * @param records the items in this batch + */ +public record ProcessorEvent(List records) { + + /** + * Rejects an event that is not a processor batch. This runs on instances the Lambda runtime deserializes as well as + * on hand-constructed ones, because a record can only be built through its canonical constructor. + */ + public ProcessorEvent { + if (records == null) { + throw new IllegalStateException("expected a distributed map processor event carrying a 'records' array, " + + "the function must be triggered by a distributed map, not invoked directly. An event source " + + "such as SQS or S3 sends a capitalised 'Records' array, which is a different contract"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java new file mode 100644 index 0000000000..c4e331fc0c --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ProcessorRecord.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * One record of a processor event. + * + * @param itemId the item's identifier, assigned by the service + * @param body the serialized item, as the item serdes produced it + */ +public record ProcessorRecord(String itemId, String body) { + + /** + * Rejects a record with no item id. The wrappers report each outcome against this id, so a record without one would + * report its outcome against no item. This runs on instances the Lambda runtime deserializes as well. + */ + public ProcessorRecord { + if (itemId == null || itemId.isEmpty()) { + throw new IllegalStateException("expected each distributed map processor record to have an 'itemId'"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java new file mode 100644 index 0000000000..3501feffcf --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderEvent.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +/** + * The request the fan-out service sends to a reader function. + * + * @param maxItemsPerPage the most items the reader may return in this page. Boxed because the Lambda runtime turns an + * absent primitive into zero, which would be indistinguishable from a real value + * @param state the opaque state from the previous call, or null on the first call + */ +public record ReaderEvent(Integer maxItemsPerPage, String state) { + + /** Rejects an event that is not a reader request. Runs on runtime-deserialized instances too. */ + public ReaderEvent { + if (maxItemsPerPage == null) { + throw new IllegalStateException( + "expected a distributed map reader event carrying a numeric 'maxItemsPerPage', " + + "the function must be registered as a distributed map reader"); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java similarity index 57% rename from sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java rename to sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java index 173bac2d55..1c428ffbd2 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/ReaderPage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderPage.java @@ -1,7 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.model; +package software.amazon.lambda.durable.dmap; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -10,6 +11,7 @@ public record ReaderPage(List items, S nextState) { /** Applies a defensive copy and defaults. */ public ReaderPage { - items = items != null ? List.copyOf(items) : Collections.emptyList(); + // A null item is valid and is sent as the JSON literal null. List.copyOf rejects null elements. + items = items != null ? Collections.unmodifiableList(new ArrayList<>(items)) : Collections.emptyList(); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java new file mode 100644 index 0000000000..721434dfe1 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/dmap/ReaderResponse.java @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import java.util.List; + +/** + * The page a reader function returns. + * + *

A null nextState is omitted from the JSON rather than sent as null, because the Lambda runtime's serializer + * excludes null fields. Omitting the key is how the reader contract signals that the source is exhausted. + * + * @param items this page's items, each serialized with the reader's item serdes + * @param nextState the state to resume from, or null when the source is exhausted + */ +public record ReaderResponse(List items, String nextState) {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java deleted file mode 100644 index 49d90359c2..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapError.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.exception; - -import software.amazon.lambda.durable.model.DistributedMapCompletionReason; -import software.amazon.lambda.durable.model.DistributedMapItemError; -import software.amazon.lambda.durable.model.DistributedMapStatus; - -/** - * Run outcome error, thrown by throwIfError when a distributed map run did not fully succeed. - * It reports an aggregate run result and is not a single operation failure, so it stays out of the DurableOperationException hierarchy. - * See DistributedMapException for a failure of the operation itself. - */ -public class DistributedMapError extends DurableExecutionException { - private final DistributedMapStatus status; - private final DistributedMapCompletionReason completionReason; - private final long failureCount; - - private DistributedMapError( - DistributedMapStatus status, - DistributedMapCompletionReason completionReason, - long failureCount, - String message) { - super(message); - this.status = status; - this.completionReason = completionReason; - this.failureCount = failureCount; - } - - /** Run-level failure describing the run status, reason, and failure count. */ - public static DistributedMapError runLevel( - DistributedMapStatus status, - DistributedMapCompletionReason completionReason, - long failureCount, - String completionDetails) { - return new DistributedMapError( - status, - completionReason, - failureCount, - runMessage(status, completionReason, failureCount, completionDetails)); - } - - /** Item-level failure surfacing the first failed item's error. */ - public static DistributedMapError itemLevel( - DistributedMapStatus status, - DistributedMapCompletionReason completionReason, - long failureCount, - DistributedMapItemError error, - String itemId) { - return new DistributedMapError(status, completionReason, failureCount, itemMessage(error, itemId)); - } - - /** Returns the terminal run status. */ - public DistributedMapStatus status() { - return status; - } - - /** Returns the completion reason. */ - public DistributedMapCompletionReason completionReason() { - return completionReason; - } - - /** Returns the count of failed items. */ - public long failureCount() { - return failureCount; - } - - private static String runMessage( - DistributedMapStatus status, - DistributedMapCompletionReason completionReason, - long failureCount, - String completionDetails) { - var message = new StringBuilder("Distributed map run ") - .append(status) - .append(" (") - .append(completionReason) - .append(")"); - if (failureCount > 0) { - message.append(", ").append(failureCount).append(" item(s) failed"); - } - if (completionDetails != null) { - message.append(": ").append(completionDetails); - } - return message.toString(); - } - - private static String itemMessage(DistributedMapItemError error, String itemId) { - if (error != null) { - return error.errorType() + ": " + error.errorMessage(); - } - return "Distributed map item " + itemId + " failed"; - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java index d8ee6d20d8..02f380eef3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/DistributedMapException.java @@ -2,17 +2,76 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.exception; -import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.model.DistributedMapError; +import software.amazon.lambda.durable.model.DistributedMapStatus; /** - * Operation failure, thrown when a distributed map operation itself terminal-fails (FAILED, TIMED_OUT, or STOPPED). - * Extends DurableOperationException. See DistributedMapError for the run outcome, which is an aggregate result rather than an operation failure. + * Thrown by throwIfError when a distributed map run did not fully succeed, or when an item failed on a run that + * otherwise succeeded. It reports an aggregate run result and is not a single operation failure, so it stays out of the + * DurableOperationException hierarchy. */ -public class DistributedMapException extends DurableOperationException { - public DistributedMapException(Operation operation) { - super( - operation, - null, - "Distributed map operation " + operation.id() + " ended with status " + operation.statusAsString()); +public class DistributedMapException extends DurableExecutionException { + private final DistributedMapStatus status; + private final DistributedMapError error; + private final long failureCount; + + /** Run outcome failure describing the run status, failure count, and the run's error details. */ + public DistributedMapException(DistributedMapStatus status, long failureCount, DistributedMapError error) { + super(runMessage(status, failureCount, error)); + this.status = status; + this.error = error; + this.failureCount = failureCount; + } + + /** Item failure surfacing the first failed item's error. */ + public DistributedMapException( + DistributedMapStatus status, long failureCount, DistributedMapError error, String itemId) { + super(itemMessage(error, itemId)); + this.status = status; + this.error = error; + this.failureCount = failureCount; + } + + /** Returns the terminal run status. */ + public DistributedMapStatus status() { + return status; + } + + /** Returns the error details, or null when none were recorded. */ + public DistributedMapError error() { + return error; + } + + /** Returns the count of failed items. */ + public long failureCount() { + return failureCount; + } + + private static String runMessage(DistributedMapStatus status, long failureCount, DistributedMapError error) { + var message = new StringBuilder("Distributed map run ").append(status); + if (failureCount > 0) { + message.append(", ").append(failureCount).append(" item(s) failed"); + } + if (error != null && error.errorMessage() != null) { + message.append(": ").append(error.errorMessage()); + } + return message.toString(); + } + + private static String itemMessage(DistributedMapError error, String itemId) { + var type = error != null ? error.errorType() : null; + var message = error != null ? error.errorMessage() : null; + var hasType = type != null && !type.isEmpty(); + var hasMessage = message != null && !message.isEmpty(); + if (hasType && hasMessage) { + return type + ": " + message; + } + if (hasMessage) { + return message; + } + if (hasType) { + return type; + } + return "Distributed map item " + itemId + " failed"; } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java index 34f44134cd..3480d1932e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/CheckpointManager.java @@ -5,6 +5,7 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -16,6 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.SdkField; +import software.amazon.awssdk.core.SdkPojo; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationUpdate; @@ -270,6 +273,43 @@ private static int estimateSize(OperationUpdate update) { + update.type().toString().length() + update.action().toString().length() + (update.payload() != null ? update.payload().length() : 0) + + stringSize(update.distributedMapOptions()) + 100; } + + /** + * Total length of every string in a model object, including nested objects, lists and maps. A distributed map START + * update carries its customer-supplied values (inline items, reader state, CSV headers, S3 locations, function + * names) in its options rather than the payload, so they are counted here. + */ + private static int stringSize(Object value) { + if (value == null) { + return 0; + } + if (value instanceof String string) { + return string.length(); + } + if (value instanceof SdkPojo pojo) { + var size = 0; + for (SdkField field : pojo.sdkFields()) { + size += stringSize(field.getValueOrDefault(pojo)); + } + return size; + } + if (value instanceof Collection collection) { + var size = 0; + for (var element : collection) { + size += stringSize(element); + } + return size; + } + if (value instanceof Map map) { + var size = 0; + for (var entry : map.entrySet()) { + size += stringSize(entry.getKey()) + stringSize(entry.getValue()); + } + return size; + } + return 0; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index d8db913265..cc838acb79 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -20,6 +20,7 @@ import software.amazon.lambda.durable.DurableContext; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.dmap.ProcessorEvent; import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.UnrecoverableDurableExecutionException; @@ -31,6 +32,7 @@ import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.PluginInfoConverter; import software.amazon.lambda.durable.plugin.PluginRunner; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.util.ExceptionHelper; @@ -46,6 +48,7 @@ public class DurableExecutor { // Lambda response size limit is 6MB minus small epsilon for envelope private static final int LAMBDA_RESPONSE_SIZE_LIMIT = 6 * 1024 * 1024 - 50; + private static final SerDes JSON = new JacksonSerDes(); private DurableExecutor() {} @@ -56,6 +59,9 @@ public static DurableExecutionOutput execute( BiFunction handler, DurableConfig config) { var pluginRunner = config.getPluginRunner(); + // A distributed map processor's input and result have a format the service defines, so a customer's SerDes + // must not change them. Durable operations inside the processor still use the config's SerDes. + var invocationSerDes = inputType.getType() == ProcessorEvent.class ? JSON : config.getSerDes(); try (var executionManager = new ExecutionManager(input, config, lambdaContext)) { var isFirstInvocation = !executionManager.isReplaying(); var requestId = lambdaContext != null ? lambdaContext.getAwsRequestId() : null; @@ -78,7 +84,7 @@ public static DurableExecutionOutput execute( Throwable inputFailure = null; try { userInput = extractUserInput( - executionManager.getExecutionOperation(), config.getSerDes(), inputType); + executionManager.getExecutionOperation(), invocationSerDes, inputType); } catch (Throwable t) { inputFailure = t; } @@ -171,11 +177,11 @@ public static DurableExecutionOutput execute( cause, pluginExecutionInput.get(), null); - return DurableExecutionOutput.failure(buildErrorObject(cause, config.getSerDes())); + return DurableExecutionOutput.failure(buildErrorObject(cause, invocationSerDes)); } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); + var outputPayload = invocationSerDes.serialize(result); var output = DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); fireOnInvocationEnd( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java deleted file mode 100644 index 17377cbf58..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapCompletionReason.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.model; - -/** Reason a distributed map run reached its terminal status. */ -public enum DistributedMapCompletionReason { - ALL_COMPLETED("ALL_COMPLETED"), - ITEM_LIMIT_REACHED("ITEM_LIMIT_REACHED"), - STOPPED("STOPPED"), - TIMED_OUT("TIMED_OUT"), - FAILURE_TOLERANCE_EXCEEDED("FAILURE_TOLERANCE_EXCEEDED"), - SOURCE_FAILED("SOURCE_FAILED"), - DESTINATION_FAILED("DESTINATION_FAILED"), - INLINE_RESULT_LIMIT_EXCEEDED("INLINE_RESULT_LIMIT_EXCEEDED"), - INVALID_CONFIGURATION("INVALID_CONFIGURATION"), - QUOTA_EXCEEDED("QUOTA_EXCEEDED"), - KMS_ACCESS_DENIED("KMS_ACCESS_DENIED"), - INTERNAL_ERROR("INTERNAL_ERROR"), - /** A value the backend returned that this SDK version does not recognize. */ - UNKNOWN_TO_SDK_VERSION("UNKNOWN_TO_SDK_VERSION"); - - private final String value; - - DistributedMapCompletionReason(String value) { - this.value = value; - } - - /** Returns the wire-format string value. */ - public String getValue() { - return value; - } - - /** Returns the reason matching a wire-format string value, or UNKNOWN_TO_SDK_VERSION if unrecognized. */ - public static DistributedMapCompletionReason fromValue(String value) { - for (var reason : values()) { - if (reason.value.equals(value)) { - return reason; - } - } - return UNKNOWN_TO_SDK_VERSION; - } - - @Override - public String toString() { - return value; - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java new file mode 100644 index 0000000000..0ebc1c5f0a --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapError.java @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import java.util.List; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * Error details on a distributed map run or one of its items. + * + *

Held as a record rather than the client's ErrorObject, so it survives serialization across checkpoint-and-replay + * cycles without requiring AWS SDK-specific Jackson modules. + */ +public record DistributedMapError(String errorType, String errorMessage, String errorData, List stackTrace) { + + /** Creates an error from a throwable. */ + public static DistributedMapError of(Throwable e) { + return new DistributedMapError( + e.getClass().getName(), e.getMessage(), null, ExceptionHelper.serializeStackTrace(e.getStackTrace())); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java deleted file mode 100644 index 4af34e1662..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapItemError.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.model; - -/** Error details for a failed distributed map item. */ -public record DistributedMapItemError(String errorType, String errorMessage) { - - /** Creates an item error from a throwable. */ - public static DistributedMapItemError of(Throwable e) { - return new DistributedMapItemError(e.getClass().getName(), e.getMessage()); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java index 7a079eb5d2..289069ad1e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResult.java @@ -4,11 +4,16 @@ import java.util.Collections; import java.util.List; -import java.util.Objects; -import software.amazon.lambda.durable.exception.DistributedMapError; +import software.amazon.lambda.durable.exception.DistributedMapException; -/** Outcome of a distributed map run with collected per-item results. */ -public record DistributedMapResult(DistributedMapSummary summary, List> items) { +/** + * Outcome of a distributed map run with collected per-item results. + * + *

An item whose status this SDK version does not recognize is kept in {@code items} with status + * {@link Item.Status#UNKNOWN_TO_SDK_VERSION}. It is excluded from {@link #succeeded()}, {@link #failed()}, + * {@link #results()} and {@link #errors()}. + */ +public record DistributedMapResult(DistributedMapSummary summary, List> items) { /** Applies a defensive copy and defaults. */ public DistributedMapResult { @@ -16,27 +21,27 @@ public record DistributedMapResult(DistributedMapSummary summary, List> succeeded() { + public List> succeeded() { return items.stream() - .filter(item -> item.status() == DistributedMapResultItem.Status.SUCCEEDED) + .filter(item -> item.status() == Item.Status.SUCCEEDED) .toList(); } /** Returns items that failed. */ - public List> failed() { + public List> failed() { return items.stream() - .filter(item -> item.status() == DistributedMapResultItem.Status.FAILED) + .filter(item -> item.status() == Item.Status.FAILED) .toList(); } - /** Returns the outputs of succeeded items. */ - public List getResults() { - return succeeded().stream().map(DistributedMapResultItem::output).filter(Objects::nonNull).toList(); + /** Returns the outputs of succeeded items (includes null outputs from successful items). */ + public List results() { + return succeeded().stream().map(Item::output).toList(); } - /** Returns the errors of failed items. */ - public List getErrors() { - return failed().stream().map(DistributedMapResultItem::error).filter(Objects::nonNull).toList(); + /** Returns the errors of failed items (includes nulls for items that failed without error details). */ + public List errors() { + return failed().stream().map(Item::error).toList(); } /** Returns the terminal run status. */ @@ -44,9 +49,9 @@ public DistributedMapStatus status() { return summary.status(); } - /** Returns the completion reason. */ - public DistributedMapCompletionReason completionReason() { - return summary.completionReason(); + /** Returns true only when the service read the complete source, or null before run creation. */ + public Boolean sourceExhausted() { + return summary.sourceExhausted(); } /** Returns the count of succeeded items. */ @@ -74,14 +79,9 @@ public String distributedMapRunArn() { return summary.distributedMapRunArn(); } - /** Returns the run id derived from the ARN, or null when the run never started. */ - public String distributedMapId() { - return summary.distributedMapId(); - } - - /** Returns the completion details, or null when absent. */ - public String completionDetails() { - return summary.completionDetails(); + /** Returns why the run did not succeed, or null when the service recorded no error details. */ + public DistributedMapError error() { + return summary.error(); } /** Returns true when any item failed. */ @@ -89,7 +89,10 @@ public boolean hasFailure() { return summary.hasFailure(); } - /** Throws DistributedMapError when the run did not fully succeed, surfacing the first failed item. */ + /** + * Throws DistributedMapException when the run did not succeed or any item failed. When the run succeeded and the + * result contains a failed item, the exception names the first failed item. + */ public void throwIfError() { if (summary.status() != DistributedMapStatus.SUCCEEDED) { summary.throwIfError(); @@ -98,15 +101,37 @@ public void throwIfError() { var failedItems = failed(); if (!failedItems.isEmpty()) { var first = failedItems.get(0); - throw DistributedMapError.itemLevel( - summary.status(), - summary.completionReason(), - summary.failureCount(), - first.error(), - first.itemId()); + throw new DistributedMapException(summary.status(), summary.failureCount(), first.error(), first.itemId()); } if (summary.hasFailure()) { summary.throwIfError(); } } + + /** Outcome of a single distributed map item. */ + public record Item(String itemId, Status status, O output, DistributedMapError error) { + + /** Status of an individual distributed map item. */ + public enum Status { + SUCCEEDED, + FAILED, + /** A status the backend returned that this SDK version does not recognize. */ + UNKNOWN_TO_SDK_VERSION + } + + /** Creates a succeeded item. */ + public static Item succeeded(String itemId, O output) { + return new Item<>(itemId, Status.SUCCEEDED, output, null); + } + + /** Creates a failed item. */ + public static Item failed(String itemId, DistributedMapError error) { + return new Item<>(itemId, Status.FAILED, null, error); + } + + /** Creates an item whose status this SDK version does not recognize. */ + public static Item unknown(String itemId) { + return new Item<>(itemId, Status.UNKNOWN_TO_SDK_VERSION, null, null); + } + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java deleted file mode 100644 index 864cae769b..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapResultItem.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.model; - -/** Outcome of a single distributed map item. */ -public record DistributedMapResultItem(String itemId, Status status, O output, DistributedMapItemError error) { - - /** Status of an individual distributed map item. */ - public enum Status { - SUCCEEDED, - FAILED - } - - /** Creates a succeeded item. */ - public static DistributedMapResultItem succeeded(String itemId, O output) { - return new DistributedMapResultItem<>(itemId, Status.SUCCEEDED, output, null); - } - - /** Creates a failed item. */ - public static DistributedMapResultItem failed(String itemId, DistributedMapItemError error) { - return new DistributedMapResultItem<>(itemId, Status.FAILED, null, error); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java index f704ca3536..421f6e836b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapStatus.java @@ -4,34 +4,8 @@ /** Terminal status of a distributed map run. */ public enum DistributedMapStatus { - SUCCEEDED("SUCCEEDED"), - FAILED("FAILED"), - STOPPED("STOPPED"), - TIMED_OUT("TIMED_OUT"); - - private final String value; - - DistributedMapStatus(String value) { - this.value = value; - } - - /** Returns the wire-format string value. */ - public String getValue() { - return value; - } - - /** Returns the status matching a wire-format string value. */ - public static DistributedMapStatus fromValue(String value) { - for (var status : values()) { - if (status.value.equals(value)) { - return status; - } - } - throw new IllegalArgumentException("Unknown DistributedMapStatus: " + value); - } - - @Override - public String toString() { - return value; - } + SUCCEEDED, + FAILED, + STOPPED, + TIMED_OUT } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java index 8f26f03f02..45a26b03a4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/DistributedMapSummary.java @@ -2,35 +2,33 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.model; -import software.amazon.lambda.durable.exception.DistributedMapError; +import software.amazon.lambda.durable.exception.DistributedMapException; -/** Outcome of a distributed map run, without per-item results. */ +/** + * Outcome of a distributed map run, without per-item results. + * + * @param sourceExhausted true only when the service read the complete source, null before run creation + * @param error why the run did not succeed, when the service recorded error details + */ public record DistributedMapSummary( DistributedMapStatus status, - DistributedMapCompletionReason completionReason, + Boolean sourceExhausted, long successCount, long failureCount, long unprocessedCount, String distributedMapRunArn, - String completionDetails, + DistributedMapError error, Long totalCount) { - /** Returns the run id derived from the ARN, or null when the run never started. */ - public String distributedMapId() { - return distributedMapRunArn != null - ? distributedMapRunArn.substring(distributedMapRunArn.lastIndexOf(':') + 1) - : null; - } - /** Returns true when any item failed. */ public boolean hasFailure() { return failureCount > 0; } - /** Throws DistributedMapError when the run did not fully succeed. */ + /** Throws DistributedMapException when the run did not fully succeed. */ public void throwIfError() { if (status != DistributedMapStatus.SUCCEEDED || hasFailure()) { - throw DistributedMapError.runLevel(status, completionReason, failureCount, completionDetails); + throw new DistributedMapException(status, failureCount, error); } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java index 2228b28f42..8ebf4968e3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/model/OperationSubType.java @@ -23,7 +23,6 @@ public enum OperationSubType { WAIT_FOR_CALLBACK(OperationType.CONTEXT, "WaitForCallback"), WAIT_FOR_CONDITION(OperationType.STEP, "WaitForCondition"), WITH_RETRY(OperationType.CONTEXT, "WithRetry"), - // Requires OperationType.DISTRIBUTED_MAP, generated once the Lambda client ships the distributed-map shapes. DISTRIBUTED_MAP(OperationType.DISTRIBUTED_MAP, "DistributedMap"); private final OperationType operationType; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 5cd40820e9..249ed42000 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -610,6 +610,10 @@ public static ErrorObject getErrorObject(Operation operation) { : null; case CONTEXT -> operation.contextDetails() != null ? operation.contextDetails().error() : null; + case DISTRIBUTED_MAP -> + operation.distributedMapDetails() != null + ? operation.distributedMapDetails().error() + : null; default -> null; }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java index 8c299cfa4a..2be2d089dc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java @@ -259,7 +259,7 @@ private Throwable translateException(Operation op, ErrorObject errorObject) { case RUN_IN_CHILD_CONTEXT, WITH_RETRY -> new ChildContextFailedException(op); // the following subtypes should not be able to reach here - case PARALLEL, MAP, WAIT_FOR_CONDITION, STEP, WAIT, CALLBACK, CHAINED_INVOKE -> + case PARALLEL, MAP, WAIT_FOR_CONDITION, STEP, WAIT, CALLBACK, CHAINED_INVOKE, DISTRIBUTED_MAP -> new IllegalStateException("Unexpected sub-type: " + getSubType()); }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java index e912ef9434..576cb17b74 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapOperation.java @@ -2,18 +2,53 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.operation; -// Written against DistributedMap* shapes not yet in the generated Lambda client, so this does not compile until they ship. - -import java.util.function.Function; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvDelimiter; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvHeaderLocation; +import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationType; import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapFunctionResponseType; +import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureInclude; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessInclude; import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapProcessorConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionMode; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3DestinationConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ListedObjectMetadataConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ListedObjectsConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ObjectConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceType; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; +import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; +import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationUpdate; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapDestination; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; import software.amazon.lambda.durable.context.DurableContextImpl; -import software.amazon.lambda.durable.exception.DistributedMapException; +import software.amazon.lambda.durable.exception.DurableExecutionException; +import software.amazon.lambda.durable.model.DistributedMapError; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; /** @@ -23,14 +58,18 @@ */ public class DistributedMapOperation extends SerializableDurableOperation { private final DistributedMapOptions options; - private final Function resultBuilder; + private final BiFunction resultBuilder; + + private static final SerDes JSON = new JacksonSerDes(); + private static final long INLINE_SIZE_LIMIT = 1024L * 1024L; + private static final long READER_STATE_LIMIT = 32L * 1024L; public DistributedMapOperation( OperationIdentifier operationIdentifier, DistributedMapOptions options, TypeToken resultType, SerDes serDes, - Function resultBuilder, + BiFunction resultBuilder, DurableContextImpl durableContext) { super(operationIdentifier, resultType, serDes, durableContext); this.options = options; @@ -40,15 +79,18 @@ public DistributedMapOperation( /** Starts the operation. */ @Override protected void start() { - sendOperationUpdate(OperationUpdate.builder().action(OperationAction.START).distributedMapOptions(options)); - pollForOperationUpdates(); + sendOperationUpdate( + OperationUpdate.builder().action(OperationAction.START).distributedMapOptions(options)); + pollUntilTerminal(); } /** Replays the operation. */ @Override protected void replay(Operation existing) { switch (existing.status()) { - case STARTED -> pollForOperationUpdates(); + // The start checkpoint records the operation as PENDING, and it reaches STARTED only once the + // service confirms the run was created. A replay in that window waits for the outcome. + case PENDING, STARTED -> pollUntilTerminal(); case SUCCEEDED, FAILED, TIMED_OUT, STOPPED -> markAlreadyCompleted(); default -> throw terminateExecutionWithIllegalDurableOperationException( @@ -56,24 +98,343 @@ protected void replay(Operation existing) { } } + /** + * Polls until the operation reaches a terminal status. A poll completes on any update, so a PENDING to STARTED + * update would otherwise end polling while get() is still waiting for the outcome. + */ + private void pollUntilTerminal() { + pollForOperationUpdates().thenAccept(op -> { + if (op != null && (op.status() == OperationStatus.PENDING || op.status() == OperationStatus.STARTED)) { + pollUntilTerminal(); + } + }); + } + /** Blocks until the operation completes and returns the resolved outcome. */ @Override public R get() { var op = waitForOperationCompletion(); + var runStatus = runStatus(op); + var details = op.distributedMapDetails(); + if (details == null) { + throw terminateExecutionWithIllegalDurableOperationException("DISTRIBUTED_MAP operation reached " + + op.statusAsString() + " but carried no DistributedMapDetails"); + } + // SourceExhausted is required at terminal once a run exists, and is absent only when the operation + // failed before one was created, which is also when the run ARN is absent. + if (details.distributedMapRunArn() != null && details.sourceExhausted() == null) { + throw terminateExecutionWithIllegalDurableOperationException( + "DISTRIBUTED_MAP operation reached " + op.statusAsString() + " but carried no SourceExhausted"); + } + return resultBuilder.apply(runStatus, details); + } + + // An unsuccessful run resolves with its summary, so callers can read the counts. Use + // DistributedMapSummary.throwIfError to opt into raising instead. + private DistributedMapStatus runStatus(Operation op) { return switch (op.status()) { - case SUCCEEDED -> { - var details = op.distributedMapDetails(); - if (details == null) { - throw terminateExecutionWithIllegalDurableOperationException( - "DISTRIBUTED_MAP operation succeeded but carried no DistributedMapDetails"); - } - yield resultBuilder.apply(details); - } - // Reaching here means the operation itself terminal-failed, so raise instead of hanging. - case FAILED, TIMED_OUT, STOPPED -> throw new DistributedMapException(op); + case SUCCEEDED -> DistributedMapStatus.SUCCEEDED; + case FAILED -> DistributedMapStatus.FAILED; + case TIMED_OUT -> DistributedMapStatus.TIMED_OUT; + case STOPPED -> DistributedMapStatus.STOPPED; default -> throw terminateExecutionWithIllegalDurableOperationException( "Unexpected distributed map operation status: " + op.statusAsString()); }; } + + public static DistributedMapOptions toOptions( + DistributedMapSource source, + DistributedMapProcessor processor, + int maxConcurrency, + DistributedMapConfig config, + SerDes defaultSerDes, + boolean collectResults) { + var builder = DistributedMapOptions.builder() + .maxConcurrency(maxConcurrency) + .source(sourceConfig(source, defaultSerDes)) + .processor(processorConfig(processor)); + if (config.destination() != null) { + var destination = destinationConfig(config.destination()); + if (destination != null) { + builder.destination(destination); + } + } + var completion = completionConfig(config); + if (completion != null) { + builder.completionConfig(completion); + } + if (collectResults) { + builder.resultCollection(DistributedMapResultCollectionConfig.builder() + .mode(DistributedMapResultCollectionMode.INLINE) + .build()); + } + if (config.timeout() != null) { + builder.timeoutSeconds((int) config.timeout().toSeconds()); + } + return builder.build(); + } + + private static DistributedMapSourceConfig sourceConfig(DistributedMapSource source, SerDes defaultSerDes) { + var builder = DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.fromValue(source.sourceType().name())); + if (source.maxItems() != null) { + builder.maxItemsToRead(source.maxItems().longValue()); + } + switch (source.sourceType()) { + case INLINE -> builder.inlineSourceConfig(inlineSourceConfig(source, defaultSerDes)); + case S3 -> builder.s3SourceConfig(s3SourceConfig(source.s3())); + case READER_FUNCTION -> builder.readerFunctionSourceConfig(readerConfig(source.reader(), defaultSerDes)); + } + return builder.build(); + } + + private static DistributedMapInlineSourceConfig inlineSourceConfig( + DistributedMapSource source, SerDes defaultSerDes) { + var serdes = source.inlineSerDes() != null ? source.inlineSerDes() : defaultSerDes; + var bodies = new ArrayList(source.inlineItems().size()); + var items = source.inlineItems(); + for (int index = 0; index < items.size(); index++) { + var item = items.get(index); + var body = serdes.serialize(item); + if (body == null) { + // The service requires every item to be a string. The default serdes returns null for a null item, + // so a null item is sent as the JSON literal null. Any other item with no serialized form is an error. + if (item != null) { + throw new IllegalArgumentException("serialization of inline item " + index + " produced no string"); + } + body = "null"; + } + bodies.add(body); + } + var arrayBytes = JSON.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; + if (arrayBytes > INLINE_SIZE_LIMIT) { + throw new IllegalArgumentException( + "inline source exceeds the 1 MB limit (serialized size: " + arrayBytes + " bytes)"); + } + return DistributedMapInlineSourceConfig.builder().items(bodies).build(); + } + + private static DistributedMapS3SourceConfig s3SourceConfig(DistributedMapSource.S3SourceConfig s3) { + var builder = DistributedMapS3SourceConfig.builder() + .type(DistributedMapS3SourceType.fromValue(s3.type().name())) + .bucket(s3.bucket()); + if (s3.expectedBucketOwner() != null) { + builder.expectedBucketOwner(s3.expectedBucketOwner()); + } + if (s3.objectConfig() != null) { + var object = s3.objectConfig(); + var block = DistributedMapS3ObjectConfig.builder() + .key(object.key()) + .format(DistributedMapSourceFormat.fromValue(object.format().name())); + if (object.csvFormat() != null) { + block.csvFormatOptions(csvFormatOptions(object.csvFormat())); + } + builder.objectConfig(block.build()); + } + if (s3.listedObjectMetadataConfig() != null) { + builder.listedObjectMetadataConfig(DistributedMapS3ListedObjectMetadataConfig.builder() + .keyPrefix(s3.listedObjectMetadataConfig().keyPrefix()) + .build()); + } + if (s3.listedObjectsConfig() != null) { + var listed = s3.listedObjectsConfig(); + var block = DistributedMapS3ListedObjectsConfig.builder() + .keyPrefix(listed.keyPrefix()) + .format(DistributedMapSourceFormat.fromValue(listed.format().name())); + if (listed.csvFormat() != null) { + block.csvFormatOptions(csvFormatOptions(listed.csvFormat())); + } + builder.listedObjectsConfig(block.build()); + } + return builder.build(); + } + + private static DistributedMapCsvFormatOptions csvFormatOptions(DistributedMapSource.CsvFormat format) { + var builder = DistributedMapCsvFormatOptions.builder() + .headerLocation(DistributedMapCsvHeaderLocation.fromValue( + format.headerLocation().name())) + .delimiter( + DistributedMapCsvDelimiter.fromValue(format.delimiter().name())); + // Column names go on the wire only for headerless files (GIVEN), expectedColumns (FIRST_ROW) is compile-time + // only. + if (format.headerLocation() == DistributedMapSource.CsvFormat.HeaderLocation.GIVEN + && format.columns() != null) { + builder.headers(format.columns()); + } + return builder.build(); + } + + private static DistributedMapReaderFunctionSourceConfig readerConfig( + DistributedMapSource.ReaderSourceConfig reader, SerDes defaultSerDes) { + var builder = DistributedMapReaderFunctionSourceConfig.builder().functionName(reader.functionName()); + if (reader.initialState() != null) { + var serdes = reader.stateSerDes() != null ? reader.stateSerDes() : defaultSerDes; + var state = serdes.serialize(reader.initialState()); + if (state == null) { + throw new IllegalArgumentException("serialization of reader initialState produced no string"); + } + if (state.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { + throw new IllegalArgumentException("reader initialState exceeds the 32 KB limit"); + } + builder.initialState(state); + } + return builder.build(); + } + + private static DistributedMapProcessorConfig processorConfig(DistributedMapProcessor processor) { + var builder = DistributedMapProcessorConfig.builder().functionName(processor.functionName()); + if (processor.durableExecutionNamePrefix() != null) { + builder.durableExecutionNamePrefix(processor.durableExecutionNamePrefix()); + } + if (processor.responseMode() == DistributedMapProcessor.ResponseMode.ITEM_OUTCOMES) { + builder.functionResponseTypes(List.of(DistributedMapFunctionResponseType.REPORT_BATCH_ITEM_OUTCOMES)); + } + if (processor.batchSize() != null) { + builder.batchSize(processor.batchSize()); + } + if (processor.retry() != null) { + var retry = processor.retry(); + if (retry.maxRetryAttempts() != null) { + builder.maxRetryAttempts(retry.maxRetryAttempts()); + } + if (retry.maxRetryDuration() != null) { + builder.maxRetryDurationSeconds((int) retry.maxRetryDuration().toSeconds()); + } + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig destinationConfig( + software.amazon.lambda.durable.config.DistributedMapDestination destination) { + if (destination.onSuccess() == null && destination.onFailure() == null) { + return null; // omit an empty destination config from the wire + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig.builder(); + DistributedMapDestination.Success onSuccess = destination.onSuccess(); + if (onSuccess != null) { + var include = new ArrayList(); + if (onSuccess.includeInput()) { + include.add(DistributedMapOnSuccessInclude.INPUT); + } + if (onSuccess.includeOutput()) { + include.add(DistributedMapOnSuccessInclude.OUTPUT); + } + builder.onSuccess(DistributedMapOnSuccessConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig( + s3Destination(onSuccess.bucket(), onSuccess.prefix(), onSuccess.expectedBucketOwner())) + .build()); + } + DistributedMapDestination.Failure onFailure = destination.onFailure(); + if (onFailure != null) { + var include = new ArrayList(); + if (onFailure.includeInput()) { + include.add(DistributedMapOnFailureInclude.INPUT); + } + if (onFailure.includeError()) { + include.add(DistributedMapOnFailureInclude.ERROR); + } + builder.onFailure(DistributedMapOnFailureConfig.builder() + .type(DistributedMapDestinationType.S3) + .include(include) + .s3DestinationConfig( + s3Destination(onFailure.bucket(), onFailure.prefix(), onFailure.expectedBucketOwner())) + .build()); + } + return builder.build(); + } + + private static DistributedMapS3DestinationConfig s3Destination( + String bucket, String keyPrefix, String expectedBucketOwner) { + var builder = DistributedMapS3DestinationConfig.builder().bucket(bucket).keyPrefix(keyPrefix); + if (expectedBucketOwner != null) { + builder.expectedBucketOwner(expectedBucketOwner); + } + return builder.build(); + } + + private static software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig completionConfig( + DistributedMapConfig config) { + var completion = config.completionConfig(); + if (completion == null) { + return null; + } + var builder = software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig.builder(); + if (completion.toleratedFailureCount() != null) { + builder.toleratedFailureCount(completion.toleratedFailureCount()); + } + if (completion.toleratedFailurePercentage() != null) { + builder.toleratedFailurePercentage( + completion.toleratedFailurePercentage().floatValue()); + } + if (completion.minimumSampleSize() != null) { + builder.minimumSampleSize(completion.minimumSampleSize()); + } + return builder.build(); + } + + public static DistributedMapSummary toSummary(DistributedMapStatus runStatus, DistributedMapDetails details) { + // The service sends all three counts at every terminal state, and TotalCount once the run succeeded. + // Reading an absent count as zero would report a run that failed items as having none, so a missing + // count is a contract violation rather than a default. + var successCount = requireCount(details.successCount(), "SuccessCount"); + var failureCount = requireCount(details.failureCount(), "FailureCount"); + var unprocessedCount = requireCount(details.unprocessedCount(), "UnprocessedCount"); + if (runStatus == DistributedMapStatus.SUCCEEDED && details.totalCount() == null) { + throw new DurableExecutionException("missing distributed map TotalCount from backend"); + } + return new DistributedMapSummary( + runStatus, + details.sourceExhausted(), + successCount, + failureCount, + unprocessedCount, + details.distributedMapRunArn(), + toError(details.error()), + details.totalCount()); + } + + private static long requireCount(Long count, String field) { + if (count == null) { + throw new DurableExecutionException("missing distributed map " + field + " from backend"); + } + return count; + } + + public static DistributedMapResult toResult( + DistributedMapStatus runStatus, DistributedMapDetails details, TypeToken resultType, SerDes serdes) { + var summary = toSummary(runStatus, details); + var items = new ArrayList>(); + if (details.results() != null) { + for (var wire : details.results()) { + if (wire.itemId() == null || wire.itemId().isEmpty()) { + throw new DurableExecutionException("missing distributed map item id from backend"); + } + var status = wire.statusAsString(); + if (status == null) { + throw new DurableExecutionException("distributed map item " + wire.itemId() + + " has no status, which the service model" + " requires"); + } + if (DistributedMapResult.Item.Status.SUCCEEDED.name().equals(status)) { + O output = wire.output() != null ? serdes.deserialize(wire.output(), resultType) : null; + items.add(DistributedMapResult.Item.succeeded(wire.itemId(), output)); + } else if (DistributedMapResult.Item.Status.FAILED.name().equals(status)) { + items.add(DistributedMapResult.Item.failed(wire.itemId(), toError(wire.error()))); + } else { + // A status added after this SDK version. Keep the item so the other items survive. + items.add(DistributedMapResult.Item.unknown(wire.itemId())); + } + } + } + return new DistributedMapResult<>(summary, items); + } + + private static DistributedMapError toError(ErrorObject error) { + return error == null + ? null + : new DistributedMapError( + error.errorType(), error.errorMessage(), error.errorData(), error.stackTrace()); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java deleted file mode 100644 index efcd1ff9d3..0000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/DistributedMapWire.java +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.operation; - -// Written against DistributedMap* shapes not yet in the generated Lambda client, so this does not compile until they ship. - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import software.amazon.awssdk.services.lambda.model.DistributedMapCsvDelimiter; -import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; -import software.amazon.awssdk.services.lambda.model.DistributedMapCsvHeaderLocation; -import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationInclude; -import software.amazon.awssdk.services.lambda.model.DistributedMapDestinationType; -import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; -import software.amazon.awssdk.services.lambda.model.DistributedMapFunctionResponseType; -import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapOnFailureConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapOnSuccessConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; -import software.amazon.awssdk.services.lambda.model.DistributedMapProcessorConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapResultCollectionMode; -import software.amazon.awssdk.services.lambda.model.DistributedMapS3DestinationConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceTransform; -import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; -import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; -import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; -import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.config.CsvFormat; -import software.amazon.lambda.durable.config.DistributedMapConfig; -import software.amazon.lambda.durable.config.DistributedMapProcessor; -import software.amazon.lambda.durable.config.DistributedMapSource; -import software.amazon.lambda.durable.config.FailureDestination; -import software.amazon.lambda.durable.config.ProcessorRetryConfig; -import software.amazon.lambda.durable.config.SuccessDestination; -import software.amazon.lambda.durable.exception.DurableExecutionException; -import software.amazon.lambda.durable.model.DistributedMapCompletionReason; -import software.amazon.lambda.durable.model.DistributedMapItemError; -import software.amazon.lambda.durable.model.DistributedMapResultItem; -import software.amazon.lambda.durable.model.DistributedMapStatus; -import software.amazon.lambda.durable.model.DistributedMapSummary; -import software.amazon.lambda.durable.model.DistributedMapResult; -import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.SerDes; - -/** Translates distributed map config to the checkpoint options and the operation details to result types. */ -public final class DistributedMapWire { - private static final SerDes JSON = new JacksonSerDes(); - private static final long INLINE_SIZE_LIMIT = 1024L * 1024L; - private static final long READER_STATE_LIMIT = 32L * 1024L; - - private DistributedMapWire() {} - - public static DistributedMapOptions toOptions( - DistributedMapSource source, - DistributedMapProcessor processor, - int maxConcurrency, - DistributedMapConfig config, - SerDes defaultSerDes, - boolean collectResults) { - var builder = DistributedMapOptions.builder() - .maxConcurrency(maxConcurrency) - .source(sourceConfig(source, defaultSerDes)) - .processor(processorConfig(processor)); - if (config.destination() != null) { - var destination = destinationConfig(config.destination()); - if (destination != null) { - builder.destination(destination); - } - } - var completion = completionConfig(config); - if (completion != null) { - builder.completionConfig(completion); - } - if (collectResults) { - builder.resultCollection(DistributedMapResultCollectionConfig.builder() - .mode(DistributedMapResultCollectionMode.INLINE) - .build()); - } - if (config.timeout() != null) { - builder.timeoutSeconds((int) config.timeout().toSeconds()); - } - return builder.build(); - } - - private static DistributedMapSourceConfig sourceConfig(DistributedMapSource source, SerDes defaultSerDes) { - var builder = DistributedMapSourceConfig.builder() - .type(DistributedMapSourceType.fromValue(source.sourceType().name())); - if (source.maxItems() != null) { - builder.maxItemsToRead(source.maxItems().longValue()); - } - switch (source.sourceType()) { - case INLINE -> builder.inlineSourceConfig(inlineSourceConfig(source, defaultSerDes)); - case S3 -> builder.s3SourceConfig(s3SourceConfig(source.s3())); - case READER_FUNCTION -> builder.readerFunctionSourceConfig(readerConfig(source.reader(), defaultSerDes)); - } - return builder.build(); - } - - private static DistributedMapInlineSourceConfig inlineSourceConfig( - DistributedMapSource source, SerDes defaultSerDes) { - var serdes = source.inlineSerDes() != null ? source.inlineSerDes() : defaultSerDes; - var bodies = new ArrayList(source.inlineItems().size()); - for (Object item : source.inlineItems()) { - bodies.add(serdes.serialize(item)); - } - var arrayBytes = JSON.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; - if (arrayBytes > INLINE_SIZE_LIMIT) { - throw new IllegalArgumentException( - "inline source exceeds the 1 MB limit (serialized size: " + arrayBytes + " bytes)"); - } - return DistributedMapInlineSourceConfig.builder().items(bodies).build(); - } - - private static DistributedMapS3SourceConfig s3SourceConfig(DistributedMapSource.S3SourceConfig s3) { - var builder = DistributedMapS3SourceConfig.builder().bucket(s3.bucket()); - if (s3.key() != null) { - builder.key(s3.key()); - } - if (s3.prefix() != null) { - builder.keyPrefix(s3.prefix()); - } - if (s3.transform() != null) { - builder.transform(DistributedMapS3SourceTransform.fromValue(s3.transform().name())); - } - if (s3.format() != null) { - builder.format(DistributedMapSourceFormat.fromValue(s3.format().name())); - } - if (s3.expectedBucketOwner() != null) { - builder.expectedBucketOwner(s3.expectedBucketOwner()); - } - if (s3.csvFormat() != null) { - builder.csvFormatOptions(csvFormatOptions(s3.csvFormat())); - } - return builder.build(); - } - - private static DistributedMapCsvFormatOptions csvFormatOptions(CsvFormat format) { - var builder = DistributedMapCsvFormatOptions.builder() - .headerLocation(DistributedMapCsvHeaderLocation.fromValue(format.headerLocation().name())) - .delimiter(DistributedMapCsvDelimiter.fromValue(format.delimiter().getValue())); - // Column names go on the wire only for headerless files (GIVEN), expectedColumns (FIRST_ROW) is compile-time only. - if (format.headerLocation() == CsvFormat.HeaderLocation.GIVEN && format.columns() != null) { - builder.headers(format.columns()); - } - return builder.build(); - } - - private static DistributedMapReaderFunctionSourceConfig readerConfig( - DistributedMapSource.ReaderSourceConfig reader, SerDes defaultSerDes) { - var builder = DistributedMapReaderFunctionSourceConfig.builder().functionName(reader.functionName()); - if (reader.initialState() != null) { - var serdes = reader.stateSerDes() != null ? reader.stateSerDes() : defaultSerDes; - var state = serdes.serialize(reader.initialState()); - if (state != null && state.getBytes(StandardCharsets.UTF_8).length > READER_STATE_LIMIT) { - throw new IllegalArgumentException("reader initialState exceeds the 32 KB limit"); - } - builder.initialState(state); - } - return builder.build(); - } - - private static DistributedMapProcessorConfig processorConfig(DistributedMapProcessor processor) { - var builder = DistributedMapProcessorConfig.builder().functionName(processor.functionName()); - if (processor.durableExecutionNamePrefix() != null) { - builder.durableExecutionNamePrefix(processor.durableExecutionNamePrefix()); - } - var responseType = processor.responseMode().getValue(); - if (responseType != null) { - builder.functionResponseTypes(List.of(DistributedMapFunctionResponseType.fromValue(responseType))); - } - if (processor.batchSize() != null) { - builder.batchSize(processor.batchSize()); - } - ProcessorRetryConfig retry = processor.retryConfig(); - if (retry != null) { - if (retry.maxRetryAttempts() != null) { - builder.maxRetryAttempts(retry.maxRetryAttempts()); - } - if (retry.maxRetryDuration() != null) { - builder.maxRetryDurationSeconds((int) retry.maxRetryDuration().toSeconds()); - } - } - return builder.build(); - } - - private static software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig destinationConfig( - software.amazon.lambda.durable.config.DistributedMapDestinationConfig destination) { - if (destination.onSuccess() == null && destination.onFailure() == null) { - return null; // omit an empty destination config from the wire - } - var builder = software.amazon.awssdk.services.lambda.model.DistributedMapDestinationConfig.builder(); - SuccessDestination onSuccess = destination.onSuccess(); - if (onSuccess != null) { - var include = new ArrayList(); - if (onSuccess.includeInput()) { - include.add(DistributedMapDestinationInclude.INPUT); - } - if (onSuccess.includeOutput()) { - include.add(DistributedMapDestinationInclude.OUTPUT); - } - builder.onSuccess(DistributedMapOnSuccessConfig.builder() - .type(DistributedMapDestinationType.S3) - .include(include) - .s3DestinationConfig(s3Destination( - onSuccess.bucket(), onSuccess.prefix(), onSuccess.expectedBucketOwner())) - .build()); - } - FailureDestination onFailure = destination.onFailure(); - if (onFailure != null) { - var include = new ArrayList(); - if (onFailure.includeInput()) { - include.add(DistributedMapDestinationInclude.INPUT); - } - if (onFailure.includeError()) { - include.add(DistributedMapDestinationInclude.ERROR); - } - builder.onFailure(DistributedMapOnFailureConfig.builder() - .type(DistributedMapDestinationType.S3) - .include(include) - .s3DestinationConfig(s3Destination( - onFailure.bucket(), onFailure.prefix(), onFailure.expectedBucketOwner())) - .build()); - } - return builder.build(); - } - - private static DistributedMapS3DestinationConfig s3Destination( - String bucket, String keyPrefix, String expectedBucketOwner) { - var builder = DistributedMapS3DestinationConfig.builder().bucket(bucket).keyPrefix(keyPrefix); - if (expectedBucketOwner != null) { - builder.expectedBucketOwner(expectedBucketOwner); - } - return builder.build(); - } - - private static software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig completionConfig( - DistributedMapConfig config) { - var completion = config.completionConfig(); - if (completion == null - || (completion.toleratedFailureCount() == null - && completion.toleratedFailurePercentage() == null - && completion.minimumSampleSize() == null)) { - return null; // omit an empty completion config from the wire - } - var builder = software.amazon.awssdk.services.lambda.model.DistributedMapCompletionConfig.builder(); - if (completion.toleratedFailureCount() != null) { - builder.toleratedFailureCount(completion.toleratedFailureCount()); - } - if (completion.toleratedFailurePercentage() != null) { - builder.toleratedFailurePercentage(completion.toleratedFailurePercentage().floatValue()); - } - if (completion.minimumSampleSize() != null) { - builder.minimumSampleSize(completion.minimumSampleSize()); - } - return builder.build(); - } - - public static DistributedMapSummary toSummary(DistributedMapDetails details) { - return new DistributedMapSummary( - parseStatus(details), - parseReason(details), - details.successCount(), - details.failureCount(), - details.unprocessedCount(), - details.distributedMapRunArn(), - details.completionDetails(), - details.totalCount()); - } - - public static DistributedMapResult toResult( - DistributedMapDetails details, TypeToken resultType, SerDes serdes) { - var summary = toSummary(details); - var items = new ArrayList>(); - if (details.results() != null) { - for (var wire : details.results()) { - var status = wire.statusAsString(); - if (DistributedMapResultItem.Status.SUCCEEDED.name().equals(status)) { - O output = wire.output() != null ? serdes.deserialize(wire.output(), resultType) : null; - items.add(DistributedMapResultItem.succeeded(wire.itemId(), output)); - } else if (DistributedMapResultItem.Status.FAILED.name().equals(status)) { - var error = wire.error() != null - ? new DistributedMapItemError( - wire.error().errorType() != null ? wire.error().errorType() : "", - wire.error().errorMessage() != null ? wire.error().errorMessage() : "") - : null; - items.add(DistributedMapResultItem.failed(wire.itemId(), error)); - } else { - throw new DurableExecutionException("unrecognized distributed map item status from the backend (" - + status + ") for item " + wire.itemId()); - } - } - } - return new DistributedMapResult<>(summary, items); - } - - private static DistributedMapStatus parseStatus(DistributedMapDetails details) { - if (details.statusAsString() == null) { - throw new DurableExecutionException("distributed map details missing the required Status field"); - } - try { - return DistributedMapStatus.fromValue(details.statusAsString()); - } catch (IllegalArgumentException e) { - throw new DurableExecutionException( - "unrecognized distributed map status from the backend (" + details.statusAsString() + ")", e); - } - } - - private static DistributedMapCompletionReason parseReason(DistributedMapDetails details) { - if (details.completionReasonAsString() == null) { - throw new DurableExecutionException("distributed map details missing the required CompletionReason field"); - } - return DistributedMapCompletionReason.fromValue(details.completionReasonAsString()); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java index f5dbba1124..92608e682e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/util/DistributedMapValidation.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.util; +import java.nio.charset.StandardCharsets; import java.util.HashSet; import java.util.List; import java.util.regex.Pattern; @@ -11,6 +12,9 @@ public final class DistributedMapValidation { private static final Pattern ACCOUNT_ID = Pattern.compile("\\d{12}"); private static final String S3_SCHEME = "s3://"; private static final int MAX_FUNCTION_NAME_LENGTH = 170; + private static final int MAX_CSV_COLUMNS = 1000; + private static final int MAX_CSV_COLUMN_LENGTH = 256; + private static final int MAX_DESTINATION_KEY_PREFIX_BYTES = 900; private DistributedMapValidation() {} @@ -32,6 +36,23 @@ public static void validateBucketOwner(String expectedBucketOwner) { } } + /** + * Rejects a destination key prefix the service would reject. The limit is 900 UTF-8 bytes, including a trailing + * slash when the URI has one, and it is measured in bytes rather than characters, so a multi-byte prefix under 900 + * characters can still exceed it. The service applies the same limit when the start checkpoint is written, so + * checking it here fails the call that built the destination instead of the operation. + */ + public static void validateDestinationPrefix(String prefix) { + if (prefix == null) { + return; + } + var bytes = prefix.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_DESTINATION_KEY_PREFIX_BYTES) { + throw new IllegalArgumentException("destination key prefix must be at most " + + MAX_DESTINATION_KEY_PREFIX_BYTES + " UTF-8 bytes, got: " + bytes); + } + } + public static void validateColumns(String parameterName, List columns) { if (columns == null) { return; @@ -39,6 +60,16 @@ public static void validateColumns(String parameterName, List columns) { if (columns.isEmpty()) { throw new IllegalArgumentException(parameterName + " cannot be empty"); } + if (columns.size() > MAX_CSV_COLUMNS) { + throw new IllegalArgumentException( + parameterName + " must hold at most " + MAX_CSV_COLUMNS + " names, got: " + columns.size()); + } + for (var column : columns) { + if (column != null && column.length() > MAX_CSV_COLUMN_LENGTH) { + throw new IllegalArgumentException("each " + parameterName + " name must be at most " + + MAX_CSV_COLUMN_LENGTH + " characters, got: " + column.length()); + } + } if (new HashSet<>(columns).size() != columns.size()) { throw new IllegalArgumentException(parameterName + " must not contain duplicates"); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java new file mode 100644 index 0000000000..71003c3544 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapConfigTest.java @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class DistributedMapConfigTest { + + @Test + void failureCount_setsCount() { + var config = DistributedMapConfig.CompletionConfig.failureCount(3); + + assertEquals(3, config.toleratedFailureCount()); + assertNull(config.toleratedFailurePercentage()); + assertNull(config.minimumSampleSize()); + } + + @Test + void failureCount_withZero_shouldPass() { + assertEquals(0, DistributedMapConfig.CompletionConfig.failureCount(0).toleratedFailureCount()); + } + + @Test + void failurePercentage_setsPercentage() { + var config = DistributedMapConfig.CompletionConfig.failurePercentage(25); + + assertNull(config.toleratedFailureCount()); + assertEquals(25, config.toleratedFailurePercentage()); + assertNull(config.minimumSampleSize()); + } + + @Test + void failurePercentage_withMinimumSampleSize_setsBoth() { + var config = DistributedMapConfig.CompletionConfig.failurePercentage(25, 10); + + assertEquals(25, config.toleratedFailurePercentage()); + assertEquals(10, config.minimumSampleSize()); + } + + @Test + void failurePercentage_atBoundaries_shouldPass() { + assertEquals( + 0, DistributedMapConfig.CompletionConfig.failurePercentage(0).toleratedFailurePercentage()); + assertEquals( + 100, + DistributedMapConfig.CompletionConfig.failurePercentage(100).toleratedFailurePercentage()); + } + + @Test + void countAndPercentage_mutuallyExclusive_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(3, 25, null)); + assertEquals( + "toleratedFailureCount and toleratedFailurePercentage are mutually exclusive", exception.getMessage()); + } + + @Test + void minimumSampleSize_withoutPercentage_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(3, null, 5)); + assertEquals("minimumSampleSize is only valid with toleratedFailurePercentage", exception.getMessage()); + } + + @Test + void completionConfig_onlyMinimumSampleSize_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(null, null, 5)); + assertEquals("minimumSampleSize is only valid with toleratedFailurePercentage", exception.getMessage()); + } + + @Test + void failureCount_negative_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failureCount(-1)); + assertEquals("toleratedFailureCount must be non-negative, got: -1", exception.getMessage()); + } + + @Test + void failurePercentage_negative_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(-1)); + assertEquals("toleratedFailurePercentage must be between 0 and 100, got: -1", exception.getMessage()); + } + + @Test + void failurePercentage_aboveHundred_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(101)); + assertEquals("toleratedFailurePercentage must be between 0 and 100, got: 101", exception.getMessage()); + } + + @Test + void minimumSampleSize_belowOne_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapConfig.CompletionConfig.failurePercentage(25, 0)); + assertEquals("minimumSampleSize must be at least 1, got: 0", exception.getMessage()); + } + + @Test + void emptyCompletionConfig_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapConfig.CompletionConfig(null, null, null)); + assertEquals( + "a completion config must set toleratedFailureCount or toleratedFailurePercentage", + exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java new file mode 100644 index 0000000000..d2680e7c79 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapDestinationTest.java @@ -0,0 +1,216 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; + +class DistributedMapDestinationTest { + + @Test + void successes_defaultsIncludeOutputOnly() { + var destination = DistributedMapDestination.s3Successes("s3://bucket/results"); + + assertEquals("bucket", destination.bucket()); + assertEquals("results", destination.prefix()); + assertFalse(destination.includeInput()); + assertTrue(destination.includeOutput()); + assertNull(destination.expectedBucketOwner()); + } + + @Test + void successes_withoutPath_prefixIsEmpty() { + var destination = DistributedMapDestination.s3Successes("s3://bucket"); + + assertEquals("bucket", destination.bucket()); + assertEquals("", destination.prefix()); + } + + @Test + void successes_withAllArguments() { + var destination = DistributedMapDestination.s3Successes("s3://bucket/out", true, true, "123456789012"); + + assertEquals("bucket", destination.bucket()); + assertEquals("out", destination.prefix()); + assertTrue(destination.includeInput()); + assertTrue(destination.includeOutput()); + assertEquals("123456789012", destination.expectedBucketOwner()); + } + + @Test + void successes_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/out", false, false, null)); + assertEquals("success destination must include at least one of input or output", exception.getMessage()); + } + + @Test + void successDestination_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Success("bucket", "", false, false, null)); + assertEquals("success destination must include at least one of input or output", exception.getMessage()); + } + + @ParameterizedTest + @NullAndEmptySource + void successDestination_nullOrEmptyBucket_shouldThrow(String bucket) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Success(bucket, "", false, true, null)); + assertEquals("success destination bucket cannot be empty", exception.getMessage()); + } + + @Test + void failures_defaultsIncludeInputAndError() { + var destination = DistributedMapDestination.s3Failures("s3://bucket/errors"); + + assertEquals("bucket", destination.bucket()); + assertEquals("errors", destination.prefix()); + assertTrue(destination.includeInput()); + assertTrue(destination.includeError()); + assertNull(destination.expectedBucketOwner()); + } + + @Test + void failures_withAllArguments() { + var destination = DistributedMapDestination.s3Failures("s3://bucket/err", true, false, "123456789012"); + + assertEquals("bucket", destination.bucket()); + assertEquals("err", destination.prefix()); + assertTrue(destination.includeInput()); + assertFalse(destination.includeError()); + assertEquals("123456789012", destination.expectedBucketOwner()); + } + + @Test + void failures_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Failures("s3://bucket/err", false, false, null)); + assertEquals("failure destination must include at least one of input or error", exception.getMessage()); + } + + @Test + void failureDestination_allIncludeOff_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Failure("bucket", "", false, false, null)); + assertEquals("failure destination must include at least one of input or error", exception.getMessage()); + } + + @ParameterizedTest + @NullAndEmptySource + void failureDestination_nullOrEmptyBucket_shouldThrow(String bucket) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapDestination.Failure(bucket, "", true, true, null)); + assertEquals("failure destination bucket cannot be empty", exception.getMessage()); + } + + @Test + void successes_invalidBucketOwner_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/out", false, true, "123")); + assertEquals("expectedBucketOwner must be a 12-digit account id, got: 123", exception.getMessage()); + } + + @Test + void successes_invalidUri_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapDestination.s3Successes("bucket/out")); + assertEquals("S3 URI must start with s3://, got: bucket/out", exception.getMessage()); + } + + @Test + void destinationConfig_onSuccess_routesSuccessOnly() { + var success = DistributedMapDestination.s3Successes("s3://bucket/ok"); + var config = DistributedMapDestination.of(success); + + assertSame(success, config.onSuccess()); + assertNull(config.onFailure()); + } + + @Test + void destinationConfig_onFailure_routesFailureOnly() { + var failure = DistributedMapDestination.s3Failures("s3://bucket/bad"); + var config = DistributedMapDestination.of(failure); + + assertNull(config.onSuccess()); + assertSame(failure, config.onFailure()); + } + + @Test + void destinationConfig_of_routesBoth() { + var success = DistributedMapDestination.s3Successes("s3://bucket/ok"); + var failure = DistributedMapDestination.s3Failures("s3://bucket/bad"); + var config = DistributedMapDestination.of(success, failure); + + assertSame(success, config.onSuccess()); + assertSame(failure, config.onFailure()); + } + + // ------------------------------------------------------------------ + // key prefix length + // ------------------------------------------------------------------ + + private static String repeat(String unit, int times) { + return unit.repeat(times); + } + + @Test + void successes_prefixAtTheByteLimit_isAccepted() { + var prefix = repeat("a", 900); + + assertEquals( + prefix, + DistributedMapDestination.s3Successes("s3://bucket/" + prefix).prefix()); + } + + @Test + void successes_prefixOverTheByteLimit_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + repeat("a", 901))); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } + + @Test + void failures_prefixOverTheByteLimit_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Failures("s3://bucket/" + repeat("a", 901))); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } + + @Test + void successes_prefixCountsUtf8BytesNotCharacters() { + // This character is two UTF-8 bytes, so 450 of them are exactly 900 bytes while being only 450 characters. + var atTheLimit = repeat("\u00e9", 450); + assertEquals(450, atTheLimit.length()); + assertEquals( + atTheLimit, + DistributedMapDestination.s3Successes("s3://bucket/" + atTheLimit) + .prefix()); + + var overTheLimit = atTheLimit + "\u00e9"; + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + overTheLimit)); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 902", exception.getMessage()); + } + + @Test + void successes_prefixLengthCountsTheTrailingSlash() { + // 900 characters plus the trailing slash is 901 bytes, so the slash is what pushes it over. + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapDestination.s3Successes("s3://bucket/" + repeat("a", 900) + "/")); + assertEquals("destination key prefix must be at most 900 UTF-8 bytes, got: 901", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java new file mode 100644 index 0000000000..67a8481e67 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorRetryConfigTest.java @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class DistributedMapProcessorRetryConfigTest { + + @Test + void build_defaultsAreNull() { + var retry = DistributedMapProcessorRetryConfig.builder().build(); + + assertNull(retry.maxRetryAttempts()); + assertNull(retry.maxRetryDuration()); + } + + @Test + void maxRetryAttempts_zero_isStored() { + var retry = + DistributedMapProcessorRetryConfig.builder().maxRetryAttempts(0).build(); + + assertEquals(0, retry.maxRetryAttempts()); + } + + @Test + void maxRetryAttempts_unlimited_isStored() { + var retry = DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .build(); + + assertEquals(-1, retry.maxRetryAttempts()); + } + + @Test + void maxRetryAttempts_otherNegative_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(-2)); + assertEquals( + "maxRetryAttempts must be non-negative or DistributedMapProcessorRetryConfig.UNLIMITED, got: -2", + exception.getMessage()); + } + + @Test + void maxRetryDuration_atBoundaries_shouldPass() { + assertEquals( + Duration.ofMinutes(1), + DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(1)) + .build() + .maxRetryDuration()); + assertEquals( + Duration.ofHours(6), + DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofHours(6)) + .build() + .maxRetryDuration()); + } + + @Test + void maxRetryDuration_belowMinimum_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofSeconds(59))); + assertEquals("maxRetryDuration must be between 1 minute and 6 hours, got: 59s", exception.getMessage()); + } + + @Test + void maxRetryDuration_aboveMaximum_shouldThrow() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofHours(7))); + assertEquals("maxRetryDuration must be between 1 minute and 6 hours, got: 25200s", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java new file mode 100644 index 0000000000..3a84e923f5 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapProcessorTest.java @@ -0,0 +1,176 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Duration; +import org.junit.jupiter.api.Test; + +class DistributedMapProcessorTest { + + @Test + void batch_setsBatchMode() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn").build(); + + assertEquals("fn", processor.functionName()); + assertEquals(DistributedMapProcessor.ResponseMode.BATCH, processor.responseMode()); + } + + @Test + void itemOutcomes_setsItemOutcomesMode() { + var processor = DistributedMapProcessor.reportItemOutcomes("fn").build(); + + assertEquals(DistributedMapProcessor.ResponseMode.ITEM_OUTCOMES, processor.responseMode()); + } + + @Test + void build_defaultsAreNull() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn").build(); + + assertNull(processor.batchSize()); + assertNull(processor.retry()); + assertNull(processor.durableExecutionNamePrefix()); + } + + @Test + void batchSize_atBoundaries_shouldPass() { + assertEquals( + 1, + DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(1) + .build() + .batchSize()); + assertEquals( + 10000, + DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(10000) + .build() + .batchSize()); + } + + @Test + void batchSize_belowMinimum_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(0) + .build()); + assertEquals("batchSize must be between 1 and 10000, got: 0", exception.getMessage()); + } + + @Test + void batchSize_aboveMaximum_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .batchSize(10001) + .build()); + assertEquals("batchSize must be between 1 and 10000, got: 10001", exception.getMessage()); + } + + @Test + void durableExecutionNamePrefix_atBoundaries_shouldPass() { + assertEquals( + "a", + DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("a") + .build() + .durableExecutionNamePrefix()); + var thirtySix = "a".repeat(36); + assertEquals( + thirtySix, + DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix(thirtySix) + .build() + .durableExecutionNamePrefix()); + } + + @Test + void durableExecutionNamePrefix_empty_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("") + .build()); + assertEquals("durableExecutionNamePrefix must be between 1 and 36 characters, got: 0", exception.getMessage()); + } + + @Test + void durableExecutionNamePrefix_tooLong_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("fn") + .durableExecutionNamePrefix("a".repeat(37)) + .build()); + assertEquals("durableExecutionNamePrefix must be between 1 and 36 characters, got: 37", exception.getMessage()); + } + + @Test + void retryFields_areStored() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(3) + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + + assertEquals(3, processor.retry().maxRetryAttempts()); + assertEquals(Duration.ofMinutes(5), processor.retry().maxRetryDuration()); + } + + @Test + void unlimitedRetryAttempts_isStored() { + var processor = DistributedMapProcessor.reportBatchOutcome("fn") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .build()) + .build(); + + assertEquals( + DistributedMapProcessorRetryConfig.UNLIMITED, processor.retry().maxRetryAttempts()); + } + + @Test + void functionName_empty_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("") + .build()); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void functionName_null_shouldThrow() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome(null) + .build()); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void functionName_tooLong_shouldThrow() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapProcessor.reportBatchOutcome("a".repeat(171)) + .build()); + assertEquals("function name must be at most 170 characters, got: 171", exception.getMessage()); + } + + @Test + void functionName_atMaxLength_shouldPass() { + var name = "a".repeat(170); + assertEquals( + name, DistributedMapProcessor.reportBatchOutcome(name).build().functionName()); + } + + @Test + void build_allFieldsSet() { + var processor = DistributedMapProcessor.reportItemOutcomes("fn") + .batchSize(50) + .durableExecutionNamePrefix("job") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + + assertEquals("fn", processor.functionName()); + assertEquals(50, processor.batchSize()); + assertEquals("job", processor.durableExecutionNamePrefix()); + assertEquals(Duration.ofMinutes(5), processor.retry().maxRetryDuration()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java new file mode 100644 index 0000000000..75d4d5789a --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/config/DistributedMapSourceTest.java @@ -0,0 +1,415 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.config; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +class DistributedMapSourceTest { + + @Test + void inline_storesItems() { + var source = DistributedMapSource.inline(List.of("a", "b")); + + assertEquals(DistributedMapSource.SourceType.INLINE, source.sourceType()); + assertEquals(List.of("a", "b"), source.inlineItems()); + assertNull(source.inlineSerDes()); + assertNull(source.s3()); + assertNull(source.reader()); + } + + @Test + void inline_withSerDes_storesSerDes() { + var serDes = new JacksonSerDes(); + + var source = DistributedMapSource.inline(List.of("a"), serDes); + + assertEquals(DistributedMapSource.SourceType.INLINE, source.sourceType()); + assertEquals(List.of("a"), source.inlineItems()); + assertSame(serDes, source.inlineSerDes()); + } + + @Test + void inline_acceptsNullItem() { + var source = DistributedMapSource.inline(java.util.Arrays.asList("a", null)); + assertEquals(java.util.Arrays.asList("a", null), source.inlineItems()); + } + + @Test + void inline_null_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.inline(null)); + assertEquals("items cannot be null", exception.getMessage()); + } + + @Test + void inline_rejectsUnorderedSet() { + var unordered = new HashSet<>(List.of("a", "b")); + + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.inline(unordered)); + assertEquals("items must have deterministic iteration order", exception.getMessage()); + } + + @Test + void inline_allowsLinkedHashSet() { + var ordered = new LinkedHashSet<>(List.of("a", "b")); + + var source = DistributedMapSource.inline(ordered); + + assertEquals(List.of("a", "b"), source.inlineItems()); + } + + @Test + void s3JsonLines_setsBucketKeyFormat() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + assertEquals(DistributedMapSource.SourceType.S3, source.sourceType()); + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.jsonl", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.JSON_LINES, s3.objectConfig().format()); + assertNull(s3.listedObjectMetadataConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3JsonLines_withoutKey_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3JsonLines("s3://bucket-only")); + assertEquals("s3JsonLines requires an S3 object key", exception.getMessage()); + } + + @Test + void s3JsonArray_setsFormat() { + DistributedMapSource source = DistributedMapSource.s3JsonArray("s3://bucket/data.json"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.json", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.JSON_ARRAY, s3.objectConfig().format()); + } + + @Test + void s3JsonArray_withoutKey_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3JsonArray("s3://bucket-only")); + assertEquals("s3JsonArray requires an S3 object key", exception.getMessage()); + } + + @Test + void s3Csv_setsFormatAndCsvFormat() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")); + + DistributedMapSource source = DistributedMapSource.s3Csv("s3://bucket/data.csv", format); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.OBJECT, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data.csv", s3.objectConfig().key()); + assertEquals(DistributedMapSource.Format.CSV, s3.objectConfig().format()); + assertSame(format, s3.objectConfig().csvFormat()); + } + + @Test + void s3Csv_withoutKey_throws() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.s3Csv("s3://bucket-only", format)); + assertEquals("s3Csv requires an S3 object key", exception.getMessage()); + } + + @Test + void s3Objects_setsListedObjectMetadataWithPrefix() { + DistributedMapSource source = DistributedMapSource.s3Objects("s3://bucket/data/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECT_METADATA, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("data/", s3.listedObjectMetadataConfig().keyPrefix()); + assertNull(s3.objectConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3Objects_wholeBucket_allowsEmptyPrefix() { + DistributedMapSource source = DistributedMapSource.s3Objects("s3://bucket"); + + var s3 = source.s3(); + assertEquals("bucket", s3.bucket()); + assertEquals("", s3.listedObjectMetadataConfig().keyPrefix()); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECT_METADATA, s3.type()); + } + + @Test + void s3FlattenedJsonLines_setsListedObjects() { + DistributedMapSource source = DistributedMapSource.s3FlattenedJsonLines("s3://bucket/prefix/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals("bucket", s3.bucket()); + assertEquals("prefix/", s3.listedObjectsConfig().keyPrefix()); + assertEquals( + DistributedMapSource.Format.JSON_LINES, s3.listedObjectsConfig().format()); + assertNull(s3.objectConfig()); + } + + @Test + void s3FlattenedJsonArray_setsListedObjects() { + DistributedMapSource source = DistributedMapSource.s3FlattenedJsonArray("s3://bucket/prefix/"); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals( + DistributedMapSource.Format.JSON_ARRAY, s3.listedObjectsConfig().format()); + } + + @Test + void s3FlattenedCsv_setsListedObjects() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + DistributedMapSource source = DistributedMapSource.s3FlattenedCsv("s3://bucket/prefix/", format); + + var s3 = source.s3(); + assertEquals(DistributedMapSource.S3SourceType.LISTED_OBJECTS, s3.type()); + assertEquals(DistributedMapSource.Format.CSV, s3.listedObjectsConfig().format()); + assertSame(format, s3.listedObjectsConfig().csvFormat()); + } + + @Test + void s3FlattenedCsv_wholeBucket_allowsEmptyPrefix() { + DistributedMapSource source = + DistributedMapSource.s3FlattenedCsv("s3://bucket", DistributedMapSource.CsvFormat.firstRow()); + + assertEquals("", source.s3().listedObjectsConfig().keyPrefix()); + } + + @Test + void reader_storesFunctionName() { + DistributedMapSource source = DistributedMapSource.reader("my-reader"); + + assertEquals(DistributedMapSource.SourceType.READER_FUNCTION, source.sourceType()); + assertEquals("my-reader", source.reader().functionName()); + assertNull(source.reader().initialState()); + assertNull(source.reader().stateSerDes()); + } + + @Test + void reader_withInitialState_storesState() { + var serDes = new JacksonSerDes(); + + DistributedMapSource source = DistributedMapSource.reader("my-reader", "start-token", serDes); + + assertEquals("my-reader", source.reader().functionName()); + assertEquals("start-token", source.reader().initialState()); + assertSame(serDes, source.reader().stateSerDes()); + } + + @Test + void reader_emptyFunctionName_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.reader("")); + assertEquals("function name cannot be empty", exception.getMessage()); + } + + @Test + void maxItems_setsMaxItems() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .maxItems(500); + + assertEquals(500L, source.maxItems()); + } + + @Test + void maxItems_zero_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.maxItems(0)); + assertEquals("maxItems must be between 1 and 100000000, got: 0", exception.getMessage()); + } + + @Test + void maxItems_atTheLimit_stores() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .maxItems(100_000_000L); + + assertEquals(100_000_000L, source.maxItems()); + } + + @Test + void maxItems_aboveTheLimit_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.maxItems(100_000_001L)); + assertEquals("maxItems must be between 1 and 100000000, got: 100000001", exception.getMessage()); + } + + @Test + void expectedBucketOwner_twelveDigits_stores() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl") + .expectedBucketOwner("123456789012"); + + assertEquals("123456789012", source.s3().expectedBucketOwner()); + } + + @Test + void expectedBucketOwner_invalid_throws() { + DistributedMapSource source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl"); + + var exception = assertThrows(IllegalArgumentException.class, () -> source.expectedBucketOwner("123")); + assertEquals("expectedBucketOwner must be a 12-digit account id, got: 123", exception.getMessage()); + } + + @Test + void expectedBucketOwner_onNonS3Source_throws() { + var source = DistributedMapSource.inline(List.of("a")); + + var exception = assertThrows(IllegalStateException.class, () -> source.expectedBucketOwner("123456789012")); + assertEquals("expectedBucketOwner only applies to S3 sources", exception.getMessage()); + } + + @Test + void s3JsonLines_missingScheme_throws() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.s3JsonLines("bucket/key")); + assertEquals("S3 URI must start with s3://, got: bucket/key", exception.getMessage()); + } + + @Test + void s3Objects_missingBucket_throws() { + var exception = assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.s3Objects("s3:///key")); + assertEquals("S3 URI must contain a bucket, got: s3:///key", exception.getMessage()); + } + + @Test + void expectedColumns_setsFirstRowAndColumns() { + var format = DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "b")); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void headers_setsGivenAndColumns() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void firstRow_hasNoColumns() { + var format = DistributedMapSource.CsvFormat.firstRow(); + + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, format.headerLocation()); + assertNull(format.columns()); + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void withDelimiter_setsDelimiter() { + var format = DistributedMapSource.CsvFormat.headers(List.of("a", "b")) + .withDelimiter(DistributedMapSource.CsvDelimiter.PIPE); + + assertEquals(DistributedMapSource.CsvDelimiter.PIPE, format.delimiter()); + assertEquals(DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, format.headerLocation()); + assertEquals(List.of("a", "b"), format.columns()); + } + + @Test + void constructor_nullDelimiter_defaultsToComma() { + var format = new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, List.of("a"), null); + + assertEquals(DistributedMapSource.CsvDelimiter.COMMA, format.delimiter()); + } + + @Test + void expectedColumns_empty_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.expectedColumns(List.of())); + assertEquals("expectedColumns cannot be empty", exception.getMessage()); + } + + @Test + void expectedColumns_aboveTheColumnLimit_throws() { + var columns = + java.util.stream.IntStream.range(0, 1001).mapToObj(i -> "c" + i).toList(); + + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.expectedColumns(columns)); + assertEquals("expectedColumns must hold at most 1000 names, got: 1001", exception.getMessage()); + } + + @Test + void headers_nameAboveTheLengthLimit_throws() { + var columns = List.of("a".repeat(257)); + + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(columns)); + assertEquals("each headers name must be at most 256 characters, got: 257", exception.getMessage()); + } + + @Test + void expectedColumns_duplicates_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "a"))); + assertEquals("expectedColumns must not contain duplicates", exception.getMessage()); + } + + @Test + void headers_empty_throws() { + var exception = + assertThrows(IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(List.of())); + assertEquals("headers cannot be empty", exception.getMessage()); + } + + @Test + void headers_duplicates_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> DistributedMapSource.CsvFormat.headers(List.of("a", "a"))); + assertEquals("headers must not contain duplicates", exception.getMessage()); + } + + @Test + void constructor_givenWithoutColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, null, null)); + assertEquals("CsvFormat with headerLocation GIVEN requires headers", exception.getMessage()); + } + + @Test + void constructor_givenWithDuplicateColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.GIVEN, List.of("a", "a"), null)); + assertEquals("headers must not contain duplicates", exception.getMessage()); + } + + @Test + void constructor_firstRowWithEmptyColumns_throws() { + var exception = assertThrows( + IllegalArgumentException.class, + () -> new DistributedMapSource.CsvFormat( + DistributedMapSource.CsvFormat.HeaderLocation.FIRST_ROW, List.of(), null)); + assertEquals("expectedColumns cannot be empty", exception.getMessage()); + } + + @Test + void csvFormat_withoutHeaderLocation_throws() { + var exception = assertThrows( + IllegalArgumentException.class, () -> new DistributedMapSource.CsvFormat(null, null, null)); + assertEquals("CsvFormat requires a headerLocation", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java b/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java new file mode 100644 index 0000000000..dea8382a63 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/context/DurableContextDistributedMapTest.java @@ -0,0 +1,194 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.context; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.serde.JacksonSerDes; + +/** Exercises the ctx.distributedMap entrypoint overloads on DurableContextImpl. */ +class DurableContextDistributedMapTest { + + private static final String EXECUTION_NAME = "349beff4-a89d-4bc8-a56f-af7a8af67a5f"; + private static final String EXECUTION_OP_ID = "20dae574-53da-37a1-bfd5-b0e2e6ec715d"; + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test/durable-execution/" + + EXECUTION_NAME + "/" + EXECUTION_OP_ID; + + private static final Operation EXECUTION_OP = Operation.builder() + .id(EXECUTION_OP_ID) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .build(); + + // The first operation created inside a context always hashes the counter value "1". + private static final String FIRST_OP_ID = TestUtils.hashOperationId("1"); + + private DurableContext createTestContext() { + return createTestContext(List.of()); + } + + private DurableContext createTestContext(List initialOperations) { + var client = TestUtils.createMockClient(); + var operations = new ArrayList<>(List.of(EXECUTION_OP)); + operations.addAll(initialOperations); + var initialExecutionState = + CheckpointUpdatedExecutionState.builder().operations(operations).build(); + var executionManager = new ExecutionManager( + new DurableExecutionInput(EXECUTION_ARN, "test-token", initialExecutionState), + DurableConfig.builder().withDurableExecutionClient(client).build(), + null); + var root = DurableContextImpl.createRootContext( + executionManager, + DurableConfig.builder().withDurableExecutionClient(client).build(), + null); + executionManager.registerActiveThread(null); + executionManager.setCurrentThreadContext(new ThreadContext(null, ThreadType.CONTEXT)); + return root; + } + + private DistributedMapSource source() { + return DistributedMapSource.inline(List.of("a")); + } + + private DistributedMapProcessor processor() { + return DistributedMapProcessor.reportBatchOutcome("test-processor").build(); + } + + // A completed distributed map operation so a replay resolves without suspending. + private Operation completedDmapOp() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L) + .build(); + return Operation.builder() + .id(FIRST_OP_ID) + .status(OperationStatus.SUCCEEDED) + .distributedMapDetails(details) + .build(); + } + + @Test + void testMaxConcurrencyZeroThrows() { + var context = createTestContext(); + + var ex = assertThrows( + IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), 0)); + assertTrue(ex.getMessage().contains("maxConcurrency must be greater than zero")); + } + + @Test + void testMaxConcurrencyNegativeThrows() { + var context = createTestContext(); + + assertThrows(IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), -1)); + } + + @Test + void testResultSerDesOnSummaryCallThrows() { + var context = createTestContext(); + // resultSerDes is only valid on a result-collecting overload, a summary call must reject it. + var config = + DistributedMapConfig.builder().resultSerDes(new JacksonSerDes()).build(); + + var ex = assertThrows( + IllegalArgumentException.class, () -> context.distributedMap("m", source(), processor(), 2, config)); + assertTrue(ex.getMessage().contains("resultSerDes")); + } + + @Test + void testSummaryOverloadReturnsSummary() { + var context = createTestContext(List.of(completedDmapOp())); + + var summary = context.distributedMap("m", source(), processor(), 2); + + assertNotNull(summary); + assertEquals(1, summary.successCount()); + } + + @Test + void testResultOverloadReturnsResult() { + var context = createTestContext(List.of(completedDmapOp())); + + DistributedMapResult result = context.distributedMap( + "m", + source(), + processor(), + 2, + String.class, + DistributedMapConfig.builder().build()); + + assertNotNull(result); + assertNotNull(result.items()); + assertEquals(1, result.summary().successCount()); + } + + @Test + void testClassOverloadDelegatesToTypeToken() { + // The Class overload forwards to the TypeToken overload, so both must resolve identically. + var classContext = createTestContext(List.of(completedDmapOp())); + DistributedMapResult fromClass = classContext.distributedMap( + "m", + source(), + processor(), + 2, + String.class, + DistributedMapConfig.builder().build()); + + var tokenContext = createTestContext(List.of(completedDmapOp())); + DistributedMapResult fromToken = tokenContext.distributedMap( + "m", + source(), + processor(), + 2, + new TypeToken() {}, + DistributedMapConfig.builder().build()); + + assertEquals(fromToken.summary(), fromClass.summary()); + assertEquals(fromToken.items(), fromClass.items()); + } + + @Test + void testSummaryConfigOptional() { + var context = createTestContext(List.of(completedDmapOp())); + + // The no-config summary overload uses an empty config internally. + DistributedMapSummary summary = context.distributedMap("m", source(), processor(), 2); + + assertNotNull(summary); + } + + @Test + void testResultConfigOptional() { + var context = createTestContext(List.of(completedDmapOp())); + + // The no-config result overload uses an empty config internally. + DistributedMapResult result = context.distributedMap("m", source(), processor(), 2, String.class); + + assertNotNull(result); + assertNotNull(result.items()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java new file mode 100644 index 0000000000..f93b8ed64a --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapDurableHandlersTest.java @@ -0,0 +1,140 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TestUtils; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.DurableExecutionInput; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** + * Tests that the durable processor wrappers run under the durable execution config the caller supplies. The config is + * how a durable processor registers plugins or supplies its own client, so the wrapper must not force the default. + */ +class DistributedMapDurableHandlersTest { + private static final String EXECUTION_OP_ID = "f3d7b0c0-1234-5678-90ab-cdef12345678"; + private static final String EXECUTION_ARN = + "arn:aws:lambda:us-east-1:123456789012:function:test" + "/durable-execution/exec-name/" + EXECUTION_OP_ID; + + /** Counts deserialize calls, so a test can prove the config's own serdes read the event. */ + private static final class RecordingSerDes implements SerDes { + private final SerDes delegate = new JacksonSerDes(); + private final AtomicInteger deserializeCount = new AtomicInteger(); + + @Override + public String serialize(Object value) { + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + deserializeCount.incrementAndGet(); + return delegate.deserialize(data, typeToken); + } + } + + private static DurableConfig configWith(SerDes serDes) { + return DurableConfig.builder() + .withDurableExecutionClient(TestUtils.createMockClient()) + .withSerDes(serDes) + .build(); + } + + // A first invocation of a durable function: one STARTED execution operation carrying the processor batch. + private static DurableExecutionInput durableInput(ProcessorEvent event) { + var executionOp = Operation.builder() + .id(EXECUTION_OP_ID) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.parse("2026-08-15T00:00:00Z")) + .executionDetails(ExecutionDetails.builder() + .inputPayload(new JacksonSerDes().serialize(event)) + .build()) + .build(); + return new DurableExecutionInput( + EXECUTION_ARN, + "token-1", + CheckpointUpdatedExecutionState.builder() + .operations(List.of(executionOp)) + .build()); + } + + private static ProcessorEvent event(String... bodies) { + var records = new java.util.ArrayList(); + for (var i = 0; i < bodies.length; i++) { + records.add(new ProcessorRecord(String.valueOf(i), bodies[i])); + } + return new ProcessorEvent(records); + } + + @Test + void durableBatchHandlerRunsUnderTheSuppliedConfig() { + var serDes = new RecordingSerDes(); + var seen = new java.util.ArrayList(); + BiConsumer> func = + (ctx, items) -> seen.addAll(ctx.step("copy", new TypeToken>() {}, stepCtx -> items)); + + var handler = DistributedMapHandlers.createDistributedMapBatchHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, configWith(serDes)); + var output = handler.handleRequest(durableInput(event("1", "2")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertEquals(List.of(1, 2), seen); + // The processor's event is always read with the default JSON SerDes. The step inside the processor uses the + // config's SerDes, so a call here means the supplied config reached the execution. + assertTrue(serDes.deserializeCount.get() > 0); + } + + @Test + void durableItemHandlerRunsUnderTheSuppliedConfig() { + var serDes = new RecordingSerDes(); + BiFunction func = (ctx, item) -> item * 2; + + var handler = DistributedMapHandlers.createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, configWith(serDes)); + var output = handler.handleRequest(durableInput(event("1", "2")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertTrue(serDes.deserializeCount.get() > 0); + } + + // The service defines the processor's event and response format, so a custom SerDes on the config must not + // rename the envelope's fields. + @Test + void customSerDesDoesNotChangeTheProcessorEnvelope() { + var snakeCase = new JacksonSerDes(new ObjectMapper() + .setPropertyNamingStrategy(PropertyNamingStrategies.SNAKE_CASE) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)); + BiFunction func = (ctx, item) -> item * 2; + + var handler = DistributedMapHandlers.createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, configWith(snakeCase)); + var output = handler.handleRequest(durableInput(event("1")), null); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + assertTrue(output.result().contains("\"batchItemResults\""), output.result()); + assertTrue(output.result().contains("\"itemIdentifier\":\"0\""), output.result()); + assertTrue(output.result().contains("\"batchItemFailures\""), output.result()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java new file mode 100644 index 0000000000..9731a22f3e --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/dmap/DistributedMapHandlersTest.java @@ -0,0 +1,460 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.dmap; + +import static org.junit.jupiter.api.Assertions.*; +import static software.amazon.lambda.durable.dmap.DistributedMapHandlers.*; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Unit tests for the distributed map authoring helpers (non-durable variants). */ +class DistributedMapHandlersTest { + + // Builds a single processor record. The body is the serialized item, as the wire carries it. + private static ProcessorRecord rec(String itemId, String body) { + return new ProcessorRecord(itemId, body); + } + + // Builds a distributed map processor event carrying the given records. + private static ProcessorEvent processorEvent(ProcessorRecord... records) { + return new ProcessorEvent(List.of(records)); + } + + // Builds a distributed map reader event carrying the state and maxItemsPerPage fields. + private static ReaderEvent readerEvent(String state, int maxItemsPerPage) { + return new ReaderEvent(maxItemsPerPage, state); + } + + private static ItemFailure.ErrorDetail error(String type, String message) { + return new ItemFailure.ErrorDetail(type, message, null, null); + } + + private static void assertFailure(ItemFailure failure, String itemId, String type, String message) { + assertEquals(itemId, failure.itemIdentifier()); + assertEquals(type, failure.error().errorType()); + assertEquals(message, failure.error().errorMessage()); + assertNull(failure.error().errorData()); + assertFalse(failure.error().stackTrace().isEmpty()); + } + + /** Serdes whose wire form is not JSON, so an item that skipped serialization cannot be read back. */ + private static final class PrefixSerDes implements SerDes { + @Override + public String serialize(Object value) { + return "p:" + value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + if (!data.startsWith("p:")) { + throw new IllegalArgumentException("not a prefixed value: " + data); + } + return (T) Integer.valueOf(data.substring(2)); + } + } + + @Test + void itemHandlerReportsResultsInOrder() { + Function func = x -> x * 2; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "2"), rec("1", "3")), null); + + assertEquals(List.of(new ItemResult("0", "4"), new ItemResult("1", "6")), resp.batchItemResults()); + assertEquals(List.of(), resp.batchItemFailures()); + } + + @Test + void itemHandlerCapturesFailuresWithFullyQualifiedErrorType() { + Function func = x -> { + if (x.equals("bad")) { + throw new IllegalStateException("boom"); + } + return x; + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(String.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "\"ok\""), rec("1", "\"bad\"")), null); + + assertEquals(List.of(new ItemResult("0", "\"ok\"")), resp.batchItemResults()); + assertEquals(1, resp.batchItemFailures().size()); + assertFailure(resp.batchItemFailures().get(0), "1", "java.lang.IllegalStateException", "boom"); + } + + @Test + void itemHandlerReportsAnEmptyResultsListWhenEveryItemFails() { + Function func = x -> { + throw new IllegalStateException("nope"); + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var resp = handler.handleRequest(processorEvent(rec("7", "1")), null); + + // Both lists are always reported, so an all-failing batch carries an empty results list rather than no key. + assertEquals(List.of(), resp.batchItemResults()); + assertEquals(1, resp.batchItemFailures().size()); + assertFailure(resp.batchItemFailures().get(0), "7", "java.lang.IllegalStateException", "nope"); + } + + @Test + void itemHandlerHonorsRequestedConcurrency() { + // Each item blocks until all four have arrived, so the batch only finishes if the pool really runs four + // items at once. At a smaller pool size the first item waits out the timeout and every item fails. + var barrier = new CyclicBarrier(4); + Function func = x -> { + try { + barrier.await(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new IllegalStateException("barrier not reached", e); + } + return x; + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 4); + + var resp = + handler.handleRequest(processorEvent(rec("0", "0"), rec("1", "1"), rec("2", "2"), rec("3", "3")), null); + + assertEquals(List.of(), resp.batchItemFailures()); + // The four items are released together, so completion order is arbitrary while results stay in item order. + assertEquals( + List.of( + new ItemResult("0", "0"), + new ItemResult("1", "1"), + new ItemResult("2", "2"), + new ItemResult("3", "3")), + resp.batchItemResults()); + } + + @Test + void itemHandlerDefaultsToOneConcurrentHandler() { + var inFlight = new AtomicInteger(); + var peakInFlight = new AtomicInteger(); + Function func = x -> { + peakInFlight.accumulateAndGet(inFlight.incrementAndGet(), Math::max); + try { + // Held briefly so a regression to a wider pool would actually overlap and be caught. + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + inFlight.decrementAndGet(); + return x; + }; + // The overload with no concurrency argument runs one handler at a time. + var handler = createDistributedMapItemHandler(func, TypeToken.get(String.class), null, null); + + var resp = handler.handleRequest(processorEvent(rec("0", "\"a\""), rec("1", "\"b\""), rec("2", "\"c\"")), null); + + assertEquals(1, peakInFlight.get()); + assertEquals(3, resp.batchItemResults().size()); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void itemHandlerRejectsNonPositiveConcurrency(int concurrency) { + var exception = assertThrows( + IllegalArgumentException.class, + () -> DistributedMapHandlers.createDistributedMapItemHandler( + item -> item, TypeToken.get(String.class), null, null, concurrency)); + assertEquals("concurrency must be at least 1, got: " + concurrency, exception.getMessage()); + } + + @Test + void itemHandlerReportsSerializationFailureAgainstTheOffendingItemOnly() { + // A result serdes that refuses exactly one value, standing in for a POJO the caller cannot serialize. + var refusingSerDes = new SerDes() { + private final SerDes delegate = new JacksonSerDes(); + + @Override + public String serialize(Object value) { + if ("boom".equals(value)) { + throw new IllegalStateException("cannot serialize"); + } + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return delegate.deserialize(data, typeToken); + } + }; + + var handler = DistributedMapHandlers.createDistributedMapItemHandler( + item -> item, TypeToken.get(String.class), null, refusingSerDes, 2); + + var resp = handler.handleRequest( + processorEvent(rec("a", "\"ok\""), rec("b", "\"boom\""), rec("c", "\"fine\"")), null); + + // The whole batch must not fail. Only item b is reported as failed. + assertEquals(2, resp.batchItemResults().size()); + assertEquals("a", resp.batchItemResults().get(0).itemIdentifier()); + assertEquals("c", resp.batchItemResults().get(1).itemIdentifier()); + assertEquals(1, resp.batchItemFailures().size()); + assertEquals("b", resp.batchItemFailures().get(0).itemIdentifier()); + } + + @Test + void processorEventRejectsMissingRecords() { + var ex = assertThrows(IllegalStateException.class, () -> new ProcessorEvent(null)); + assertTrue(ex.getMessage().contains("'records' array")); + assertTrue(ex.getMessage().contains("not invoked directly")); + } + + @Test + void readerEventRejectsMissingMaxItemsPerPage() { + var ex = assertThrows(IllegalStateException.class, () -> new ReaderEvent(null, "state")); + assertTrue(ex.getMessage().contains("'maxItemsPerPage'")); + } + + @Test + void batchHandlerSucceedsAllItemsWhenFunctionReturns() { + var seen = new ArrayList(); + Consumer> func = seen::addAll; + var handler = createDistributedMapBatchHandler(func, TypeToken.get(Integer.class), null); + + var resp = handler.handleRequest(processorEvent(rec("0", "1"), rec("1", "2")), null); + + assertNull(resp); + assertEquals(List.of(1, 2), seen); + } + + @Test + void batchHandlerFailsBatchWhenFunctionThrows() { + Consumer> func = items -> { + throw new RuntimeException("batch failed"); + }; + var handler = createDistributedMapBatchHandler(func, TypeToken.get(Integer.class), null); + + var ex = assertThrows(RuntimeException.class, () -> handler.handleRequest(processorEvent(rec("0", "1")), null)); + assertTrue(ex.getMessage().contains("batch failed")); + } + + @Test + void readerReturnsItemsAndNextStateThenExhausts() { + BiFunction> read = (state, maxItemsPerPage) -> + state == null ? new ReaderPage<>(List.of(1, 2), 1) : new ReaderPage<>(List.of(3), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var first = handler.handleRequest(readerEvent(null, 10), null); + assertEquals(List.of("1", "2"), first.items()); + assertEquals("1", first.nextState()); + + var second = handler.handleRequest(readerEvent("1", 10), null); + assertEquals(List.of("3"), second.items()); + // Null rather than empty, so the runtime omits the key, which is how exhaustion is signalled. + assertNull(second.nextState()); + } + + @Test + void readerReceivesMaxItemsPerPage() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(maxItemsPerPage), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var page = handler.handleRequest(readerEvent(null, 7), null); + + assertEquals(List.of("7"), page.items()); + } + + @Test + void readerRejectsPageOverMaxItemsPerPage() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1, 2, 3), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 2), null)); + assertTrue(ex.getMessage().contains("exceeding maxItemsPerPage")); + } + + @Test + void readerRejectsOversizedNextState() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), "x".repeat(40_000)); + var handler = createDistributedMapReader(read, TypeToken.get(String.class), null, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertTrue(ex.getMessage().contains("32 KB")); + } + + @Test + void readerSendsNullItemAsJsonNull() { + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(java.util.Arrays.asList(1, null), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, null); + + var page = handler.handleRequest(readerEvent(null, 10), null); + + assertEquals(List.of("1", "null"), page.items()); + } + + @Test + void readerRejectsItemSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), null); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), null, nullSerDes); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertEquals("serialization of reader item 0 produced no string", ex.getMessage()); + } + + @Test + void readerRejectsNextStateSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1), 1); + var handler = createDistributedMapReader(read, TypeToken.get(Integer.class), nullSerDes, null); + + var ex = assertThrows(IllegalStateException.class, () -> handler.handleRequest(readerEvent(null, 10), null)); + assertEquals("serialization of reader nextState produced no string", ex.getMessage()); + } + + @Test + void readerItemsRoundTripThroughTheProcessorUnderACustomItemSerdes() { + var serdes = new PrefixSerDes(); + BiFunction> read = + (state, maxItemsPerPage) -> new ReaderPage<>(List.of(1, 2), null); + var reader = createDistributedMapReader(read, TypeToken.get(Integer.class), null, serdes); + + var page = reader.handleRequest(readerEvent(null, 10), null); + assertEquals(List.of("p:1", "p:2"), page.items()); + + var seen = new ArrayList(); + var processor = createDistributedMapItemHandler( + (Function) item -> { + seen.add(item); + return item; + }, + TypeToken.get(Integer.class), + serdes, + null); + processor.handleRequest( + processorEvent( + rec("0", page.items().get(0)), rec("1", page.items().get(1))), + null); + + assertEquals(List.of(1, 2), seen); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void durableItemHandlerRejectsNonPositiveConcurrency(int concurrency) { + BiFunction func = (ctx, item) -> item; + var exception = assertThrows( + IllegalArgumentException.class, + () -> createDistributedMapItemHandlerWithDurableExecution( + func, TypeToken.get(Integer.class), null, null, concurrency)); + assertEquals("concurrency must be at least 1, got: " + concurrency, exception.getMessage()); + } + + // The Lambda runtime serializes the response, so these assert the wire shape it produces. The mapper mirrors the + // runtime's null exclusion, which is what keeps an absent key absent. + private static final ObjectMapper WIRE = new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL); + + @Test + void itemHandlerReportsNoOutputWhenTheHandlerReturnsNull() { + // PrefixSerDes stringifies whatever it is handed, so an output is reported only if the handler + // never calls it for an item that returned nothing. + Function func = x -> null; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, new PrefixSerDes(), 1); + + var resp = handler.handleRequest(processorEvent(rec("0", "1")), null); + + assertEquals(List.of(), resp.batchItemFailures()); + assertEquals(List.of(new ItemResult("0", null)), resp.batchItemResults()); + } + + @Test + void itemResultWithNoOutputLeavesTheOutputOffTheWire() throws Exception { + var json = WIRE.writeValueAsString(new ItemResult("0", null)); + + assertEquals("{\"itemIdentifier\":\"0\"}", json); + } + + @Test + void itemFailureCarriesTheStackTraceAndNoErrorData() { + Function func = x -> { + throw new IllegalStateException("nope"); + }; + var handler = createDistributedMapItemHandler(func, TypeToken.get(Integer.class), null, null, 1); + + var error = handler.handleRequest(processorEvent(rec("0", "1")), null) + .batchItemFailures() + .get(0) + .error(); + + assertNull(error.errorData()); + assertTrue(error.stackTrace().get(0).startsWith(DistributedMapHandlersTest.class.getName())); + } + + @Test + void itemFailureNestsTheErrorObject() throws Exception { + var json = WIRE.writeValueAsString(new ItemFailure("1", error("SomeType", "boom"))); + + assertEquals( + "{\"itemIdentifier\":\"1\",\"error\":{\"errorType\":\"SomeType\",\"errorMessage\":\"boom\"}}", json); + } + + @Test + void itemFailureWritesEveryErrorMember() throws Exception { + var detail = new ItemFailure.ErrorDetail("E", "boom", "{\"k\":1}", List.of("f1")); + var json = WIRE.writeValueAsString(new ItemFailure("1", detail)); + + assertEquals( + "{\"itemIdentifier\":\"1\",\"error\":{\"errorType\":\"E\",\"errorMessage\":\"boom\"," + + "\"errorData\":\"{\\\"k\\\":1}\",\"stackTrace\":[\"f1\"]}}", + json); + } + + @Test + void exhaustedReaderPageLeavesNextStateOffTheWire() throws Exception { + var json = WIRE.writeValueAsString(new ReaderResponse(List.of("1", "2"), null)); + + assertEquals("{\"items\":[\"1\",\"2\"]}", json); + } + + @ParameterizedTest + @NullAndEmptySource + void processorRecordWithoutItemIdIsRejected(String itemId) { + var exception = assertThrows(IllegalStateException.class, () -> new ProcessorRecord(itemId, "1")); + assertEquals("expected each distributed map processor record to have an 'itemId'", exception.getMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java index 61f534e26a..54e7b7bbe7 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/execution/CheckpointManagerTest.java @@ -9,10 +9,12 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; @@ -20,6 +22,16 @@ import software.amazon.awssdk.http.SdkHttpResponse; import software.amazon.awssdk.services.lambda.model.CheckpointDurableExecutionResponse; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.DistributedMapCsvFormatOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapInlineSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapReaderFunctionSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3ObjectConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapS3SourceType; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceConfig; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceFormat; +import software.amazon.awssdk.services.lambda.model.DistributedMapSourceType; import software.amazon.awssdk.services.lambda.model.GetDurableExecutionStateResponse; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -75,6 +87,143 @@ void checkpoint_sendsUpdateAndReturnsCompletedFuture() throws Exception { assertTrue(future.isDone()); } + @Test + void checkpoint_sendsLargeInlineSourceInItsOwnRequest() throws Exception { + // 800 KB of inline items, over the 750 KB batch limit, so this update cannot share a request. + var items = Collections.nCopies(800, "x".repeat(1024)); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.INLINE) + .inlineSourceConfig(DistributedMapInlineSourceConfig.builder() + .items(items) + .build()) + .build()) + .build()) + .build(); + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(dmapStart); + var second = batcher.checkpoint(step); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the oversized update must not share a request"); + assertEquals( + List.of("op-dmap"), + sent.get(0).stream().map(OperationUpdate::id).toList()); + assertEquals( + List.of("op-step"), + sent.get(1).stream().map(OperationUpdate::id).toList()); + } + + @Test + void checkpoint_countsCsvHeadersTowardBatchSize() throws Exception { + // The step payload fills the 750 KB batch limit to within 15,000 characters. The two updates fit in one + // request only if the 30,000 characters of CSV headers are left out of the estimate. + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .payload("x".repeat(750 * 1024 - 243 - 15_000)) + .build(); + var headers = IntStream.range(0, 200) + .mapToObj(i -> String.format("%03d", i) + "h".repeat(147)) + .toList(); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.S3) + .s3SourceConfig(DistributedMapS3SourceConfig.builder() + .type(DistributedMapS3SourceType.OBJECT) + .bucket("bucket") + .objectConfig(DistributedMapS3ObjectConfig.builder() + .key("data.csv") + .format(DistributedMapSourceFormat.CSV) + .csvFormatOptions(DistributedMapCsvFormatOptions.builder() + .headers(headers) + .build()) + .build()) + .build()) + .build()) + .build()) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(step); + var second = batcher.checkpoint(dmapStart); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the CSV headers must count toward the batch size"); + } + + @Test + void checkpoint_countsReaderInitialStateTowardBatchSize() throws Exception { + // The step payload fills the 750 KB batch limit to within 15,000 characters. The two updates fit in one + // request only if the reader's 30,000-character initial state is left out of the estimate. + var step = OperationUpdate.builder() + .id("op-step") + .type(OperationType.STEP) + .action(OperationAction.START) + .payload("x".repeat(750 * 1024 - 243 - 15_000)) + .build(); + var dmapStart = OperationUpdate.builder() + .id("op-dmap") + .type(OperationType.DISTRIBUTED_MAP) + .action(OperationAction.START) + .distributedMapOptions(DistributedMapOptions.builder() + .source(DistributedMapSourceConfig.builder() + .type(DistributedMapSourceType.READER_FUNCTION) + .readerFunctionSourceConfig(DistributedMapReaderFunctionSourceConfig.builder() + .functionName("reader") + .initialState("s".repeat(30_000)) + .build()) + .build()) + .build()) + .build(); + + var sent = new ArrayList>(); + when(client.checkpoint(anyString(), anyString(), anyList())).thenAnswer(invocation -> { + sent.add(invocation.getArgument(2)); + return CheckpointDurableExecutionResponse.builder() + .checkpointToken("token-2") + .build(); + }); + + var first = batcher.checkpoint(step); + var second = batcher.checkpoint(dmapStart); + first.get(500, TimeUnit.MILLISECONDS); + second.get(500, TimeUnit.MILLISECONDS); + + assertEquals(2, sent.size(), "the reader state must count toward the batch size"); + } + @Test void checkpoint_skipsApiCallWhenExecutionAlreadyCompleted() throws Exception { var finishCheckpointProcessing = mock(Runnable.class); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java new file mode 100644 index 0000000000..a134f7425d --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapResultTest.java @@ -0,0 +1,191 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.DistributedMapException; + +class DistributedMapResultTest { + + private static DistributedMapSummary summary(DistributedMapStatus status, long failureCount) { + return new DistributedMapSummary( + status, + true, + 2, + failureCount, + 1, + "arn:aws:states:us-east-1:123456789012:mapRun:run-42", + new DistributedMapError("RunFailed", "details", null, null), + 4L); + } + + private static DistributedMapError itemError(String message) { + return new DistributedMapError("java.lang.RuntimeException", message, null, null); + } + + @Test + void succeeded_returnsAllSucceededItems() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("fail-b")), + DistributedMapResult.Item.succeeded("c", "out-c"))); + + var succeeded = result.succeeded(); + assertEquals(2, succeeded.size()); + assertEquals("a", succeeded.get(0).itemId()); + assertEquals("c", succeeded.get(1).itemId()); + } + + @Test + void failed_returnsAllFailedItems() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("fail-b")))); + + var failed = result.failed(); + assertEquals(1, failed.size()); + assertEquals("b", failed.get(0).itemId()); + } + + @Test + void results_preservesNullOutputs() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 0), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.succeeded("b", null), + DistributedMapResult.Item.succeeded("c", "out-c"))); + + // cardinality must match succeeded() so callers can align outputs with items + assertEquals(3, result.results().size()); + assertEquals(Arrays.asList("out-a", null, "out-c"), result.results()); + assertEquals(result.succeeded().size(), result.results().size()); + } + + @Test + void errors_preservesNullErrors() { + DistributedMapResult.Item withError = DistributedMapResult.Item.failed("a", itemError("fail-a")); + var withoutError = + new DistributedMapResult.Item("b", DistributedMapResult.Item.Status.FAILED, null, null); + var result = new DistributedMapResult( + summary(DistributedMapStatus.SUCCEEDED, 2), List.of(withError, withoutError)); + + // cardinality must match failed() so callers can align errors with items + var errors = result.errors(); + assertEquals(2, errors.size()); + assertEquals("fail-a", errors.get(0).errorMessage()); + assertNull(errors.get(1)); + assertEquals(result.failed().size(), errors.size()); + } + + @Test + void throwIfError_throwsRunLevelOnNonSucceededStatus() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.FAILED, 0), List.of(DistributedMapResult.Item.succeeded("a", "out-a"))); + + var error = assertThrows(DistributedMapException.class, result::throwIfError); + assertEquals(DistributedMapStatus.FAILED, error.status()); + } + + @Test + void throwIfError_surfacesFirstFailedItemWhenSucceededWithFailures() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.failed("b", itemError("boom-b")))); + + var error = assertThrows(DistributedMapException.class, result::throwIfError); + assertTrue(error.getMessage().contains("boom-b")); + } + + private static String itemFailureMessage(DistributedMapError error) { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 1), List.of(DistributedMapResult.Item.failed("b", error))); + return assertThrows(DistributedMapException.class, result::throwIfError).getMessage(); + } + + @Test + void throwIfError_itemMessageJoinsTypeAndMessage() { + assertEquals("E: boom", itemFailureMessage(new DistributedMapError("E", "boom", null, null))); + } + + @Test + void throwIfError_itemMessageUsesMessageWhenTypeIsAbsent() { + assertEquals("boom", itemFailureMessage(new DistributedMapError(null, "boom", null, null))); + assertEquals("boom", itemFailureMessage(new DistributedMapError("", "boom", null, null))); + } + + @Test + void throwIfError_itemMessageUsesTypeWhenMessageIsAbsent() { + assertEquals("E", itemFailureMessage(new DistributedMapError("E", null, null, null))); + assertEquals("E", itemFailureMessage(new DistributedMapError("E", "", null, null))); + } + + @Test + void throwIfError_itemMessageFallsBackWhenTypeAndMessageAreAbsent() { + assertEquals( + "Distributed map item b failed", itemFailureMessage(new DistributedMapError(null, null, null, null))); + assertEquals("Distributed map item b failed", itemFailureMessage(new DistributedMapError("", "", null, null))); + assertEquals("Distributed map item b failed", itemFailureMessage(null)); + } + + @Test + void throwIfError_doesNothingOnFullSuccess() { + var result = new DistributedMapResult<>( + summary(DistributedMapStatus.SUCCEEDED, 0), + List.of( + DistributedMapResult.Item.succeeded("a", "out-a"), + DistributedMapResult.Item.succeeded("b", "out-b"))); + + assertDoesNotThrow(result::throwIfError); + } + + @Test + void delegators_returnComposedSummaryValues() { + var summary = summary(DistributedMapStatus.SUCCEEDED, 1); + var result = new DistributedMapResult<>(summary, List.of(DistributedMapResult.Item.succeeded("a", "out-a"))); + + assertEquals(summary.status(), result.status()); + assertEquals(summary.sourceExhausted(), result.sourceExhausted()); + assertEquals(summary.successCount(), result.successCount()); + assertEquals(summary.failureCount(), result.failureCount()); + assertEquals(summary.unprocessedCount(), result.unprocessedCount()); + assertEquals(summary.totalCount(), result.totalCount()); + assertEquals(summary.distributedMapRunArn(), result.distributedMapRunArn()); + assertEquals(summary.error(), result.error()); + assertEquals(summary.hasFailure(), result.hasFailure()); + } + + @Test + void items_defaultToEmptyWhenNull() { + var result = new DistributedMapResult(summary(DistributedMapStatus.SUCCEEDED, 0), null); + + assertTrue(result.items().isEmpty()); + } + + @Test + void of_usesFullyQualifiedClassName() { + var error = DistributedMapError.of(new IllegalStateException("boom")); + + assertEquals("java.lang.IllegalStateException", error.errorType()); + assertEquals("boom", error.errorMessage()); + assertFalse(error.stackTrace().isEmpty()); + } + + @Test + void of_preservesNullMessage() { + var error = DistributedMapError.of(new RuntimeException()); + + assertEquals("java.lang.RuntimeException", error.errorType()); + assertNull(error.errorMessage()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java new file mode 100644 index 0000000000..4c8137524f --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/model/DistributedMapSummaryTest.java @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.model; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.DistributedMapException; + +class DistributedMapSummaryTest { + + private static DistributedMapSummary summary( + DistributedMapStatus status, long failureCount, String arn, String errorMessage) { + var error = errorMessage != null ? new DistributedMapError("RunFailed", errorMessage, null, null) : null; + return new DistributedMapSummary(status, true, 5, failureCount, 0, arn, error, 5L); + } + + @Test + void distributedMapRunArn_exposedAsGiven() { + var s = summary( + DistributedMapStatus.SUCCEEDED, + 0, + "arn:aws:lambda:us-east-1:123456789012:distributed-map-run/abc", + null); + + assertEquals("arn:aws:lambda:us-east-1:123456789012:distributed-map-run/abc", s.distributedMapRunArn()); + } + + @Test + void distributedMapRunArn_nullWhenAbsent() { + var s = summary(DistributedMapStatus.SUCCEEDED, 0, null, null); + + assertNull(s.distributedMapRunArn()); + } + + @Test + void hasFailure_trueWhenFailureCountPositive() { + assertTrue(summary(DistributedMapStatus.SUCCEEDED, 3, "arn:x", null).hasFailure()); + } + + @Test + void hasFailure_falseWhenNoFailures() { + assertFalse(summary(DistributedMapStatus.SUCCEEDED, 0, "arn:x", null).hasFailure()); + } + + @Test + void throwIfError_throwsOnNonSucceededStatus() { + var s = summary(DistributedMapStatus.FAILED, 0, "arn:x", "run blew up"); + + var error = assertThrows(DistributedMapException.class, s::throwIfError); + assertEquals(DistributedMapStatus.FAILED, error.status()); + assertEquals("run blew up", error.error().errorMessage()); + assertTrue(error.getMessage().contains("run blew up")); + } + + @Test + void throwIfError_throwsWhenSucceededWithFailures() { + var s = summary(DistributedMapStatus.SUCCEEDED, 2, "arn:x", null); + + assertThrows(DistributedMapException.class, s::throwIfError); + } + + @Test + void throwIfError_doesNothingOnFullSuccess() { + var s = summary(DistributedMapStatus.SUCCEEDED, 0, "arn:x", null); + + assertDoesNotThrow(s::throwIfError); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java new file mode 100644 index 0000000000..ed1dcd6608 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTest.java @@ -0,0 +1,333 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiFunction; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DistributedMapException; +import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.IllegalDurableOperationException; +import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.execution.ThreadContext; +import software.amazon.lambda.durable.execution.ThreadType; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.model.DistributedMapSummary; +import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +class DistributedMapOperationTest { + private static final String OPERATION_ID = "7"; + private static final String OPERATION_NAME = "test-distributed-map"; + private static final OperationIdentifier OPERATION_IDENTIFIER = + OperationIdentifier.of(OPERATION_ID, OPERATION_NAME, OperationSubType.DISTRIBUTED_MAP); + private static final SerDes SER_DES = new JacksonSerDes(); + + private ExecutionManager executionManager; + private DurableContextImpl durableContext; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + @BeforeEach + void setUp() { + executionManager = mock(ExecutionManager.class); + durableContext = mock(DurableContextImpl.class); + when(durableContext.getExecutionManager()).thenReturn(executionManager); + when(executionManager.getCurrentThreadContext()).thenReturn(new ThreadContext("root", ThreadType.CONTEXT)); + } + + @Test + void getReturnsSummaryWhenOperationSucceeded() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(5L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(5L) + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertTrue(summary.sourceExhausted()); + assertEquals(5L, summary.successCount()); + assertEquals(0L, summary.failureCount()); + assertEquals(5L, summary.totalCount()); + assertEquals("arn:aws:lambda:us-east-1:123456789012:map-run:abc", summary.distributedMapRunArn()); + assertFalse(summary.hasFailure()); + } + + @Test + void getReturnsResultWhenResultCollecting() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(2L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(2L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + @SuppressWarnings({"unchecked", "rawtypes"}) + TypeToken> resultType = (TypeToken) TypeToken.get(DistributedMapResult.class); + BiFunction> resultBuilder = + (s, d) -> DistributedMapOperation.toResult(s, d, TypeToken.get(Integer.class), SER_DES); + + var operation = newOperation(resultType, resultBuilder); + operation.onCheckpointComplete(op); + + var result = operation.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, result.status()); + assertEquals(2L, result.successCount()); + assertTrue(result.results().isEmpty()); + assertTrue(result.errors().isEmpty()); + } + + @Test + void getResolvesWithSummaryWhenOperationFailed() { + assertGetResolvesWithSummary(OperationStatus.FAILED, DistributedMapStatus.FAILED); + } + + @Test + void getResolvesWithSummaryWhenOperationTimedOut() { + assertGetResolvesWithSummary(OperationStatus.TIMED_OUT, DistributedMapStatus.TIMED_OUT); + } + + @Test + void getResolvesWithSummaryWhenOperationStopped() { + assertGetResolvesWithSummary(OperationStatus.STOPPED, DistributedMapStatus.STOPPED); + } + + @Test + void getTerminatesWhenSucceededWithNoDetails() { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + assertThrows(IllegalDurableOperationException.class, operation::get); + verify(executionManager).terminateExecution(any(IllegalDurableOperationException.class)); + } + + @Test + void getTerminatesWhenARunExistedButNoSourceExhausted() { + var details = DistributedMapDetails.builder() + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + assertThrows(IllegalDurableOperationException.class, operation::get); + verify(executionManager).terminateExecution(any(IllegalDurableOperationException.class)); + } + + @Test + void getResolvesWhenNoRunWasCreatedAndSourceExhaustedIsAbsent() { + var details = DistributedMapDetails.builder() + .successCount(0L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.FAILED) + .distributedMapDetails(details) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(DistributedMapStatus.FAILED, summary.status()); + assertNull(summary.sourceExhausted()); + } + + @Test + void getSuspendsUntilCheckpointCompletesThenReturnsSummary() + throws InterruptedException, ExecutionException, TimeoutException { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L) + .build(); + var op = succeededOperation(details); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + + // get blocks because the operation has not reached a terminal checkpoint yet + var future = executor.submit(operation::get); + try { + future.get(500, TimeUnit.MILLISECONDS); + fail(); + } catch (TimeoutException e) { + // feed the terminal checkpoint, which unblocks get and resolves the summary + operation.onCheckpointComplete(op); + var summary = future.get(); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertEquals(1L, summary.successCount()); + } + } + + // A terminal-failed operation resolves with its summary so the counts are reachable. + private void assertGetResolvesWithSummary(OperationStatus status, DistributedMapStatus expected) { + var details = DistributedMapDetails.builder() + .sourceExhausted(false) + .successCount(3L) + .failureCount(2L) + .unprocessedCount(1L) + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(status) + .distributedMapDetails(details) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary); + operation.onCheckpointComplete(op); + + var summary = operation.get(); + assertEquals(expected, summary.status()); + assertFalse(summary.sourceExhausted()); + assertEquals(3L, summary.successCount()); + assertEquals(2L, summary.failureCount()); + assertEquals(1L, summary.unprocessedCount()); + assertTrue(summary.hasFailure()); + assertThrows(DistributedMapException.class, summary::throwIfError); + } + + @Test + void replayPollsWhileTheOperationIsPending() { + assertReplayPolls(OperationStatus.PENDING); + } + + @Test + void replayPollsWhileTheOperationIsStarted() { + assertReplayPolls(OperationStatus.STARTED); + } + + @Test + void pollingContinuesWhenPendingBecomesStarted() { + var pending = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.DISTRIBUTED_MAP) + .subType(OperationSubType.DISTRIBUTED_MAP.getValue()) + .status(OperationStatus.PENDING) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(pending); + var started = pending.toBuilder().status(OperationStatus.STARTED).build(); + when(executionManager.pollForOperationUpdates(OPERATION_ID)) + .thenReturn(CompletableFuture.completedFuture(started), new CompletableFuture<>()); + + newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary) + .execute(); + + verify(executionManager, times(2)).pollForOperationUpdates(OPERATION_ID); + } + + @Test + void failedRunErrorIsReportedToPlugins() { + var error = ErrorObject.builder() + .errorType("SourceReadError") + .errorMessage("object not found") + .build(); + var op = Operation.builder() + .id(OPERATION_ID) + .type(OperationType.DISTRIBUTED_MAP) + .status(OperationStatus.FAILED) + .distributedMapDetails( + DistributedMapDetails.builder().error(error).build()) + .build(); + + var extracted = BaseDurableOperation.extractErrorFromOperation(op); + + var exception = assertInstanceOf(DurableOperationException.class, extracted); + assertEquals(error, exception.getErrorObject()); + } + + // A non-terminal replay waits for the terminal outcome rather than terminating the execution. + private void assertReplayPolls(OperationStatus status) { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.DISTRIBUTED_MAP) + .subType(OperationSubType.DISTRIBUTED_MAP.getValue()) + .status(status) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + newOperation(TypeToken.get(DistributedMapSummary.class), DistributedMapOperation::toSummary) + .execute(); + + verify(executionManager).pollForOperationUpdates(OPERATION_ID); + verify(executionManager, never()).terminateExecution(any()); + } + + private DistributedMapOperation newOperation( + TypeToken resultType, BiFunction resultBuilder) { + return new DistributedMapOperation<>( + OPERATION_IDENTIFIER, + DistributedMapOptions.builder().build(), + resultType, + SER_DES, + resultBuilder, + durableContext); + } + + private Operation succeededOperation(DistributedMapDetails details) { + return Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .distributedMapDetails(details) + .build(); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java new file mode 100644 index 0000000000..607576db5b --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/DistributedMapOperationTranslationTest.java @@ -0,0 +1,852 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.operation; + +import static org.junit.jupiter.api.Assertions.*; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; +import software.amazon.awssdk.services.lambda.model.DistributedMapDetails; +import software.amazon.awssdk.services.lambda.model.DistributedMapOptions; +import software.amazon.awssdk.services.lambda.model.DistributedMapResultItem; +import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.config.DistributedMapConfig; +import software.amazon.lambda.durable.config.DistributedMapDestination; +import software.amazon.lambda.durable.config.DistributedMapProcessor; +import software.amazon.lambda.durable.config.DistributedMapProcessorRetryConfig; +import software.amazon.lambda.durable.config.DistributedMapSource; +import software.amazon.lambda.durable.exception.DurableExecutionException; +import software.amazon.lambda.durable.model.DistributedMapResult; +import software.amazon.lambda.durable.model.DistributedMapStatus; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; + +/** Translation tests for DistributedMapOperation toOptions, toSummary and toResult. */ +class DistributedMapOperationTranslationTest { + + private static final SerDes SERDES = new JacksonSerDes(); + + // ------------------------------------------------------------------ + // helpers + // ------------------------------------------------------------------ + + private static DistributedMapOptions options( + DistributedMapSource source, + DistributedMapProcessor processor, + DistributedMapConfig config, + boolean collect) { + return DistributedMapOperation.toOptions(source, processor, 4, config, SERDES, collect); + } + + private static DistributedMapOptions options(DistributedMapSource source, DistributedMapProcessor processor) { + return options(source, processor, DistributedMapConfig.builder().build(), false); + } + + private static DistributedMapProcessor batchProcessor() { + return DistributedMapProcessor.reportBatchOutcome("proc").build(); + } + + private static DistributedMapDetails.Builder detailsBuilder() { + return DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .totalCount(1L); + } + + private static DistributedMapResultItem succeededItem(String itemId, String output) { + return DistributedMapResultItem.builder() + .itemId(itemId) + .status("SUCCEEDED") + .output(output) + .build(); + } + + private static DistributedMapResultItem failedItem(String itemId, ErrorObject error) { + return DistributedMapResultItem.builder() + .itemId(itemId) + .status("FAILED") + .error(error) + .build(); + } + + /** Serdes that upper-cases on serialize and upper-cases plus counts on deserialize. */ + private static final class UpperSerDes implements SerDes { + int deserializeCount = 0; + + @Override + public String serialize(Object value) { + return String.valueOf(value).toUpperCase(Locale.ROOT); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + deserializeCount++; + return (T) data.toUpperCase(Locale.ROOT); + } + } + + // ------------------------------------------------------------------ + // toOptions - top level + // ------------------------------------------------------------------ + + @Test + void inlineSourceSerializesItemsThroughDefaultSerdes() { + var options = options(DistributedMapSource.inline(List.of("a", "b")), batchProcessor()); + assertEquals(4, options.maxConcurrency().intValue()); + assertEquals("INLINE", options.source().typeAsString()); + assertEquals( + List.of(SERDES.serialize("a"), SERDES.serialize("b")), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceSendsNullItemAsJsonNull() { + var options = options(DistributedMapSource.inline(java.util.Arrays.asList("a", null)), batchProcessor()); + assertEquals( + List.of(SERDES.serialize("a"), "null"), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceRejectsItemSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var exception = assertThrows( + IllegalArgumentException.class, + () -> options(DistributedMapSource.inline(List.of("a"), nullSerDes), batchProcessor())); + assertEquals("serialization of inline item 0 produced no string", exception.getMessage()); + } + + /** One item of "é" characters, padded with "a", whose serialized inline array is exactly totalBytes. */ + private static List inlineItemsOfSerializedSize(int totalBytes) { + // The serdes wraps the item in quotes. The array escapes those quotes, then adds its own quotes and brackets. + // So the array is 8 bytes larger than the item text. Each "é" is 2 UTF-8 bytes. + int bodyBytes = totalBytes - 8; + var body = (bodyBytes % 2 == 1 ? "a" : "") + "é".repeat(bodyBytes / 2); + return List.of(body); + } + + private static int serializedInlineBytes(List items) { + var bodies = items.stream().map(SERDES::serialize).toList(); + return SERDES.serialize(bodies).getBytes(StandardCharsets.UTF_8).length; + } + + @Test + void inlineSourceAcceptsSerializedSizeAtTheLimit() { + var items = inlineItemsOfSerializedSize(1024 * 1024); + assertEquals(1024 * 1024, serializedInlineBytes(items)); + + var options = options(DistributedMapSource.inline(items), batchProcessor()); + assertEquals( + List.of(SERDES.serialize(items.get(0))), + options.source().inlineSourceConfig().items()); + } + + @Test + void inlineSourceRejectsSerializedSizeOneByteOverTheLimit() { + var items = inlineItemsOfSerializedSize(1024 * 1024 + 1); + assertEquals(1024 * 1024 + 1, serializedInlineBytes(items)); + // The character count is below the limit, so only a byte count rejects this source. + assertTrue(SERDES.serialize(List.of(SERDES.serialize(items.get(0)))).length() < 1024 * 1024); + + var exception = assertThrows( + IllegalArgumentException.class, () -> options(DistributedMapSource.inline(items), batchProcessor())); + assertEquals("inline source exceeds the 1 MB limit (serialized size: 1048577 bytes)", exception.getMessage()); + } + + @Test + void inlineSourceUsesCustomSerdesForEachItem() { + var options = options(DistributedMapSource.inline(List.of("a", "b"), new UpperSerDes()), batchProcessor()); + assertEquals(List.of("A", "B"), options.source().inlineSourceConfig().items()); + } + + @Test + void maxConcurrencyPassedThrough() { + var options = DistributedMapOperation.toOptions( + DistributedMapSource.inline(List.of("a")), + batchProcessor(), + 42, + DistributedMapConfig.builder().build(), + SERDES, + false); + assertEquals(42, options.maxConcurrency().intValue()); + } + + @Test + void timeoutMapsToWholeSeconds() { + var config = + DistributedMapConfig.builder().timeout(Duration.ofMinutes(30)).build(); + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false); + assertEquals(1800, options.timeoutSeconds().intValue()); + } + + @Test + void timeoutOmittedWhenAbsent() { + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor()); + assertNull(options.timeoutSeconds()); + } + + // ------------------------------------------------------------------ + // toOptions - processor + // ------------------------------------------------------------------ + + @Test + void itemOutcomesSetsResponseTypeAndBatchSize() { + var processor = + DistributedMapProcessor.reportItemOutcomes("proc").batchSize(25).build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals("proc", wire.functionName()); + assertEquals(List.of("REPORT_BATCH_ITEM_OUTCOMES"), wire.functionResponseTypesAsStrings()); + assertEquals(25, wire.batchSize().intValue()); + } + + @Test + void batchOmitsResponseTypes() { + var wire = options(DistributedMapSource.inline(List.of("a")), batchProcessor()) + .processor(); + assertFalse(wire.hasFunctionResponseTypes()); + } + + @Test + void unlimitedRetriesMapToNegativeOne() { + var processor = DistributedMapProcessor.reportItemOutcomes("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(DistributedMapProcessorRetryConfig.UNLIMITED) + .maxRetryDuration(Duration.ofHours(1)) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(-1, wire.maxRetryAttempts().intValue()); + assertEquals(3600, wire.maxRetryDurationSeconds().intValue()); + } + + @Test + void explicitRetryAttemptsPassThrough() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryAttempts(0) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(0, wire.maxRetryAttempts().intValue()); + } + + @Test + void retryDurationOnlyOmitsAttempts() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .retry(DistributedMapProcessorRetryConfig.builder() + .maxRetryDuration(Duration.ofMinutes(5)) + .build()) + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals(300, wire.maxRetryDurationSeconds().intValue()); + assertNull(wire.maxRetryAttempts()); + } + + @Test + void omittedRetryConfigSendsNeitherRetryField() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc").build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertNull(wire.maxRetryAttempts()); + assertNull(wire.maxRetryDurationSeconds()); + } + + @Test + void durableExecutionNamePrefixSent() { + var processor = DistributedMapProcessor.reportBatchOutcome("proc") + .durableExecutionNamePrefix("pfx") + .build(); + var wire = options(DistributedMapSource.inline(List.of("a")), processor).processor(); + assertEquals("pfx", wire.durableExecutionNamePrefix()); + } + + // ------------------------------------------------------------------ + // toOptions - source variants + // ------------------------------------------------------------------ + + @Test + void s3JsonLinesSourceSerializesConfig() { + var source = DistributedMapSource.s3JsonLines("s3://bucket/data.jsonl").maxItems(500); + var wire = options(source, batchProcessor()).source(); + assertEquals("S3", wire.typeAsString()); + assertEquals(500L, wire.maxItemsToRead().longValue()); + assertEquals("OBJECT", wire.s3SourceConfig().typeAsString()); + assertEquals("bucket", wire.s3SourceConfig().bucket()); + assertEquals("data.jsonl", wire.s3SourceConfig().objectConfig().key()); + assertEquals("JSON_LINES", wire.s3SourceConfig().objectConfig().formatAsString()); + } + + @Test + void s3ObjectsSourceUsesListedObjectMetadataAndPrefix() { + var wire = options(DistributedMapSource.s3Objects("s3://b/prefix/"), batchProcessor()) + .source(); + var s3 = wire.s3SourceConfig(); + assertEquals("LISTED_OBJECT_METADATA", s3.typeAsString()); + assertEquals("prefix/", s3.listedObjectMetadataConfig().keyPrefix()); + assertNull(s3.objectConfig()); + assertNull(s3.listedObjectsConfig()); + } + + @Test + void s3FlattenedJsonLinesUsesListedObjects() { + var s3 = options(DistributedMapSource.s3FlattenedJsonLines("s3://b/prefix/"), batchProcessor()) + .source() + .s3SourceConfig(); + assertEquals("LISTED_OBJECTS", s3.typeAsString()); + assertEquals("prefix/", s3.listedObjectsConfig().keyPrefix()); + assertEquals("JSON_LINES", s3.listedObjectsConfig().formatAsString()); + assertNull(s3.objectConfig()); + } + + @Test + void s3FlattenedCsvUsesListedObjectsAndCsvOptions() { + var source = DistributedMapSource.s3FlattenedCsv( + "s3://b/prefix", DistributedMapSource.CsvFormat.headers(List.of("a", "b"))); + var listed = options(source, batchProcessor()).source().s3SourceConfig().listedObjectsConfig(); + assertEquals("CSV", listed.formatAsString()); + assertEquals("GIVEN", listed.csvFormatOptions().headerLocationAsString()); + assertEquals(List.of("a", "b"), listed.csvFormatOptions().headers()); + } + + @Test + void s3ExpectedBucketOwnerSent() { + var source = DistributedMapSource.s3JsonLines("s3://b/k.jsonl").expectedBucketOwner("123456789012"); + var s3 = options(source, batchProcessor()).source().s3SourceConfig(); + assertEquals("123456789012", s3.expectedBucketOwner()); + } + + @Test + void readerSourceSerializesInitialState() { + var source = DistributedMapSource.reader("reader", Map.of("page", 0), null); + var reader = options(source, batchProcessor()).source().readerFunctionSourceConfig(); + assertEquals("reader", reader.functionName()); + assertEquals(SERDES.serialize(Map.of("page", 0)), reader.initialState()); + } + + @Test + void readerSourceRejectsInitialStateSerializedToNull() { + SerDes nullSerDes = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var source = DistributedMapSource.reader("reader", Map.of("page", 0), nullSerDes); + var exception = assertThrows(IllegalArgumentException.class, () -> options(source, batchProcessor())); + assertEquals("serialization of reader initialState produced no string", exception.getMessage()); + } + + @Test + void readerSourceWithoutInitialStateOmitsState() { + var reader = options(DistributedMapSource.reader("reader"), batchProcessor()) + .source() + .readerFunctionSourceConfig(); + assertEquals("reader", reader.functionName()); + assertNull(reader.initialState()); + } + + // ------------------------------------------------------------------ + // toOptions - CSV header handling + // ------------------------------------------------------------------ + + @Test + void csvHeadersMapToGivenAndAreSent() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", DistributedMapSource.CsvFormat.headers(List.of("a", "b"))); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("GIVEN", csv.headerLocationAsString()); + assertEquals(List.of("a", "b"), csv.headers()); + assertEquals("COMMA", csv.delimiterAsString()); + } + + @Test + void csvFirstRowOmitsHeaders() { + var source = DistributedMapSource.s3Csv("s3://b/data.csv", DistributedMapSource.CsvFormat.firstRow()); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("FIRST_ROW", csv.headerLocationAsString()); + assertFalse(csv.hasHeaders()); + assertEquals("COMMA", csv.delimiterAsString()); + } + + @Test + void csvExpectedColumnsStayClientSide() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", DistributedMapSource.CsvFormat.expectedColumns(List.of("a", "b"))); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("FIRST_ROW", csv.headerLocationAsString()); + assertFalse(csv.hasHeaders()); + } + + @Test + void csvDelimiterMapsToWireValue() { + var source = DistributedMapSource.s3Csv( + "s3://b/data.csv", + DistributedMapSource.CsvFormat.headers(List.of("a")) + .withDelimiter(DistributedMapSource.CsvDelimiter.PIPE)); + var csv = options(source, batchProcessor()) + .source() + .s3SourceConfig() + .objectConfig() + .csvFormatOptions(); + assertEquals("PIPE", csv.delimiterAsString()); + } + + // ------------------------------------------------------------------ + // toOptions - completion config + // ------------------------------------------------------------------ + + @Test + void completionPercentageAndSampleSizeSerialize() { + var config = DistributedMapConfig.builder() + .completionConfig(DistributedMapConfig.CompletionConfig.failurePercentage(5, 200)) + .build(); + var completion = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .completionConfig(); + assertEquals(5.0f, completion.toleratedFailurePercentage(), 0.0001f); + assertEquals(200, completion.minimumSampleSize().intValue()); + assertNull(completion.toleratedFailureCount()); + } + + @Test + void completionFailureCountSerializes() { + var config = DistributedMapConfig.builder() + .completionConfig(DistributedMapConfig.CompletionConfig.failureCount(2)) + .build(); + var completion = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .completionConfig(); + assertEquals(2, completion.toleratedFailureCount().intValue()); + } + + // ------------------------------------------------------------------ + // toOptions - destinations + // ------------------------------------------------------------------ + + @Test + void destinationBothSuccessAndFailureSerialize() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", false, true, null), + new DistributedMapDestination.Failure("out", "bad", true, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + var onSuccess = destination.onSuccess(); + assertEquals("S3", onSuccess.typeAsString()); + assertEquals(List.of("OUTPUT"), onSuccess.includeAsStrings()); + assertEquals("out", onSuccess.s3DestinationConfig().bucket()); + assertEquals("ok", onSuccess.s3DestinationConfig().keyPrefix()); + var onFailure = destination.onFailure(); + assertEquals(List.of("INPUT", "ERROR"), onFailure.includeAsStrings()); + assertEquals("bad", onFailure.s3DestinationConfig().keyPrefix()); + } + + @Test + void destinationOnlySuccessOmitsFailure() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", false, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + assertNotNull(destination.onSuccess()); + assertNull(destination.onFailure()); + } + + @Test + void destinationOnlyFailureOmitsSuccess() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", false, true, null))) + .build(); + var destination = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination(); + assertNotNull(destination.onFailure()); + assertNull(destination.onSuccess()); + } + + @Test + void allNullDestinationOmittedFromWire() { + var config = DistributedMapConfig.builder() + .destination(new DistributedMapDestination(null, null)) + .build(); + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false); + assertNull(options.destination()); + } + + @Test + void successDestinationIncludeInputAndOutputWithOwner() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", true, true, "123456789012"))) + .build(); + var onSuccess = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onSuccess(); + assertEquals(List.of("INPUT", "OUTPUT"), onSuccess.includeAsStrings()); + assertEquals("123456789012", onSuccess.s3DestinationConfig().expectedBucketOwner()); + } + + @Test + void successDestinationInputOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Success("out", "ok", true, false, null))) + .build(); + var onSuccess = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onSuccess(); + assertEquals(List.of("INPUT"), onSuccess.includeAsStrings()); + } + + @Test + void failureDestinationErrorOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", false, true, "123456789012"))) + .build(); + var onFailure = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onFailure(); + assertEquals(List.of("ERROR"), onFailure.includeAsStrings()); + assertEquals("123456789012", onFailure.s3DestinationConfig().expectedBucketOwner()); + } + + @Test + void failureDestinationInputOnly() { + var config = DistributedMapConfig.builder() + .destination(DistributedMapDestination.of( + new DistributedMapDestination.Failure("out", "bad", true, false, null))) + .build(); + var onFailure = options(DistributedMapSource.inline(List.of("a")), batchProcessor(), config, false) + .destination() + .onFailure(); + assertEquals(List.of("INPUT"), onFailure.includeAsStrings()); + } + + // ------------------------------------------------------------------ + // toOptions - result collection + // ------------------------------------------------------------------ + + @Test + void resultCollectionInlineWhenCollecting() { + var options = options( + DistributedMapSource.inline(List.of("a")), + batchProcessor(), + DistributedMapConfig.builder().build(), + true); + assertEquals("INLINE", options.resultCollection().modeAsString()); + } + + @Test + void resultCollectionOmittedWhenNotCollecting() { + var options = options(DistributedMapSource.inline(List.of("a")), batchProcessor()); + assertNull(options.resultCollection()); + } + + // ------------------------------------------------------------------ + // toSummary + // ------------------------------------------------------------------ + + @Test + void toSummaryMapsDetailFields() { + var details = detailsBuilder() + .successCount(5L) + .totalCount(5L) + .distributedMapRunArn("arn:aws:lambda:us-east-1:123456789012:map-run:abc") + .build(); + var summary = DistributedMapOperation.toSummary(DistributedMapStatus.SUCCEEDED, details); + assertEquals(DistributedMapStatus.SUCCEEDED, summary.status()); + assertTrue(summary.sourceExhausted()); + assertEquals(5L, summary.successCount()); + assertEquals(5L, summary.totalCount().longValue()); + assertEquals("arn:aws:lambda:us-east-1:123456789012:map-run:abc", summary.distributedMapRunArn()); + } + + @Test + void toSummaryCarriesTheRunError() { + var details = detailsBuilder() + .sourceExhausted(false) + .error(ErrorObject.builder() + .errorType("SourceFailed") + .errorMessage("bad object") + .errorData("{}") + .stackTrace(List.of("frame")) + .build()) + .build(); + var summary = DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details); + assertFalse(summary.sourceExhausted()); + assertEquals("SourceFailed", summary.error().errorType()); + assertEquals("bad object", summary.error().errorMessage()); + assertEquals("{}", summary.error().errorData()); + assertEquals(List.of("frame"), summary.error().stackTrace()); + } + + @Test + void toSummaryUsesTheOperationStatus() { + assertEquals( + DistributedMapStatus.FAILED, + DistributedMapOperation.toSummary( + DistributedMapStatus.FAILED, detailsBuilder().build()) + .status()); + } + + @Test + void toSummaryLeavesSourceExhaustedNullWhenAbsent() { + var details = DistributedMapDetails.builder() + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + assertNull(DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details) + .sourceExhausted()); + } + + @ParameterizedTest + @ValueSource(strings = {"SuccessCount", "FailureCount", "UnprocessedCount"}) + void toSummaryThrowsWhenATerminalCountIsMissing(String missing) { + var details = DistributedMapDetails.builder() + .successCount("SuccessCount".equals(missing) ? null : 1L) + .failureCount("FailureCount".equals(missing) ? null : 0L) + .unprocessedCount("UnprocessedCount".equals(missing) ? null : 0L) + .build(); + + var exception = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details)); + + assertEquals("missing distributed map " + missing + " from backend", exception.getMessage()); + } + + @Test + void toSummaryThrowsWhenASucceededRunHasNoTotalCount() { + var details = DistributedMapDetails.builder() + .sourceExhausted(true) + .successCount(1L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + + var exception = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toSummary(DistributedMapStatus.SUCCEEDED, details)); + + assertEquals("missing distributed map TotalCount from backend", exception.getMessage()); + } + + @Test + void toSummaryAcceptsAnAbsentTotalCountWhenTheRunDidNotSucceed() { + // TotalCount is required only for a succeeded run, so a failed run may omit it. + var details = DistributedMapDetails.builder() + .successCount(0L) + .failureCount(0L) + .unprocessedCount(0L) + .build(); + + assertNull(DistributedMapOperation.toSummary(DistributedMapStatus.FAILED, details) + .totalCount()); + } + + @Test + void toSummaryLeavesTheRunErrorNullWhenAbsent() { + assertNull(DistributedMapOperation.toSummary( + DistributedMapStatus.SUCCEEDED, detailsBuilder().build()) + .error()); + } + + // ------------------------------------------------------------------ + // toResult + // ------------------------------------------------------------------ + + @Test + void toResultParsesSucceededAndFailedItems() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of( + succeededItem("0", "42"), + failedItem( + "1", + ErrorObject.builder() + .errorType("E") + .errorMessage("boom") + .build()))) + .build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(Integer.class), SERDES); + assertEquals(List.of(42), result.results()); + assertEquals(1, result.errors().size()); + assertEquals("E", result.errors().get(0).errorType()); + assertEquals("boom", result.errors().get(0).errorMessage()); + } + + @Test + void toResultDecodesOutputExactlyOnceThroughSerdes() { + var serdes = new UpperSerDes(); + var details = + detailsBuilder().results(List.of(succeededItem("0", "abc"))).build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), serdes); + assertEquals(List.of("ABC"), result.results()); + assertEquals(1, serdes.deserializeCount); + } + + @Test + void toResultKeepsNullOutputNull() { + var details = + detailsBuilder().results(List.of(succeededItem("0", null))).build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + assertNull(result.succeeded().get(0).output()); + assertEquals(1, result.results().size()); + assertNull(result.results().get(0)); + } + + @Test + void toResultKeepsNullItemErrorFieldsNull() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem("0", ErrorObject.builder().build()))) + .build(); + var error = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES) + .errors() + .get(0); + assertNull(error.errorType()); + assertNull(error.errorMessage()); + } + + @Test + void toResultCarriesEveryItemErrorMember() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem( + "0", + ErrorObject.builder() + .errorType("E") + .errorMessage("boom") + .errorData("{\"k\":1}") + .stackTrace(List.of("frame-1", "frame-2")) + .build()))) + .build(); + var error = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES) + .errors() + .get(0); + assertEquals("E", error.errorType()); + assertEquals("boom", error.errorMessage()); + assertEquals("{\"k\":1}", error.errorData()); + assertEquals(List.of("frame-1", "frame-2"), error.stackTrace()); + } + + @Test + void toResultKeepsFailedItemErrorNullWhenAbsent() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of(failedItem("0", null))) + .build(); + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + assertNull(result.failed().get(0).error()); + assertEquals(1, result.errors().size()); + assertNull(result.errors().get(0)); + } + + @Test + void toResultMapsUnrecognizedItemStatusToUnknownAndKeepsTheOtherItems() { + var details = detailsBuilder() + .failureCount(1L) + .results(List.of( + succeededItem("0", "\"a\""), + DistributedMapResultItem.builder() + .itemId("1") + .status("BOGUS") + .build(), + failedItem("2", ErrorObject.builder().errorType("E").build()))) + .build(); + + var result = DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES); + + assertEquals(3, result.items().size()); + assertEquals( + DistributedMapResult.Item.Status.UNKNOWN_TO_SDK_VERSION, + result.items().get(1).status()); + assertNull(result.items().get(1).output()); + assertNull(result.items().get(1).error()); + // The unknown item is in items() but in none of the four filtered accessors. + assertEquals( + List.of("0"), + result.succeeded().stream() + .map(DistributedMapResult.Item::itemId) + .toList()); + assertEquals( + List.of("2"), + result.failed().stream().map(DistributedMapResult.Item::itemId).toList()); + assertEquals(List.of("a"), result.results()); + assertEquals(1, result.errors().size()); + } + + @Test + void toResultThrowsOnMissingItemStatus() { + var details = detailsBuilder() + .results(List.of(DistributedMapResultItem.builder().itemId("0").build())) + .build(); + assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES)); + } + + @ParameterizedTest + @NullAndEmptySource + void toResultThrowsOnMissingItemId(String itemId) { + var details = detailsBuilder() + .results(List.of(DistributedMapResultItem.builder() + .itemId(itemId) + .status("SUCCEEDED") + .output("\"x\"") + .build())) + .build(); + var e = assertThrows( + DurableExecutionException.class, + () -> DistributedMapOperation.toResult( + DistributedMapStatus.SUCCEEDED, details, TypeToken.get(String.class), SERDES)); + assertEquals("missing distributed map item id from backend", e.getMessage()); + } +}