diff --git a/.changelog/feature-timechange.json b/.changelog/feature-timechange.json new file mode 100644 index 000000000000..7f3d88deca18 --- /dev/null +++ b/.changelog/feature-timechange.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "aws-cpp-sdk-core", + "contributor": "kaiion", + "description": "Add clock skew header that adjusts request signing timestamps to match the service clock" +} diff --git a/src/aws-cpp-sdk-core/include/aws/core/internal/ClockSkew.h b/src/aws-cpp-sdk-core/include/aws/core/internal/ClockSkew.h new file mode 100644 index 000000000000..4b0dda5c4419 --- /dev/null +++ b/src/aws-cpp-sdk-core/include/aws/core/internal/ClockSkew.h @@ -0,0 +1,161 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ + namespace Internal + { + // AttemptSkew is owned by the client, not here. + + static const std::chrono::milliseconds CLOCK_SKEW_DETECTION_THRESHOLD = std::chrono::minutes(4); + static const std::chrono::milliseconds CLOCK_SKEW_MAX_TRUSTED_REQUEST_DURATION = std::chrono::minutes(15); + static const char CLOCK_SKEW_AGE_HEADER[] = "age"; + + struct ClockSkewMeasurement + { + Aws::Crt::Optional serverTime; // empty when the response carries no usable Date header + Aws::Utils::DateTime timeRequestSent; + Aws::Utils::DateTime timeResponseReceived; + bool servedFromCache; + + ClockSkewMeasurement(const Aws::Crt::Optional& serverTime, + const Aws::Utils::DateTime& timeRequestSent, + const Aws::Utils::DateTime& timeResponseReceived, + bool servedFromCache) + : serverTime(serverTime), + timeRequestSent(timeRequestSent), + timeResponseReceived(timeResponseReceived), + servedFromCache(servedFromCache) {} + }; + + inline ClockSkewMeasurement MakeClockSkewMeasurement(const Aws::Http::HeaderValueCollection& headers, + const Aws::Utils::DateTime& timeRequestSent, + const Aws::Utils::DateTime& timeResponseReceived) + { + const auto dateIt = headers.find(Aws::Utils::StringUtils::ToLower(Aws::Http::DATE_HEADER)); + Aws::Crt::Optional serverTime; + if (dateIt != headers.end()) + { + const Aws::Utils::DateTime parsed(dateIt->second.c_str(), Aws::Utils::DateFormat::AutoDetect); + if (parsed.WasParseSuccessful()) + { + serverTime = parsed; + } + } + const bool servedFromCache = headers.find(CLOCK_SKEW_AGE_HEADER) != headers.end(); + return ClockSkewMeasurement(serverTime, timeRequestSent, timeResponseReceived, servedFromCache); + } + + inline Aws::Crt::Optional ComputeClockSkewCandidate(const ClockSkewMeasurement& measurement) + { + if (!measurement.serverTime.has_value() || measurement.servedFromCache) + { + return Aws::Crt::Optional{}; + } + + const int64_t sentMs = measurement.timeRequestSent.Millis(); + const int64_t receivedMs = measurement.timeResponseReceived.Millis(); + const int64_t serverMs = measurement.serverTime.value().Millis(); + + const std::chrono::milliseconds elapsed{receivedMs - sentMs}; + if (elapsed > CLOCK_SKEW_MAX_TRUSTED_REQUEST_DURATION) + { + return Aws::Crt::Optional{}; + } + + // Server Date is assumed to be from the round-trip midpoint (NTP offset, RFC 5905 s8). + const int64_t midpointMs = sentMs + (receivedMs - sentMs) / 2; + return Aws::Crt::Optional(std::chrono::milliseconds{serverMs - midpointMs}); + } + + // Clock-skew error codes. Matched by CoreErrors enum where one exists; AuthFailure (EC2) has no + // CoreErrors enum, so it is matched by name -- the same enum+name shape as IsThrottlingResponse. + inline bool IsClockSkewError(const Aws::Client::AWSError& error) + { + switch (error.GetErrorType()) + { + case Aws::Client::CoreErrors::INVALID_SIGNATURE: + case Aws::Client::CoreErrors::SIGNATURE_DOES_NOT_MATCH: + case Aws::Client::CoreErrors::REQUEST_TIME_TOO_SKEWED: + case Aws::Client::CoreErrors::ACCESS_DENIED: + return true; + default: + return error.GetExceptionName() == "AuthFailure"; + } + } + + struct ClockSkewAdjustment + { + bool skewExceedsThreshold = false; // caller ANDs this with the error-code check before retrying + std::chrono::milliseconds skew{0}; + }; + + class AWS_CORE_LOCAL ClientSkew + { + public: + explicit ClientSkew(std::chrono::milliseconds initial) : m_skew(initial) {} + + ClientSkew(const ClientSkew& other) : m_skew(other.m_skew.load()) {} + ClientSkew(ClientSkew&& other) noexcept : m_skew(other.m_skew.load()) {} + ClientSkew& operator=(const ClientSkew& other) + { + if (this != &other) + { + m_skew = other.m_skew.load(); + } + return *this; + } + ClientSkew& operator=(ClientSkew&& other) noexcept + { + m_skew = other.m_skew.load(); + return *this; + } + ~ClientSkew() = default; + + std::chrono::milliseconds Load() const { return m_skew.load(); } + + // Runs on every response; a surviving candidate is stored, so a stale value self-heals. + Aws::Crt::Optional RecordResponse(const ClockSkewMeasurement& measurement) + { + Aws::Crt::Optional candidate = ComputeClockSkewCandidate(measurement); + if (candidate.has_value()) + { + m_skew.store(candidate.value()); + } + return candidate; + } + + ClockSkewAdjustment EvaluateFailure(const ClockSkewMeasurement& measurement, std::chrono::milliseconds attemptSkew) + { + const Aws::Crt::Optional candidate = RecordResponse(measurement); + ClockSkewAdjustment adjustment; + adjustment.skew = m_skew.load(); + // Retryable when the applied skew is off from the observed skew by more than the threshold. + if (candidate.has_value()) + { + const std::chrono::milliseconds absDelta = Aws::chrono::abs(attemptSkew - candidate.value()); + adjustment.skewExceedsThreshold = absDelta > CLOCK_SKEW_DETECTION_THRESHOLD; + } + return adjustment; + } + + private: + std::atomic m_skew; + }; + } // namespace Internal +} // namespace Aws diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/Chrono.h b/src/aws-cpp-sdk-core/include/aws/core/utils/Chrono.h new file mode 100644 index 000000000000..120b575b8eaf --- /dev/null +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/Chrono.h @@ -0,0 +1,40 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include + +// TODO: delete the fallback branch once the SDK's minimum standard is C++17. +#if defined(__cpp_lib_chrono) && __cpp_lib_chrono >= 201510L + +namespace Aws +{ + namespace chrono + { + using std::chrono::abs; + } +} + +#else + +#include +#include + +namespace Aws +{ + namespace chrono + { + template + constexpr typename std::enable_if::is_signed, + std::chrono::duration>::type + abs(std::chrono::duration d) + { + return d >= d.zero() ? d : -d; + } + } +} + +#endif diff --git a/tests/aws-cpp-sdk-core-tests/CMakeLists.txt b/tests/aws-cpp-sdk-core-tests/CMakeLists.txt index 4ce72d351177..22bf90bfd8ce 100644 --- a/tests/aws-cpp-sdk-core-tests/CMakeLists.txt +++ b/tests/aws-cpp-sdk-core-tests/CMakeLists.txt @@ -126,6 +126,9 @@ endif() target_link_libraries(${PROJECT_NAME} ${PROJECT_LIBS} ${CLIENT_LIBS}) +target_compile_definitions(${PROJECT_NAME} PRIVATE + "CLOCK_SKEW_TEST_CASES_PATH=\"${CMAKE_CURRENT_SOURCE_DIR}/resources/clock-skew-test-cases.json\"") + add_custom_command(TARGET aws-cpp-sdk-core-tests PRE_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/resources ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/aws-cpp-sdk-core-tests/resources/clock-skew-test-cases.json b/tests/aws-cpp-sdk-core-tests/resources/clock-skew-test-cases.json new file mode 100644 index 000000000000..5ac9f44cd888 --- /dev/null +++ b/tests/aws-cpp-sdk-core-tests/resources/clock-skew-test-cases.json @@ -0,0 +1,340 @@ +{ + "documentation": "Test cases for clock skew correction. Each test defines a sequence of operations on a single client. Each operation defines a sequence of attempts (the retry loop). The SDK under test must use the provided 'clientTime' values instead of the real clock. 'initialClientSkew' is the value of ClientSkew at the start of the test (0 for the first operation unless the test specifies otherwise, carried forward for subsequent operations). After all attempts in an operation complete, 'expectedClientSkew' is asserted.", + "tests": [ + { + "description": "Clocks agree, request succeeds on first attempt, ClientSkew updated to 0", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:00:01 GMT" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success" + } + ] + }, + { + "description": "Definite clock skew error on first attempt, retry succeeds, ClientSkew persists to next operation", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 403, + "headers": { + "Date": "Thu, 01 Jan 2026 00:05:01 GMT" + }, + "errorCode": "RequestTimeTooSkewed" + } + }, + { + "clientTimeAtSend": "2026-01-01T00:00:02Z", + "clientTimeAtReceive": "2026-01-01T00:00:04Z", + "expectedSigningTime": "2026-01-01T00:05:02Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:05:03 GMT" + } + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "success" + }, + { + "initialClientSkew": 300, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:01:00Z", + "clientTimeAtReceive": "2026-01-01T00:01:02Z", + "expectedSigningTime": "2026-01-01T00:06:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:06:01 GMT" + } + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "success" + } + ] + }, + { + "description": "Clock skew error with no retry budget still updates ClientSkew", + "operations": [ + { + "initialClientSkew": 0, + "maxAttempts": 1, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 403, + "headers": { + "Date": "Thu, 01 Jan 2026 00:05:01 GMT" + }, + "errorCode": "RequestTimeTooSkewed" + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "error" + } + ] + }, + { + "description": "Delayed response exceeding trust threshold does not update ClientSkew", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:16:00Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:10:00 GMT" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success" + }, + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:17:00Z", + "clientTimeAtReceive": "2026-01-01T00:17:02Z", + "expectedSigningTime": "2026-01-01T00:17:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:22:01 GMT" + } + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "success" + } + ] + }, + { + "description": "Stale ClientSkew is silently corrected by a successful response without error or retry", + "operations": [ + { + "initialClientSkew": 300, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:05:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:00:01 GMT" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success" + } + ] + }, + { + "description": "Possible clock skew error with skew below detection threshold is not retried as clock skew", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 400, + "headers": { + "Date": "Thu, 01 Jan 2026 00:02:01 GMT" + }, + "errorCode": "InvalidSignatureException" + } + } + ], + "expectedClientSkew": 120, + "expectedOutcome": "error", + "documentation": "Skew is ~2 minutes which is below the 4-minute detection threshold. The error is NOT retried as a clock skew candidate. ClientSkew is still updated from the Date header (unconditional recording)." + } + ] + }, + { + "description": "Possible clock skew error with skew above detection threshold is retried and succeeds", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 400, + "headers": { + "Date": "Thu, 01 Jan 2026 00:05:01 GMT" + }, + "errorCode": "InvalidSignatureException" + } + }, + { + "clientTimeAtSend": "2026-01-01T00:00:02Z", + "clientTimeAtReceive": "2026-01-01T00:00:04Z", + "expectedSigningTime": "2026-01-01T00:05:02Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 00:05:03 GMT" + } + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "success" + } + ] + }, + { + "description": "Clock skew error with no Date header is not treated as a candidate, ClientSkew is not updated", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T00:00:00Z", + "clientTimeAtReceive": "2026-01-01T00:00:02Z", + "expectedSigningTime": "2026-01-01T00:00:00Z", + "response": { + "statusCode": 403, + "headers": {}, + "errorCode": "RequestTimeTooSkewed" + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "error" + } + ] + }, + { + "description": "Cached response with Age header does not update ClientSkew", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T16:51:00Z", + "clientTimeAtReceive": "2026-01-01T16:51:02Z", + "expectedSigningTime": "2026-01-01T16:51:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 12:03:01 GMT", + "Age": "17280" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success", + "documentation": "The Date header is ~4.8 hours in the past because the response is served from a CDN cache. The Age header (17280 seconds) signals this is a cached response. The SDK MUST NOT compute skew from this response." + }, + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T16:51:10Z", + "clientTimeAtReceive": "2026-01-01T16:51:12Z", + "expectedSigningTime": "2026-01-01T16:51:10Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 16:56:11 GMT" + } + } + } + ], + "expectedClientSkew": 300, + "expectedOutcome": "success", + "documentation": "Subsequent non-cached response (no Age header) with actual 5-minute skew is learned normally." + } + ] + }, + { + "description": "Cached response with Age header does not poison ClientSkew even when followed by a clock skew error", + "operations": [ + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T16:51:00Z", + "clientTimeAtReceive": "2026-01-01T16:51:02Z", + "expectedSigningTime": "2026-01-01T16:51:00Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 12:03:01 GMT", + "Age": "17280" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success", + "documentation": "First operation returns a cached response. ClientSkew stays at 0." + }, + { + "initialClientSkew": 0, + "attempts": [ + { + "clientTimeAtSend": "2026-01-01T16:51:10Z", + "clientTimeAtReceive": "2026-01-01T16:51:12Z", + "expectedSigningTime": "2026-01-01T16:51:10Z", + "response": { + "statusCode": 200, + "headers": { + "Date": "Thu, 01 Jan 2026 16:51:11 GMT" + } + } + } + ], + "expectedClientSkew": 0, + "expectedOutcome": "success", + "documentation": "Second operation hits a non-cached endpoint. Signing time is correct (no false skew applied). Request succeeds." + } + ] + } + ] +} diff --git a/tests/aws-cpp-sdk-core-tests/utils/ClockSkewTest.cpp b/tests/aws-cpp-sdk-core-tests/utils/ClockSkewTest.cpp new file mode 100644 index 000000000000..046c78e76c06 --- /dev/null +++ b/tests/aws-cpp-sdk-core-tests/utils/ClockSkewTest.cpp @@ -0,0 +1,148 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// Path to the test cases, injected by CMake. +#ifndef CLOCK_SKEW_TEST_CASES_PATH +#define CLOCK_SKEW_TEST_CASES_PATH "" +#endif + +using namespace Aws::Internal; +using Aws::Utils::DateTime; +using Aws::Utils::DateFormat; +using Aws::Utils::Json::JsonValue; +using Aws::Utils::Json::JsonView; + +namespace +{ + ClockSkewMeasurement MeasurementFromAttempt(const JsonView& attempt) + { + // Route the corpus through the production MakeClockSkewMeasurement so it exercises the real parse + // path. Headers are lowercased here as a real HttpResponse stores them. + const JsonView jsonHeaders = attempt.GetObject("response").GetObject("headers"); + Aws::Http::HeaderValueCollection headers; + for (const auto& header : jsonHeaders.GetAllObjects()) + { + headers.emplace(Aws::Utils::StringUtils::ToLower(header.first.c_str()), header.second.AsString()); + } + return MakeClockSkewMeasurement( + headers, + DateTime(attempt.GetString("clientTimeAtSend"), DateFormat::ISO_8601), + DateTime(attempt.GetString("clientTimeAtReceive"), DateFormat::ISO_8601)); + } + + bool IsSuccessfulResponse(int statusCode) { return statusCode >= 200 && statusCode <= 299; } +} + +class ClockSkewTest : public Aws::Testing::AwsCppSdkGTestSuite, public testing::WithParamInterface +{ +public: + static const size_t TEST_CASE_COUNT; +}; +const size_t ClockSkewTest::TEST_CASE_COUNT = 10; + +TEST_P(ClockSkewTest, RunTestCase) +{ + const Aws::String path = CLOCK_SKEW_TEST_CASES_PATH; + ASSERT_FALSE(path.empty()) << "CLOCK_SKEW_TEST_CASES_PATH was not defined at compile time"; + + Aws::IFStream inputFile(path.c_str()); + ASSERT_TRUE(inputFile.good()) << "Could not open clock skew test cases at " << path; + + JsonValue doc(inputFile); + ASSERT_TRUE(doc.WasParseSuccessful()) << "Failed to parse clock skew test cases JSON at " << path; + + const auto tests = doc.View().GetArray("tests"); + ASSERT_EQ(TEST_CASE_COUNT, tests.GetLength()); + + const size_t testIdx = GetParam(); + const JsonView testCase = tests[testIdx]; + SCOPED_TRACE(Aws::String("TEST CASE # ") + Aws::Utils::StringUtils::to_string(testIdx) + ": " + testCase.GetString("description")); + + const auto operations = testCase.GetArray("operations"); + ASSERT_GT(operations.GetLength(), 0u); + + // One ClientSkew shared across the test's operations, so the multi-operation cases prove persistence. + ClientSkew clientSkew(std::chrono::milliseconds{static_cast(operations[0].GetInteger("initialClientSkew")) * 1000}); + + for (size_t o = 0; o < operations.GetLength(); ++o) + { + const JsonView operation = operations[o]; + + // The value persisted from the prior operation must match this operation's initialClientSkew. + const std::chrono::milliseconds initialSkew{static_cast(operation.GetInteger("initialClientSkew")) * 1000}; + EXPECT_EQ(initialSkew.count(), clientSkew.Load().count()) << "ClientSkew not persisted into operation " << o; + + // AttemptSkew is the per-operation copy seeded from ClientSkew, updated after each attempt. + std::chrono::milliseconds attemptSkew = clientSkew.Load(); + + const auto attempts = operation.GetArray("attempts"); + // maxAttempts (when present) caps the retry budget; absent means the budget did not run out. + const bool budgetLimited = operation.KeyExists("maxAttempts") && + static_cast(operation.GetInteger("maxAttempts")) <= attempts.GetLength(); + + bool operationSucceeded = false; + for (size_t a = 0; a < attempts.GetLength(); ++a) + { + const JsonView attempt = attempts[a]; + + const DateTime signingTime = DateTime(attempt.GetString("clientTimeAtSend"), DateFormat::ISO_8601) + attemptSkew; + const DateTime expectedSigningTime = DateTime(attempt.GetString("expectedSigningTime"), DateFormat::ISO_8601); + EXPECT_EQ(expectedSigningTime.Millis(), signingTime.Millis()) + << "signing time mismatch on attempt " << a << " of operation " << o; + + const ClockSkewMeasurement measurement = MeasurementFromAttempt(attempt); + const int statusCode = attempt.GetObject("response").GetInteger("statusCode"); + if (IsSuccessfulResponse(statusCode)) + { + clientSkew.RecordResponse(measurement); + operationSucceeded = true; + } + else + { + const ClockSkewAdjustment adjustment = clientSkew.EvaluateFailure(measurement, attemptSkew); + // Map the wire error code to a CoreErrors enum the way the real error marshaller does. + const Aws::Client::AWSError error = + Aws::Client::CoreErrorsMapper::GetErrorForName(attempt.GetObject("response").GetString("errorCode").c_str()); + const bool retriedForSkew = IsClockSkewError(error) && adjustment.skewExceedsThreshold; + + if (a + 1 < attempts.GetLength()) + { + // A further attempt exists, so the SDK must have decided to retry this skew error. + EXPECT_TRUE(retriedForSkew) << "expected a clock-skew retry after attempt " << a << " of operation " << o; + } + else if (!budgetLimited) + { + // Final attempt with budget remaining, so the SDK must have declined to retry. + EXPECT_FALSE(retriedForSkew) << "expected no clock-skew retry after final attempt of operation " << o; + } + } + attemptSkew = clientSkew.Load(); + } + + const Aws::String expectedOutcome = operation.GetString("expectedOutcome"); + EXPECT_EQ(expectedOutcome, operationSucceeded ? Aws::String("success") : Aws::String("error")) + << "outcome mismatch on operation " << o; + + const std::chrono::milliseconds expectedClientSkew{static_cast(operation.GetInteger("expectedClientSkew")) * 1000}; + EXPECT_EQ(expectedClientSkew.count(), clientSkew.Load().count()) + << "ClientSkew mismatch after operation " << o; + } +} + +INSTANTIATE_TEST_SUITE_P(ClockSkew, ClockSkewTest, ::testing::Range((size_t) 0u, ClockSkewTest::TEST_CASE_COUNT));