From 9589eb845b20d3ebaccc751babd76986aba60985 Mon Sep 17 00:00:00 2001 From: kaiion <43284426+kai-ion@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:15:51 +0000 Subject: [PATCH] Wire clock-skew correction into the legacy and smithy client pipelines --- .changelog/feature-clock-skew-correction.json | 6 ++ .../include/aws/core/client/AWSClient.h | 8 +- .../include/aws/core/http/HttpRequest.h | 13 +++ .../include/smithy/client/AwsSmithyClient.h | 5 -- .../AwsSmithyClientAsyncRequestContext.h | 4 + .../smithy/client/AwsSmithyClientBase.h | 9 +- .../client/common/AwsSmithyRequestSigning.h | 82 ------------------- .../identity/signer/built-in/SigV4aSigner.h | 3 +- .../signer/AWSAuthEventStreamV4Signer.cpp | 2 +- .../source/auth/signer/AWSAuthV4Signer.cpp | 5 +- .../source/client/AWSClient.cpp | 58 +++++++------ .../source/client/ClientConfiguration.cpp | 12 +++ .../smithy/client/AwsSmithyClientBase.cpp | 51 ++++++++++-- .../aws/client/AWSClientTest.cpp | 21 +++-- .../monitoring/MonitoringTest.cpp | 6 +- .../testing/mocks/aws/client/MockAWSClient.h | 26 ++++++ 16 files changed, 175 insertions(+), 136 deletions(-) create mode 100644 .changelog/feature-clock-skew-correction.json diff --git a/.changelog/feature-clock-skew-correction.json b/.changelog/feature-clock-skew-correction.json new file mode 100644 index 000000000000..0acbae5fc1e2 --- /dev/null +++ b/.changelog/feature-clock-skew-correction.json @@ -0,0 +1,6 @@ +{ + "type": "feature", + "category": "aws-cpp-sdk-core", + "contributor": "kaiion", + "description": "Add clock skew correction: the SDK adjusts request signing timestamps by the observed client-to-service skew and retries signature errors caused by skew, so requests keep working when the client clock is off. Disable with AWS_DISABLE_CLOCK_SKEW_CORRECTION." +} diff --git a/src/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h b/src/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h index 1815b88abd9f..238535033580 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h +++ b/src/aws-cpp-sdk-core/include/aws/core/client/AWSClient.h @@ -60,6 +60,11 @@ namespace Aws class AmazonWebServiceRequest; + namespace Internal + { + class ClientSkew; + } + namespace Client { template @@ -345,7 +350,7 @@ namespace Aws * Try to adjust signer's clock * return true if signer's clock is adjusted, false otherwise. */ - bool AdjustClockSkew(HttpResponseOutcome& outcome, const char* signerName) const; + bool AdjustClockSkew(HttpResponseOutcome& outcome, const Aws::Utils::DateTime& timeRequestSent, const Aws::Utils::DateTime& timeResponseReceived, std::chrono::milliseconds attemptSkew) const; void AddHeadersToRequest(const std::shared_ptr& httpRequest, const Http::HeaderValueCollection& headerValues) const; void AddContentBodyToRequest(const std::shared_ptr& httpRequest, const std::shared_ptr& body, bool needsContentMd5 = false, bool isChunked = false) const; @@ -359,6 +364,7 @@ namespace Aws std::shared_ptr m_hash; long m_requestTimeoutMs; bool m_enableClockSkewAdjustment; + mutable std::shared_ptr m_clientSkew; Aws::String m_serviceName = "AWSBaseClient"; Aws::Client::RequestCompressionConfig m_requestCompressionConfig; std::shared_ptr m_userAgentInterceptor; diff --git a/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h b/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h index cb21d943d253..020425e82948 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h +++ b/src/aws-cpp-sdk-core/include/aws/core/http/HttpRequest.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -558,6 +560,16 @@ namespace Aws */ inline void SetSigningRegion(const Aws::String& region) { m_signingRegion = region; } + /** + * Gets the per-attempt signing timestamp override set for clock-skew correction, if any. + */ + inline const Aws::Crt::Optional& GetSigningTimestampOverride() const { return m_signingTimestampOverride; } + /** + * Sets an explicit signing timestamp for this attempt (now() + AttemptSkew). The signer uses it + * instead of its own clock; set per attempt so concurrent operations don't share skew state. + */ + inline void SetSigningTimestampOverride(const Aws::Utils::DateTime& signingTime) { m_signingTimestampOverride = signingTime; } + /** * Add a request metric * @param key, HttpClientMetricsKey defined in HttpClientMetrics.cpp @@ -618,6 +630,7 @@ namespace Aws DataSentEventHandler m_onDataSent; ContinueRequestHandler m_continueRequest; Aws::String m_signingRegion; + Aws::Crt::Optional m_signingTimestampOverride; Aws::String m_signingAccessKey; Aws::String m_resolvedRemoteHost; Aws::Monitoring::HttpClientMetricsCollection m_httpRequestMetrics; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClient.h b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClient.h index b5760248c530..fb5e388a9242 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClient.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClient.h @@ -229,11 +229,6 @@ namespace client return AwsClientRequestSigning::SignEventMessage(message, seed, ctx, m_authSchemes); } - bool AdjustClockSkew(HttpResponseOutcome& outcome, const AuthSchemeOption& authSchemeOption) const override - { - return AwsClientRequestSigning::AdjustClockSkew(outcome, authSchemeOption, m_authSchemes); - } - IdentityOutcome ResolveIdentity(const AwsSmithyClientAsyncRequestContext& ctx) const override { return AwsClientRequestSigning::ResolveIdentity(ctx, m_authSchemes); } diff --git a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientAsyncRequestContext.h b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientAsyncRequestContext.h index 1796f541916c..603feb589daf 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientAsyncRequestContext.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientAsyncRequestContext.h @@ -66,6 +66,10 @@ namespace smithy Aws::Crt::Optional m_lastError; + std::chrono::milliseconds m_attemptSkew{0}; + Aws::Utils::DateTime m_timeRequestSent; + Aws::Utils::DateTime m_timeResponseReceived; + size_t m_retryCount; Aws::Vector m_monitoringContexts; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h index b60c808d9f80..2300276c0e76 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h @@ -58,6 +58,11 @@ namespace Aws } class AmazonWebServiceRequest; + + namespace Internal + { + class ClientSkew; + } } namespace Aws @@ -222,7 +227,8 @@ namespace client virtual ResolveEndpointOutcome ResolveEndpoint(const Aws::Endpoint::EndpointParameters& endpointParameters, EndpointUpdateCallback&& epCallback) const = 0; virtual SelectAuthSchemeOptionOutcome SelectAuthSchemeOption(const AwsSmithyClientAsyncRequestContext& ctx) const = 0; virtual SigningOutcome SignHttpRequest(std::shared_ptr httpRequest, const AwsSmithyClientAsyncRequestContext& ctx) const = 0; - virtual bool AdjustClockSkew(HttpResponseOutcome& outcome, const AuthSchemeOption& authSchemeOption) const = 0; + bool AdjustClockSkew(HttpResponseOutcome& outcome, const AwsSmithyClientAsyncRequestContext& ctx) const; + void RecordClockSkew(const Aws::Http::HttpResponse& response, const AwsSmithyClientAsyncRequestContext& ctx) const; virtual IdentityOutcome ResolveIdentity(const AwsSmithyClientAsyncRequestContext& ctx) const = 0; virtual GetContextEndpointParametersOutcome GetContextEndpointParameters(const AwsSmithyClientAsyncRequestContext& ctx) const = 0; AwsSmithyClientBase::ResolveEndpointOutcome ResolveEndpointFromRequest( @@ -241,6 +247,7 @@ namespace client std::shared_ptr m_errorMarshaller; Aws::Vector> m_interceptors{}; std::shared_ptr m_userAgentInterceptor; + mutable std::shared_ptr m_clientSkew; private: void UpdateAuthSchemeFromEndpoint(const Aws::Endpoint::AWSEndpoint& endpoint, AuthSchemeOption& authscheme) const; diff --git a/src/aws-cpp-sdk-core/include/smithy/client/common/AwsSmithyRequestSigning.h b/src/aws-cpp-sdk-core/include/smithy/client/common/AwsSmithyRequestSigning.h index 682646a6afd6..3ed803c40658 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/common/AwsSmithyRequestSigning.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/common/AwsSmithyRequestSigning.h @@ -26,10 +26,6 @@ namespace smithy { static const char AWS_SMITHY_CLIENT_SIGNING_TAG[] = "AwsClientRequestSigning"; - //4 Minutes - static const std::chrono::milliseconds TIME_DIFF_MAX = std::chrono::minutes(4); - //-4 Minutes - static const std::chrono::milliseconds TIME_DIFF_MIN = std::chrono::minutes(-4); template class AwsClientRequestSigning @@ -142,28 +138,6 @@ namespace smithy return {authScheme}; } - static bool AdjustClockSkew(HttpResponseOutcome& outcome, const AuthSchemeOption& authSchemeOption, - const Aws::UnorderedMap& authSchemes) - { - assert(!outcome.IsSuccess()); - AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_SIGNING_TAG, "If the signature check failed. This could be because of a time skew. Attempting to adjust the signer."); - - using DateTime = Aws::Utils::DateTime; - DateTime serverTime = smithy::client::Utils::GetServerTimeFromError(outcome.GetError()); - - auto authSchemeOutcome = ResolveAuthScheme(authSchemeOption, authSchemes); - if (!authSchemeOutcome.IsSuccess()) - { - return false; - } - - ClockSkewVisitor visitor(outcome, serverTime, authSchemeOption); - AuthSchemesVariantT authScheme = authSchemeOutcome.GetResult().value(); - authScheme.Visit(visitor); - - return visitor.m_resultShouldWait; - } - protected: struct IdentityVisitor @@ -360,61 +334,5 @@ namespace smithy return std::move(*visitor.result); } - struct ClockSkewVisitor - { - using DateTime = Aws::Utils::DateTime; - using DateFormat = Aws::Utils::DateFormat; - using ClientError = Aws::Client::AWSError; - - ClockSkewVisitor(HttpResponseOutcome& outcome, const DateTime& serverTime, const AuthSchemeOption& targetAuthSchemeOption) - : m_outcome(outcome), m_serverTime(serverTime), m_targetAuthSchemeOption(targetAuthSchemeOption) - { - } - - bool m_resultShouldWait = false; - HttpResponseOutcome& m_outcome; - const Aws::Utils::DateTime& m_serverTime; - const AuthSchemeOption& m_targetAuthSchemeOption; - - template - void operator()(AuthSchemeAlternativeT& authScheme) - { - // Auth Scheme Variant alternative contains the requested auth option - assert(strcmp(authScheme.schemeId, m_targetAuthSchemeOption.schemeId) == 0); - - using IdentityT = typename std::remove_reference::type::IdentityT; - using Signer = AwsSignerBase; - - std::shared_ptr signer = authScheme.signer(); - if (!signer) - { - AWS_LOGSTREAM_ERROR(AWS_SMITHY_CLIENT_SIGNING_TAG, "Failed to adjust signing clock skew. Signer is null."); - return; - } - - const auto signingTimestamp = signer->GetSigningTimestamp(); - if (!m_serverTime.WasParseSuccessful() || m_serverTime == DateTime()) - { - AWS_LOGSTREAM_DEBUG(AWS_SMITHY_CLIENT_SIGNING_TAG, "Date header was not found in the response, can't attempt to detect clock skew"); - return; - } - - AWS_LOGSTREAM_DEBUG(AWS_SMITHY_CLIENT_SIGNING_TAG, "Server time is " << m_serverTime.ToGmtString(DateFormat::RFC822) << ", while client time is " << DateTime::Now().ToGmtString(DateFormat::RFC822)); - auto diff = DateTime::Diff(m_serverTime, signingTimestamp); - //only try again if clock skew was the cause of the error. - if (diff >= TIME_DIFF_MAX || diff <= TIME_DIFF_MIN) - { - diff = DateTime::Diff(m_serverTime, DateTime::Now()); - AWS_LOGSTREAM_INFO(AWS_SMITHY_CLIENT_SIGNING_TAG, "Computed time difference as " << diff.count() << " milliseconds. Adjusting signer with the skew."); - signer->SetClockSkew(diff); - ClientError newError(m_outcome.GetError()); - newError.SetRetryableType(Aws::Client::RetryableType::RETRYABLE); - - m_outcome = std::move(newError); - m_resultShouldWait = true; - } - } - }; - }; } \ No newline at end of file diff --git a/src/aws-cpp-sdk-core/include/smithy/identity/signer/built-in/SigV4aSigner.h b/src/aws-cpp-sdk-core/include/smithy/identity/signer/built-in/SigV4aSigner.h index 0e93d7e6a1a3..cba522d93992 100644 --- a/src/aws-cpp-sdk-core/include/smithy/identity/signer/built-in/SigV4aSigner.h +++ b/src/aws-cpp-sdk-core/include/smithy/identity/signer/built-in/SigV4aSigner.h @@ -97,7 +97,8 @@ namespace smithy { awsSigningConfig.SetSignatureType(signatureType); awsSigningConfig.SetRegion(serviceName.c_str()); awsSigningConfig.SetService(region.c_str()); - awsSigningConfig.SetSigningTimepoint(GetSigningTimestamp().UnderlyingTimestamp()); + const Aws::Utils::DateTime sigV4aSigningTime = request.GetSigningTimestampOverride() ? request.GetSigningTimestampOverride().value() : GetSigningTimestamp(); + awsSigningConfig.SetSigningTimepoint(sigV4aSigningTime.UnderlyingTimestamp()); awsSigningConfig.SetUseDoubleUriEncode(m_urlEscape); awsSigningConfig.SetShouldNormalizeUriPath(true); awsSigningConfig.SetOmitSessionToken(false); diff --git a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthEventStreamV4Signer.cpp b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthEventStreamV4Signer.cpp index cf39123eebf7..2d938a7daf06 100644 --- a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthEventStreamV4Signer.cpp +++ b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthEventStreamV4Signer.cpp @@ -79,7 +79,7 @@ bool AWSAuthEventStreamV4Signer::SignRequestWithCreds(Http::HttpRequest& request request.SetHeaderValue(Aws::Auth::AWSAuthHelper::X_AMZ_CONTENT_SHA256, EVENT_STREAM_CONTENT_SHA256); //calculate date header to use in internal signature (this also goes into date header). - DateTime now = GetSigningTimestamp(); + DateTime now = request.GetSigningTimestampOverride() ? request.GetSigningTimestampOverride().value() : GetSigningTimestamp(); Aws::String dateHeaderValue = now.ToGmtString(DateFormat::ISO_8601_BASIC); request.SetHeaderValue(AWS_DATE_HEADER, dateHeaderValue); diff --git a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp index 7ebf65689524..841b12822ff9 100644 --- a/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp +++ b/src/aws-cpp-sdk-core/source/auth/signer/AWSAuthV4Signer.cpp @@ -96,7 +96,8 @@ bool AWSAuthV4Signer::SignRequestWithSigV4a(Aws::Http::HttpRequest& request, con awsSigningConfig.SetSignatureType(signatureType); awsSigningConfig.SetRegion(region); awsSigningConfig.SetService(serviceName); - awsSigningConfig.SetSigningTimepoint(GetSigningTimestamp().UnderlyingTimestamp()); + const DateTime sigV4aSigningTime = request.GetSigningTimestampOverride() ? request.GetSigningTimestampOverride().value() : GetSigningTimestamp(); + awsSigningConfig.SetSigningTimepoint(sigV4aSigningTime.UnderlyingTimestamp()); awsSigningConfig.SetUseDoubleUriEncode(m_urlEscapePath); awsSigningConfig.SetShouldNormalizeUriPath(true); awsSigningConfig.SetOmitSessionToken(false); @@ -254,7 +255,7 @@ bool AWSAuthV4Signer::SignRequestWithCreds(Aws::Http::HttpRequest& request, cons } //calculate date header to use in internal signature (this also goes into date header). - DateTime now = GetSigningTimestamp(); + DateTime now = request.GetSigningTimestampOverride() ? request.GetSigningTimestampOverride().value() : GetSigningTimestamp(); Aws::String dateHeaderValue = now.ToGmtString(DateFormat::ISO_8601_BASIC); request.SetHeaderValue(AWS_DATE_HEADER, dateHeaderValue); diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index c0253901d1ed..0c1c40a250f0 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -68,11 +69,6 @@ static const char AWS_CLIENT_LOG_TAG[] = "AWSClient"; static const char AWS_LAMBDA_FUNCTION_NAME[] = "AWS_LAMBDA_FUNCTION_NAME"; static const char X_AMZN_TRACE_ID[] = "_X_AMZN_TRACE_ID"; -//4 Minutes -static const std::chrono::milliseconds TIME_DIFF_MAX = std::chrono::minutes(4); -//-4 Minutes -static const std::chrono::milliseconds TIME_DIFF_MIN = std::chrono::minutes(-4); - CoreErrors AWSClient::GuessBodylessErrorType(Aws::Http::HttpResponseCode responseCode) { switch (responseCode) @@ -138,6 +134,7 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration, m_hash(Aws::Utils::Crypto::CreateMD5Implementation()), m_requestTimeoutMs(configuration.requestTimeoutMs), m_enableClockSkewAdjustment(configuration.enableClockSkewAdjustment), + m_clientSkew(Aws::MakeShared(AWS_CLIENT_LOG_TAG, std::chrono::milliseconds(0))), m_requestCompressionConfig(configuration.requestCompressionConfig), m_userAgentInterceptor{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)}, m_interceptors{Aws::MakeShared(AWS_CLIENT_LOG_TAG), Aws::MakeShared(AWS_CLIENT_LOG_TAG, @@ -169,6 +166,7 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration, m_hash(Aws::Utils::Crypto::CreateMD5Implementation()), m_requestTimeoutMs(configuration.requestTimeoutMs), m_enableClockSkewAdjustment(configuration.enableClockSkewAdjustment), + m_clientSkew(Aws::MakeShared(AWS_CLIENT_LOG_TAG, std::chrono::milliseconds(0))), m_requestCompressionConfig(configuration.requestCompressionConfig), m_userAgentInterceptor{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)}, m_interceptors{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration), Aws::MakeShared(AWS_CLIENT_LOG_TAG, @@ -228,30 +226,16 @@ static DateTime GetServerTimeFromError(const AWSError error) } } -bool AWSClient::AdjustClockSkew(HttpResponseOutcome& outcome, const char* signerName) const +bool AWSClient::AdjustClockSkew(HttpResponseOutcome& outcome, const Aws::Utils::DateTime& timeRequestSent, const Aws::Utils::DateTime& timeResponseReceived, std::chrono::milliseconds attemptSkew) const { if (m_enableClockSkewAdjustment) { - auto signer = GetSignerByName(signerName); - //detect clock skew and try to correct. - AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "If the signature check failed. This could be because of a time skew. Attempting to adjust the signer."); - - DateTime serverTime = GetServerTimeFromError(outcome.GetError()); - const auto signingTimestamp = signer->GetSigningTimestamp(); - if (!serverTime.WasParseSuccessful() || serverTime == DateTime()) - { - AWS_LOGSTREAM_DEBUG(AWS_CLIENT_LOG_TAG, "Date header was not found in the response, can't attempt to detect clock skew"); - return false; - } - - AWS_LOGSTREAM_DEBUG(AWS_CLIENT_LOG_TAG, "Server time is " << serverTime.ToGmtString(DateFormat::RFC822) << ", while client time is " << DateTime::Now().ToGmtString(DateFormat::RFC822)); - auto diff = DateTime::Diff(serverTime, signingTimestamp); - //only try again if clock skew was the cause of the error. - if (diff >= TIME_DIFF_MAX || diff <= TIME_DIFF_MIN) + const auto measurement = Aws::Internal::MakeClockSkewMeasurement(outcome.GetError().GetResponseHeaders(), timeRequestSent, timeResponseReceived); + const auto adjustment = m_clientSkew->EvaluateFailure(measurement, attemptSkew); + // Force a retry only when the error is a clock-skew error code and the skew exceeds the threshold. + if (Aws::Internal::IsClockSkewError(outcome.GetError()) && adjustment.skewExceedsThreshold) { - diff = DateTime::Diff(serverTime, DateTime::Now()); - AWS_LOGSTREAM_INFO(AWS_CLIENT_LOG_TAG, "Computed time difference as " << diff.count() << " milliseconds. Adjusting signer with the skew."); - signer->SetClockSkew(diff); + AWS_LOGSTREAM_WARN(AWS_CLIENT_LOG_TAG, "Signature check likely failed due to clock skew; adjusting the signing timestamp and retrying."); AWSError newError( outcome.GetError().GetErrorType(), outcome.GetError().GetExceptionName(), outcome.GetError().GetMessage(), true); newError.SetResponseHeaders(outcome.GetError().GetResponseHeaders()); @@ -290,6 +274,8 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, httpRequest->SetHeaderValue(Http::SDK_REQUEST_HEADER, requestInfo); AppendRecursionDetectionHeader(httpRequest); + // AttemptSkew: seeded from the client-level skew, updated after each attempt. + std::chrono::milliseconds attemptSkew = m_clientSkew->Load(); for (long retries = 0;; retries++) { if(!m_retryStrategy->HasSendToken()) @@ -303,7 +289,10 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, httpRequest->SetEventStreamRequest(request.IsEventStreamRequest()); httpRequest->SetHasEventStreamResponse(request.HasEventStreamResponse()); + const DateTime attemptSentTime = DateTime::Now(); + httpRequest->SetSigningTimestampOverride(attemptSentTime + attemptSkew); outcome = AttemptOneRequest(httpRequest, request, signerName, signerRegion, signerServiceNameOverride); + const DateTime timeResponseReceived = DateTime::Now(); outcome.SetRetryCount(retries); if (retries == 0) { @@ -320,6 +309,10 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); if (outcome.IsSuccess()) { + if (m_enableClockSkewAdjustment && outcome.GetResult()) + { + m_clientSkew->RecordResponse(Aws::Internal::MakeClockSkewMeasurement(outcome.GetResult()->GetHeaders(), attemptSentTime, timeResponseReceived)); + } Aws::Monitoring::OnRequestSucceeded(this->GetServiceClientName(), request.GetServiceRequestName(), httpRequest, outcome, coreMetrics, contexts); AWS_LOGSTREAM_TRACE(AWS_CLIENT_LOG_TAG, "Request successful returning."); break; @@ -362,7 +355,8 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, {{TracingUtils::SMITHY_METHOD_DIMENSION, request.GetServiceRequestName()},{TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); //AdjustClockSkew returns true means clock skew was the problem and skew was adjusted, false otherwise. //sleep if clock skew and region was NOT the problem. AdjustClockSkew may update error inside outcome. - bool shouldSleep = !AdjustClockSkew(outcome, signerName) && !retryWithCorrectRegion; + bool shouldSleep = !AdjustClockSkew(outcome, attemptSentTime, timeResponseReceived, attemptSkew) && !retryWithCorrectRegion; + attemptSkew = m_clientSkew->Load(); if (!retryWithCorrectRegion && !m_retryStrategy->ShouldRetry(outcome.GetError(), retries)) { @@ -464,6 +458,8 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, httpRequest->SetHeaderValue(Http::SDK_REQUEST_HEADER, requestInfo); AppendRecursionDetectionHeader(httpRequest); + // AttemptSkew: seeded from the client-level skew, updated after each attempt. + std::chrono::milliseconds attemptSkew = m_clientSkew->Load(); for (long retries = 0;; retries++) { if(!m_retryStrategy->HasSendToken()) @@ -474,7 +470,10 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, false/*retryable*/)); }; + const DateTime attemptSentTime = DateTime::Now(); + httpRequest->SetSigningTimestampOverride(attemptSentTime + attemptSkew); outcome = AttemptOneRequest(httpRequest, signerName, requestName, signerRegion, signerServiceNameOverride); + const DateTime timeResponseReceived = DateTime::Now(); outcome.SetRetryCount(retries); if (retries == 0) { @@ -490,6 +489,10 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, {{TracingUtils::SMITHY_METHOD_DIMENSION, requestName},{TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); if (outcome.IsSuccess()) { + if (m_enableClockSkewAdjustment && outcome.GetResult()) + { + m_clientSkew->RecordResponse(Aws::Internal::MakeClockSkewMeasurement(outcome.GetResult()->GetHeaders(), attemptSentTime, timeResponseReceived)); + } Aws::Monitoring::OnRequestSucceeded(this->GetServiceClientName(), requestName, httpRequest, outcome, coreMetrics, contexts); AWS_LOGSTREAM_TRACE(AWS_CLIENT_LOG_TAG, "Request successful returning."); break; @@ -532,7 +535,8 @@ HttpResponseOutcome AWSClient::AttemptExhaustively(const Aws::Http::URI& uri, {{TracingUtils::SMITHY_METHOD_DIMENSION, requestName},{TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); //AdjustClockSkew returns true means clock skew was the problem and skew was adjusted, false otherwise. //sleep if clock skew and region was NOT the problem. AdjustClockSkew may update error inside outcome. - bool shouldSleep = !AdjustClockSkew(outcome, signerName) && !retryWithCorrectRegion; + bool shouldSleep = !AdjustClockSkew(outcome, attemptSentTime, timeResponseReceived, attemptSkew) && !retryWithCorrectRegion; + attemptSkew = m_clientSkew->Load(); if (!retryWithCorrectRegion && !m_retryStrategy->ShouldRetry(outcome.GetError(), retries)) { diff --git a/src/aws-cpp-sdk-core/source/client/ClientConfiguration.cpp b/src/aws-cpp-sdk-core/source/client/ClientConfiguration.cpp index 78f3168866c6..ac41987427f7 100644 --- a/src/aws-cpp-sdk-core/source/client/ClientConfiguration.cpp +++ b/src/aws-cpp-sdk-core/source/client/ClientConfiguration.cpp @@ -39,6 +39,8 @@ static const char* REQUEST_MIN_COMPRESSION_SIZE_BYTES_CONFIG_VAR = "request_min_ static const char* AWS_EXECUTION_ENV = "AWS_EXECUTION_ENV"; static const char* DISABLE_IMDSV1_CONFIG_VAR = "AWS_EC2_METADATA_V1_DISABLED"; static const char* DISABLE_IMDSV1_ENV_VAR = "ec2_metadata_v1_disabled"; +static const char* DISABLE_CLOCK_SKEW_CORRECTION_ENV_VAR = "AWS_DISABLE_CLOCK_SKEW_CORRECTION"; +static const char* DISABLE_CLOCK_SKEW_CORRECTION_CONFIG_VAR = "disable_clock_skew_correction"; static const char* AWS_ACCOUNT_ID_ENDPOINT_MODE_ENVIRONMENT_VARIABLE = "AWS_ACCOUNT_ID_ENDPOINT_MODE"; static const char* AWS_ACCOUNT_ID_ENDPOINT_MODE_CONFIG_FILE_OPTION = "account_id_endpoint_mode"; static const char* AWS_METADATA_SERVICE_TIMEOUT_ENV_VAR = "AWS_METADATA_SERVICE_TIMEOUT"; @@ -336,6 +338,16 @@ void setConfigFromEnvOrProfile(ClientConfiguration &config) config.credentialProviderConfig.imdsConfig.disableImdsV1 = true; } + // Knob is a "disable" flag (AWS ..._DISABLED convention); the config field is "enable" and defaults on. + const bool disableClockSkewCorrection = ClientConfiguration::LoadConfigFromEnvOrProfile(DISABLE_CLOCK_SKEW_CORRECTION_ENV_VAR, + config.profileName, + DISABLE_CLOCK_SKEW_CORRECTION_CONFIG_VAR, + {"true", "false"}, + "false") == "true"; + if (disableClockSkewCorrection) { + config.enableClockSkewAdjustment = false; + } + // accountId is intentionally not set here: AWS_ACCOUNT_ID env variable may not match the provided credentials. // it must be set by an auth provider / identity resolver or by an SDK user. config.accountIdEndpointMode = ClientConfiguration::LoadConfigFromEnvOrProfile(AWS_ACCOUNT_ID_ENDPOINT_MODE_ENVIRONMENT_VARIABLE, diff --git a/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp b/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp index d925a2efd68d..ace89ff4ab80 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -69,6 +70,7 @@ void createFromFactoriesIfPresent(T& entity, std::function& factory) { void AwsSmithyClientBase::baseInit() { AWS_CHECK_PTR(AWS_SMITHY_CLIENT_LOG, m_clientConfig); + m_clientSkew = Aws::MakeShared(AWS_SMITHY_CLIENT_LOG, std::chrono::milliseconds(0)); createFromFactories(m_clientConfig->retryStrategy, m_clientConfig->configFactories.retryStrategyCreateFn); createFromFactories(m_clientConfig->executor, m_clientConfig->configFactories.executorCreateFn); createFromFactories(m_clientConfig->writeRateLimiter, m_clientConfig->configFactories.writeRateLimiterCreateFn); @@ -85,6 +87,7 @@ void AwsSmithyClientBase::baseInit() { void AwsSmithyClientBase::baseCopyInit() { AWS_CHECK_PTR(AWS_SMITHY_CLIENT_LOG, m_clientConfig); + m_clientSkew = Aws::MakeShared(AWS_SMITHY_CLIENT_LOG, std::chrono::milliseconds(0)); createFromFactoriesIfPresent(m_clientConfig->retryStrategy, m_clientConfig->configFactories.retryStrategyCreateFn); createFromFactoriesIfPresent(m_clientConfig->executor, m_clientConfig->configFactories.executorCreateFn); createFromFactoriesIfPresent(m_clientConfig->writeRateLimiter, m_clientConfig->configFactories.writeRateLimiterCreateFn); @@ -118,6 +121,7 @@ void AwsSmithyClientBase::baseCopyAssign(const AwsSmithyClientBase& other, } void AwsSmithyClientBase::baseMoveAssign(AwsSmithyClientBase&& other) { + m_clientSkew = Aws::MakeShared(AWS_SMITHY_CLIENT_LOG, std::chrono::milliseconds(0)); m_serviceName = std::move(other.m_serviceName); m_serviceUserAgentName = std::move(other.m_serviceUserAgentName); m_httpClient = std::move(other.m_httpClient); @@ -305,6 +309,7 @@ void AwsSmithyClientBase::MakeRequestAsync(Aws::AmazonWebServiceRequest const* c return; } pRequestCtx->m_requestInfo.attempt = 1; + pRequestCtx->m_attemptSkew = m_clientSkew->Load(); pRequestCtx->m_requestInfo.maxAttempts = Aws::Environment::GetEnv("AWS_NEW_RETRIES_2026") == "true" ? m_clientConfig->retryStrategy->GetMaxAttempts() @@ -410,6 +415,8 @@ void AwsSmithyClientBase::AttemptOneRequestAsync(std::shared_ptrm_timeRequestSent = Aws::Utils::DateTime::Now(); + pRequestCtx->m_httpRequest->SetSigningTimestampOverride(pRequestCtx->m_timeRequestSent + pRequestCtx->m_attemptSkew); SigningOutcome signingOutcome = TracingUtils::MakeCallWithTiming([&]() -> SigningOutcome { return this->SignHttpRequest(pRequestCtx->m_httpRequest, *pRequestCtx); }, @@ -488,11 +495,41 @@ void AwsSmithyClientBase::AttemptOneRequestAsync(std::shared_ptrenableClockSkewAdjustment) + { + m_clientSkew->RecordResponse(Aws::Internal::MakeClockSkewMeasurement(response.GetHeaders(), ctx.m_timeRequestSent, ctx.m_timeResponseReceived)); + } +} + +bool AwsSmithyClientBase::AdjustClockSkew(HttpResponseOutcome& outcome, const AwsSmithyClientAsyncRequestContext& ctx) const +{ + if (!m_clientConfig->enableClockSkewAdjustment) + { + return false; + } + const auto measurement = Aws::Internal::MakeClockSkewMeasurement(outcome.GetError().GetResponseHeaders(), ctx.m_timeRequestSent, ctx.m_timeResponseReceived); + const auto adjustment = m_clientSkew->EvaluateFailure(measurement, ctx.m_attemptSkew); + // Force a retry only when the error is a clock-skew error code and the skew exceeds the threshold. + if (Aws::Internal::IsClockSkewError(outcome.GetError()) && adjustment.skewExceedsThreshold) + { + AWS_LOGSTREAM_WARN(AWS_SMITHY_CLIENT_LOG, "Signature check likely failed due to clock skew; adjusting the signing timestamp and retrying."); + auto newError = outcome.GetError(); + newError.SetRetryableType(Aws::Client::RetryableType::RETRYABLE); + outcome = std::move(newError); + return true; + } + return false; +} + void AwsSmithyClientBase::HandleAsyncReply(std::shared_ptr pRequestCtx, std::shared_ptr httpResponse) const { assert(pRequestCtx && httpResponse); + pRequestCtx->m_timeResponseReceived = Aws::Utils::DateTime::Now(); + pRequestCtx->m_interceptorContext->SetTransmitResponse(httpResponse); for (const auto& interceptor : m_interceptors) { @@ -546,6 +583,10 @@ void AwsSmithyClientBase::HandleAsyncReply(std::shared_ptrGetServiceClientName()}}); if (outcome.IsSuccess()) { + if (outcome.GetResult()) + { + RecordClockSkew(*outcome.GetResult(), *pRequestCtx); + } Aws::Monitoring::OnRequestSucceeded(this->GetServiceClientName(), pRequestCtx->m_requestName, pRequestCtx->m_httpRequest, @@ -612,13 +653,9 @@ void AwsSmithyClientBase::HandleAsyncReply(std::shared_ptrtelemetryProvider->getMeter(this->GetServiceClientName(), {}), {{TracingUtils::SMITHY_METHOD_DIMENSION, pRequestCtx->m_requestName}, {TracingUtils::SMITHY_SERVICE_DIMENSION, this->GetServiceClientName()}}); - bool shouldSleep = !retryWithCorrectRegion; - if (m_clientConfig->enableClockSkewAdjustment) - { - // AdjustClockSkew returns true means clock skew was the problem and skew was adjusted, false otherwise. - // sleep if clock skew and region was NOT the problem. AdjustClockSkew may update error inside outcome. - shouldSleep |= !this->AdjustClockSkew(outcome, pRequestCtx->m_authSchemeOption); - } + // Sleep only if neither clock skew nor region caused the failure. AdjustClockSkew self-gates on the disable knob. + bool shouldSleep = !this->AdjustClockSkew(outcome, *pRequestCtx) && !retryWithCorrectRegion; + pRequestCtx->m_attemptSkew = m_clientSkew->Load(); if (!retryWithCorrectRegion && !m_clientConfig->retryStrategy->ShouldRetry(outcome.GetError(), static_cast(pRequestCtx->m_retryCount))) { diff --git a/tests/aws-cpp-sdk-core-tests/aws/client/AWSClientTest.cpp b/tests/aws-cpp-sdk-core-tests/aws/client/AWSClientTest.cpp index 11695b9f7602..56e20c5e99cc 100644 --- a/tests/aws-cpp-sdk-core-tests/aws/client/AWSClientTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/aws/client/AWSClientTest.cpp @@ -69,7 +69,7 @@ class AWSClientTestSuite : public Aws::Testing::AwsCppSdkGTestSuite protected: std::shared_ptr mockHttpClient; std::shared_ptr mockHttpClientFactory; - Aws::UniquePtr client; + Aws::UniquePtr client; virtual void SetUp() { @@ -84,7 +84,7 @@ class AWSClientTestSuite : public Aws::Testing::AwsCppSdkGTestSuite mockHttpClientFactory = Aws::MakeShared(ALLOCATION_TAG); mockHttpClientFactory->SetClient(mockHttpClient); SetHttpClientFactory(mockHttpClientFactory); - client = Aws::MakeUnique(ALLOCATION_TAG, config); + client = Aws::MakeUnique(ALLOCATION_TAG, config); } void TearDown() @@ -167,7 +167,7 @@ class XMLClientTestSuite : public AWSClientTestSuite mockHttpClientFactory = Aws::MakeShared(ALLOCATION_TAG); mockHttpClientFactory->SetClient(mockHttpClient); SetHttpClientFactory(mockHttpClientFactory); - client = Aws::MakeUnique(ALLOCATION_TAG, config, Aws::MakeShared("xmlErrorMarshaller")); + client = Aws::MakeUnique(ALLOCATION_TAG, config, Aws::MakeShared("xmlErrorMarshaller")); } }; @@ -215,6 +215,7 @@ TEST_F(AWSClientTestSuite, TestClockSkewOutsideAcceptableRange) { HeaderValueCollection responseHeaders; responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::hours(1)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 1 hour + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code AmazonWebServiceRequestMock request; QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); @@ -227,6 +228,7 @@ TEST_F(AWSClientTestSuite, TestClockSkewWithinAcceptableRange) { HeaderValueCollection responseHeaders; responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::minutes(2)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 2 minutes + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code, but skew is below threshold AmazonWebServiceRequestMock request; QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); auto outcome = client->MakeRequest(request); @@ -239,6 +241,7 @@ TEST_F(AWSClientTestSuite, TestClockSkewConsecutiveRequests) // first request should set the skew offset and retry, but following requests should not HeaderValueCollection responseHeaders; responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::hours(1)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 1 hour + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code AmazonWebServiceRequestMock request; QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); @@ -248,14 +251,14 @@ TEST_F(AWSClientTestSuite, TestClockSkewConsecutiveRequests) QueueMockResponse(HttpResponseCode::UNAUTHORIZED, responseHeaders); outcome = client->MakeRequest(request); - ASSERT_FALSE(outcome.IsSuccess()); // should _not_ attempt to adjust clock skew and retry the request. + ASSERT_FALSE(outcome.IsSuccess()); // skew already applied; the offset now matches, so no retry. ASSERT_EQ(HttpResponseCode::UNAUTHORIZED, outcome.GetError().GetResponseCode()); ASSERT_STREQ("127.0.0.1", outcome.GetError().GetRemoteHostIpAddress().c_str()); ASSERT_EQ(0, client->GetRequestAttemptedRetries()); QueueMockResponse(HttpResponseCode::FORBIDDEN, responseHeaders); outcome = client->MakeRequest(request); - ASSERT_FALSE(outcome.IsSuccess()); // should _not_ attempt to adjust clock skew and retry the request. + ASSERT_FALSE(outcome.IsSuccess()); // skew already applied; the offset now matches, so no retry. ASSERT_EQ(HttpResponseCode::FORBIDDEN, outcome.GetError().GetResponseCode()); ASSERT_STREQ("127.0.0.1", outcome.GetError().GetRemoteHostIpAddress().c_str()); ASSERT_EQ(0, client->GetRequestAttemptedRetries()); @@ -270,6 +273,7 @@ TEST_F(AWSClientTestSuite, TestClockChangesAfterSkewHasBeenSet) // make an initial request so that a skew adjustment is set HeaderValueCollection responseHeaders; responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::hours(1)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 1 hour + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code AmazonWebServiceRequestMock request; QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); @@ -280,6 +284,7 @@ TEST_F(AWSClientTestSuite, TestClockChangesAfterSkewHasBeenSet) // make another request with the clock skewed even further responseHeaders.clear(); responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::hours(2)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 2 hours + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); QueueMockResponse(HttpResponseCode::FORBIDDEN, responseHeaders); QueueMockResponse(HttpResponseCode::FORBIDDEN, responseHeaders); outcome = client->MakeRequest(request); @@ -289,6 +294,7 @@ TEST_F(AWSClientTestSuite, TestClockChangesAfterSkewHasBeenSet) // make another request with the clock in sync with the server responseHeaders.clear(); responseHeaders.emplace("Date", DateTime::Now().ToGmtString(DateFormat::RFC822)); // server is in sync with client + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); QueueMockResponse(HttpResponseCode::FORBIDDEN, responseHeaders); QueueMockResponse(HttpResponseCode::FORBIDDEN, responseHeaders); outcome = client->MakeRequest(request); @@ -300,10 +306,10 @@ TEST_F(AWSClientTestSuite, TestRetryHeaders) { // The first server time is ahead of us by 1 hour. DateTime serverTime1 = DateTime::Now() + std::chrono::hours(1); - QueueMockResponse(HttpResponseCode::REQUEST_NOT_MADE, HeaderValueCollection{std::make_pair("Date", serverTime1.ToGmtString(DateFormat::RFC822))}); + QueueMockResponse(HttpResponseCode::REQUEST_NOT_MADE, HeaderValueCollection{std::make_pair("Date", serverTime1.ToGmtString(DateFormat::RFC822)), std::make_pair("x-amzn-errortype", Aws::String("RequestTimeTooSkewed"))}); // The second server time is ahead of us by 2 hour. DateTime serverTime2 = DateTime::Now() + std::chrono::hours(2); - QueueMockResponse(HttpResponseCode::REQUEST_NOT_MADE, HeaderValueCollection{std::make_pair("Date", serverTime2.ToGmtString(DateFormat::RFC822))}); + QueueMockResponse(HttpResponseCode::REQUEST_NOT_MADE, HeaderValueCollection{std::make_pair("Date", serverTime2.ToGmtString(DateFormat::RFC822)), std::make_pair("x-amzn-errortype", Aws::String("RequestTimeTooSkewed"))}); // The third server time is ahead of us by 3 hour. DateTime serverTime3 = DateTime::Now() + std::chrono::hours(3); QueueMockResponse(HttpResponseCode::OK, HeaderValueCollection{std::make_pair("Date", serverTime3.ToGmtString(DateFormat::RFC822))}); @@ -345,6 +351,7 @@ TEST_F(AWSClientTestSuite, TestRetryURIs) { HeaderValueCollection responseHeaders; responseHeaders.emplace("Date", (DateTime::Now() + std::chrono::hours(1)).ToGmtString(DateFormat::RFC822)); // server is ahead of us by 1 hour + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code QueueMockResponse(HttpResponseCode::INTERNAL_SERVER_ERROR, responseHeaders); QueueMockResponse(HttpResponseCode::INTERNAL_SERVER_ERROR, responseHeaders); URI uri("http://www.uri.com/path with space/to/res"); diff --git a/tests/aws-cpp-sdk-core-tests/monitoring/MonitoringTest.cpp b/tests/aws-cpp-sdk-core-tests/monitoring/MonitoringTest.cpp index bcfdc6e41284..5180345e5d70 100644 --- a/tests/aws-cpp-sdk-core-tests/monitoring/MonitoringTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/monitoring/MonitoringTest.cpp @@ -188,7 +188,7 @@ class MonitoringTestSuite : public Aws::Testing::AwsCppSdkGTestSuite protected: std::shared_ptr mockHttpClient; std::shared_ptr mockHttpClientFactory; - Aws::UniquePtr client; + Aws::UniquePtr client; void SetUp() { @@ -203,7 +203,7 @@ class MonitoringTestSuite : public Aws::Testing::AwsCppSdkGTestSuite mockHttpClientFactory = Aws::MakeShared(ALLOCATION_TAG); mockHttpClientFactory->SetClient(mockHttpClient); SetHttpClientFactory(mockHttpClientFactory); - client = Aws::MakeUnique(ALLOCATION_TAG, config); + client = Aws::MakeUnique(ALLOCATION_TAG, config); Aws::Monitoring::CleanupMonitoring(); std::vector factoryFunctions; @@ -249,6 +249,7 @@ TEST_F(MonitoringTestSuite, TestMonitoringListenersAreCalledCorrectlyWithRetryAn HeaderValueCollection responseHeaders, requestHeaders; responseHeaders.emplace("Date", (Aws::Utils::DateTime::Now() + std::chrono::hours(1)).ToGmtString(Aws::Utils::DateFormat::RFC822)); // server is ahead of us by 1 hour AmazonWebServiceRequestMock request; + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code requestHeaders.emplace("X-Amz-Date", Aws::Utils::DateTime::Now().ToGmtString(Aws::Utils::DateFormat::ISO_8601)); request.SetHeaders(requestHeaders); // BAD_REQUEST is not retryable, but since this is triggered by clock skew, it's set to mandatory retryable. @@ -271,6 +272,7 @@ TEST_F(MonitoringTestSuite, TestMonitoringListenersAreCalledCorrectlyWithRetryAn HeaderValueCollection responseHeaders, requestHeaders; responseHeaders.emplace("Date", (Aws::Utils::DateTime::Now() + std::chrono::hours(1)).ToGmtString(Aws::Utils::DateFormat::RFC822)); // server is ahead of us by 1 hour AmazonWebServiceRequestMock request; + responseHeaders.emplace("x-amzn-errortype", "RequestTimeTooSkewed"); // clock-skew error code requestHeaders.emplace("X-Amz-Date", Aws::Utils::DateTime::Now().ToGmtString(Aws::Utils::DateFormat::ISO_8601)); request.SetHeaders(requestHeaders); QueueMockResponse(HttpResponseCode::BAD_REQUEST, responseHeaders); diff --git a/tests/testing-resources/include/aws/testing/mocks/aws/client/MockAWSClient.h b/tests/testing-resources/include/aws/testing/mocks/aws/client/MockAWSClient.h index cff1a22c630f..07f5e6a5910e 100644 --- a/tests/testing-resources/include/aws/testing/mocks/aws/client/MockAWSClient.h +++ b/tests/testing-resources/include/aws/testing/mocks/aws/client/MockAWSClient.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -246,3 +247,28 @@ class MockAWSClientWithStandardRetryStrategy : Aws::Client::AWSClient return err; } }; + +// A MockAWSClient whose error responses carry a service error code via the x-amzn-errortype header +// (the header a JSON error marshaller reads), so a clock-skew retry can be exercised end to end. +class ClockSkewMockAWSClient : public MockAWSClient +{ +public: + using MockAWSClient::MockAWSClient; + +protected: + Aws::Client::AWSError BuildAWSError(const std::shared_ptr& response) const override + { + const auto& headers = response->GetHeaders(); + const auto it = headers.find("x-amzn-errortype"); + if (it == headers.end()) + { + return MockAWSClient::BuildAWSError(response); + } + Aws::Client::AWSError error( + Aws::Client::CoreErrorsMapper::GetErrorForName(it->second.c_str()).GetErrorType(), it->second, "", false); + error.SetResponseHeaders(headers); + error.SetResponseCode(response->GetResponseCode()); + error.SetRemoteHostIpAddress(response->GetOriginatingRequest().GetResolvedRemoteHost()); + return error; + } +};