fault: add dry_run mode for HTTP fault abort injection

api/envoy/extensions/filters/http/fault/v3/fault.proto

@@ -23,7 +23,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // Fault Injection :ref:`configuration overview <config_http_filters_fault_injection>`.
 // [#extension: envoy.filters.http.fault]
 
-// [#next-free-field: 7]
+// [#next-free-field: 8]
 message FaultAbort {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.http.fault.v2.FaultAbort";
@@ -58,6 +58,19 @@ message FaultAbort {
   // The response body to be returned when aborting the request.
   // If not specified, defaults to "fault filter abort".
   string response_body = 6;
+
+  // If true, requests matching the abort fault will be recorded in dry-run
+  // stats (``aborts_dry_run``) and :ref:`filter_metadata
+  // <envoy_v3_api_field_extensions.filters.http.fault.v3.HTTPFault.filter_metadata>`
+  // dynamic metadata without actually aborting the request. Defaults to ``false``.
+  //
+  // .. note::
+  //   This setting intentionally applies only to abort fault injection. Any
+  //   :ref:`delay <envoy_v3_api_field_extensions.filters.http.fault.v3.HTTPFault.delay>` or
+  //   :ref:`response_rate_limit
+  //   <envoy_v3_api_field_extensions.filters.http.fault.v3.HTTPFault.response_rate_limit>`
+  //   configured on the same filter is still injected normally.
+  bool dry_run = 7;
 }
 
 // [#next-free-field: 17]

changelogs/changelogs.yaml

@@ -109,6 +109,8 @@ areas:
   ext_proc:
     title: ext_proc
 
+  fault:
+    title: fault
   filter_chain:
     title: filter_chain
   formatter:

changelogs/current/new_features/fault__abort-dry-run.rst

@@ -0,0 +1,6 @@
+Added :ref:`dry_run <envoy_v3_api_field_extensions.filters.http.fault.v3.FaultAbort.dry_run>` to the
+HTTP fault injection filter to support dry-run abort evaluation. When enabled, requests matching an
+abort fault increment the ``aborts_dry_run`` :ref:`counter <config_http_filters_fault_injection_stats>`
+and record configured :ref:`filter_metadata
+<envoy_v3_api_field_extensions.filters.http.fault.v3.HTTPFault.filter_metadata>` in dynamic metadata
+without aborting the request, while any configured delay or response rate limit is still injected.

docs/root/configuration/http/http_filters/fault_filter.rst

@@ -220,8 +220,10 @@ owning HTTP connection manager.
 
   delays_injected, Counter, Total requests that were delayed
   aborts_injected, Counter, Total requests that were aborted
+  aborts_dry_run, Counter, "Total requests matching an abort fault in :ref:`dry_run <envoy_v3_api_field_extensions.filters.http.fault.v3.FaultAbort.dry_run>` mode without being aborted (any configured delay or response rate limit on the same request is still injected normally)"
   response_rl_injected, Counter, "Total requests that had a response rate limit selected for injection (actually injection may not occur due to disconnect, reset, no body, etc.)"
   faults_overflow, Counter, Total number of faults that were not injected due to overflowing the :ref:`max_active_faults <envoy_v3_api_field_extensions.filters.http.fault.v3.HTTPFault.max_active_faults>` setting
   active_faults, Gauge, Total number of faults active at the current time
   <downstream-cluster>.delays_injected, Counter, Total delayed requests for the given downstream cluster
   <downstream-cluster>.aborts_injected, Counter, Total aborted requests for the given downstream cluster
+  <downstream-cluster>.aborts_dry_run, Counter, Total requests matching an abort fault in dry-run mode for the given downstream cluster

source/extensions/filters/common/fault/fault_config.cc

@@ -37,7 +37,7 @@ HeaderPercentageProvider::percentage(const Http::RequestHeaderMap* request_heade
 
 FaultAbortConfig::FaultAbortConfig(
     const envoy::extensions::filters::http::fault::v3::FaultAbort& abort_config)
-    : response_body_(abort_config.response_body()) {
+    : response_body_(abort_config.response_body()), dry_run_(abort_config.dry_run()) {
   switch (abort_config.error_type_case()) {
   case envoy::extensions::filters::http::fault::v3::FaultAbort::ErrorTypeCase::kHttpStatus:
     provider_ =

source/extensions/filters/common/fault/fault_config.h

@@ -69,6 +69,7 @@ class FaultAbortConfig {
   }
 
   const std::string& responseBody() const { return response_body_; }
+  bool dryRun() const { return dry_run_; }
 
 private:
   // Abstract abort provider.
@@ -145,6 +146,7 @@ class FaultAbortConfig {
 
   AbortProviderPtr provider_;
   const std::string response_body_;
+  const bool dry_run_;
 };
 
 using FaultAbortConfigPtr = std::unique_ptr<FaultAbortConfig>;

source/extensions/filters/http/fault/fault_filter.cc

@@ -86,11 +86,13 @@ FaultFilterConfig::FaultFilterConfig(
     : settings_(fault, context), runtime_(context.runtime()),
       stats_(generateStats(stats_prefix, scope)), scope_(scope), time_source_(context.timeSource()),
       stat_name_set_(scope.symbolTable().makeSet("Fault")),
+      aborts_dry_run_(stat_name_set_->add("aborts_dry_run")),
       aborts_injected_(stat_name_set_->add("aborts_injected")),
       delays_injected_(stat_name_set_->add("delays_injected")),
       stats_prefix_(stat_name_set_->add(absl::StrCat(stats_prefix, "fault"))),
       downstream_cluster_tag_(
           stat_name_set_->add(Envoy::Config::TagNames::get().FAULT_DOWNSTREAM_CLUSTER)),
+      aborts_dry_run_base_(stat_name_set_->add(absl::StrCat(stats_prefix, "fault.aborts_dry_run"))),
       aborts_injected_base_(
           stat_name_set_->add(absl::StrCat(stats_prefix, "fault.aborts_injected"))),
       delays_injected_base_(
@@ -183,9 +185,7 @@ bool FaultFilter::maybeSetupDelay(const Http::RequestHeaderMap& request_headers)
     delay_timer_->enableTimer(duration.value(), &decoder_callbacks_->scope());
     recordDelaysInjectedStats();
     decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::CoreResponseFlag::DelayInjected);
-    auto& dynamic_metadata = decoder_callbacks_->streamInfo().dynamicMetadata();
-    (*dynamic_metadata.mutable_filter_metadata())[decoder_callbacks_->filterConfigName()] =
-        fault_settings_->filterMetadata();
+    setFilterMetadata();
     return true;
   }
   return false;
@@ -196,9 +196,15 @@ bool FaultFilter::maybeDoAbort(const Http::RequestHeaderMap& request_headers) {
   std::optional<Grpc::Status::GrpcStatus> grpc_status;
   std::tie(http_status, grpc_status) = abortStatus(request_headers);
 
-  if (http_status.has_value() && tryIncActiveFaults()) {
-    abortWithStatus(http_status.value(), grpc_status);
-    return true;
+  if (http_status.has_value()) {
+    if (isAbortDryRun()) {
+      onAbortDryRun();
+      return false;
+    }
+    if (tryIncActiveFaults()) {
+      abortWithStatus(http_status.value(), grpc_status);
+      return true;
+    }
   }
 
   return false;
@@ -265,6 +271,10 @@ bool FaultFilter::isDelayEnabled(const Http::RequestHeaderMap& request_headers)
                                                       request_delay->percentage(&request_headers));
 }
 
+bool FaultFilter::isAbortDryRun() const {
+  return fault_settings_->requestAbort() != nullptr && fault_settings_->requestAbort()->dryRun();
+}
+
 bool FaultFilter::isAbortEnabled(const Http::RequestHeaderMap& request_headers) {
   const auto request_abort = fault_settings_->requestAbort();
   if (request_abort == nullptr) {
@@ -390,6 +400,15 @@ void FaultFilter::recordDelaysInjectedStats() {
   config_->stats().delays_injected_.inc();
 }
 
+void FaultFilter::recordAbortsDryRunStats() {
+  // Downstream specific stats.
+  if (!downstream_cluster_.empty() && !fault_settings_->disableDownstreamClusterStats()) {
+    config_->incAbortsDryRun(downstream_cluster_storage_->statName());
+  }
+
+  config_->stats().aborts_dry_run_.inc();
+}
+
 void FaultFilter::recordAbortsInjectedStats() {
   // Downstream specific stats.
   if (!downstream_cluster_.empty() && !fault_settings_->disableDownstreamClusterStats()) {
@@ -464,30 +483,32 @@ void FaultFilter::postDelayInjection(const Http::RequestHeaderMap& request_heade
   std::tie(http_status, grpc_status) = abortStatus(request_headers);
 
   if (http_status.has_value()) {
-    abortWithStatus(http_status.value(), grpc_status);
-  } else {
-    // Should not continue to count as an active fault after the delay has elapsed if no other type
-    // of fault is active. As the delay timer is always done at this point and followed abort faults
-    // have been checked earlier, here we just check if there's a response rate limit configured.
-    ASSERT(fault_active_);
-    ASSERT(delay_timer_ == nullptr);
-    if (!isResponseRateLimitConfigured()) {
-      config_->stats().active_faults_.dec();
-      fault_active_ = false;
+    if (!isAbortDryRun()) {
+      abortWithStatus(http_status.value(), grpc_status);
+      return;
     }
+    onAbortDryRun();
+  }
 
-    // Continue request processing.
-    decoder_callbacks_->continueDecoding();
+  // Should not continue to count as an active fault after the delay has elapsed if no other type
+  // of fault is active. As the delay timer is always done at this point and followed abort faults
+  // have been checked earlier, here we just check if there's a response rate limit configured.
+  ASSERT(fault_active_);
+  ASSERT(delay_timer_ == nullptr);
+  if (!isResponseRateLimitConfigured()) {
+    config_->stats().active_faults_.dec();
+    fault_active_ = false;
   }
+
+  // Continue request processing.
+  decoder_callbacks_->continueDecoding();
 }
 
 void FaultFilter::abortWithStatus(Http::Code http_status_code,
                                   std::optional<Grpc::Status::GrpcStatus> grpc_status) {
   recordAbortsInjectedStats();
   decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::CoreResponseFlag::FaultInjected);
-  auto& dynamic_metadata = decoder_callbacks_->streamInfo().dynamicMetadata();
-  (*dynamic_metadata.mutable_filter_metadata())[decoder_callbacks_->filterConfigName()] =
-      fault_settings_->filterMetadata();
+  setFilterMetadata();
   absl::string_view body = "fault filter abort";
   if (fault_settings_->requestAbort() != nullptr &&
       !fault_settings_->requestAbort()->responseBody().empty()) {
@@ -497,6 +518,17 @@ void FaultFilter::abortWithStatus(Http::Code http_status_code,
                                      RcDetails::get().FaultAbort);
 }
 
+void FaultFilter::onAbortDryRun() {
+  recordAbortsDryRunStats();
+  setFilterMetadata();
+}
+
+void FaultFilter::setFilterMetadata() {
+  auto& dynamic_metadata = decoder_callbacks_->streamInfo().dynamicMetadata();
+  (*dynamic_metadata.mutable_filter_metadata())[decoder_callbacks_->filterConfigName()] =
+      fault_settings_->filterMetadata();
+}
+
 bool FaultFilter::matchesTargetUpstreamCluster() {
   bool matches = true;
 

source/extensions/filters/http/fault/fault_filter.h

@@ -31,6 +31,7 @@ namespace Fault {
  * All stats for the fault filter. @see stats_macros.h
  */
 #define ALL_FAULT_FILTER_STATS(COUNTER, GAUGE)                                                     \
+  COUNTER(aborts_dry_run)                                                                          \
   COUNTER(aborts_injected)                                                                         \
   COUNTER(delays_injected)                                                                         \
   COUNTER(faults_overflow)                                                                         \
@@ -129,6 +130,10 @@ class FaultFilterConfig {
   const FaultSettings* settings() { return &settings_; }
   TimeSource& timeSource() { return time_source_; }
 
+  void incAbortsDryRun(Stats::StatName downstream_cluster) {
+    incCounter(downstream_cluster, aborts_dry_run_, aborts_dry_run_base_);
+  }
+
   void incDelays(Stats::StatName downstream_cluster) {
     incCounter(downstream_cluster, delays_injected_, delays_injected_base_);
   }
@@ -148,13 +153,15 @@ class FaultFilterConfig {
   Stats::Scope& scope_;
   TimeSource& time_source_;
   Stats::StatNameSetPtr stat_name_set_;
+  const Stats::StatName aborts_dry_run_;
   const Stats::StatName aborts_injected_;
   const Stats::StatName delays_injected_;
   const Stats::StatName stats_prefix_; // Includes "fault".
   // The tag the downstream cluster is emitted as on the per-cluster stats.
   const Stats::StatName downstream_cluster_tag_;
   // The tag-extracted names of the per-cluster stats, pre-joined once so that charging a stat
   // only joins the full name with the dynamic cluster segment.
+  const Stats::StatName aborts_dry_run_base_;
   const Stats::StatName aborts_injected_base_;
   const Stats::StatName delays_injected_base_;
 };
@@ -201,14 +208,18 @@ class FaultFilter : public Http::StreamFilter, Logger::Loggable<Logger::Id::filt
 
 private:
   bool faultOverflow();
+  void recordAbortsDryRunStats();
   void recordAbortsInjectedStats();
   void recordDelaysInjectedStats();
   void resetTimerState();
   void postDelayInjection(const Http::RequestHeaderMap& request_headers);
   void abortWithStatus(Http::Code http_status_code,
                        std::optional<Grpc::Status::GrpcStatus> grpc_status_code);
+  void onAbortDryRun();
+  void setFilterMetadata();
   bool matchesTargetUpstreamCluster();
   bool matchesDownstreamNodes(const Http::RequestHeaderMap& headers);
+  bool isAbortDryRun() const;
   bool isAbortEnabled(const Http::RequestHeaderMap& request_headers);
   bool isDelayEnabled(const Http::RequestHeaderMap& request_headers);
   bool isResponseRateLimitEnabled(const Http::RequestHeaderMap& request_headers);

test/extensions/filters/common/fault/fault_config_test.cc

@@ -42,6 +42,16 @@ TEST(FaultConfigTest, FaultAbortResponseBody) {
   FaultAbortConfig config(proto_config);
 
   EXPECT_EQ("custom response body", config.responseBody());
+  EXPECT_FALSE(config.dryRun());
+}
+
+TEST(FaultConfigTest, FaultAbortDryRun) {
+  envoy::extensions::filters::http::fault::v3::FaultAbort proto_config;
+  proto_config.set_http_status(503);
+  proto_config.set_dry_run(true);
+  FaultAbortConfig config(proto_config);
+
+  EXPECT_TRUE(config.dryRun());
 }
 
 TEST(FaultConfigTest, FaultAbortGrpcHeaderConfig) {

test/extensions/filters/http/fault/fault_filter_integration_test.cc

@@ -127,6 +127,7 @@ name: fault
   auto response =
       sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 1024);
 
+  EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.aborts_dry_run")->value());
   EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.aborts_injected")->value());
   EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value());
   EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value());
@@ -135,6 +136,8 @@ name: fault
   // The filter's stats live in the connection manager's scope, so they carry its prefix in both
   // their flat name and its tag.
   const std::string& hcm_prefix = Config::TagNames::get().HTTP_CONN_MANAGER_PREFIX;
+  expectStat("http.config_test.fault.aborts_dry_run", "http.fault.aborts_dry_run",
+             {{hcm_prefix, "config_test"}});
   expectStat("http.config_test.fault.aborts_injected", "http.fault.aborts_injected",
              {{hcm_prefix, "config_test"}});
   expectStat("http.config_test.fault.delays_injected", "http.fault.delays_injected",
@@ -434,6 +437,50 @@ TEST_P(FaultIntegrationTestAllProtocols, FaultAbortGrpcConfig) {
   EXPECT_EQ(0UL, test_server_->gauge("http.config_test.fault.active_faults")->value());
 }
 
+// Request abort with dry_run enabled forwards the request upstream and records dry_run stats.
+TEST_P(FaultIntegrationTestAllProtocols, FaultAbortDryRunConfig) {
+  const std::string filter_config =
+      R"EOF(
+name: fault
+typed_config:
+  "@type": type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault
+  abort:
+    http_status: 429
+    dry_run: true
+    percentage:
+      numerator: 100
+)EOF";
+
+  initializeFilter(filter_config);
+  codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http")));
+
+  auto response = codec_client_->makeHeaderOnlyRequest(
+      Http::TestRequestHeaderMapImpl{{":method", "GET"},
+                                     {":path", "/test/long/url"},
+                                     {":scheme", "http"},
+                                     {":authority", "host"},
+                                     {"x-envoy-downstream-service-cluster", "superman"}});
+  waitForNextUpstreamRequest();
+  upstream_request_->encodeHeaders(default_response_headers_, true);
+  ASSERT_TRUE(response->waitForEndStream());
+
+  EXPECT_TRUE(response->complete());
+  EXPECT_THAT(response->headers(), Envoy::Http::HttpStatusIs("200"));
+
+  EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.aborts_dry_run")->value());
+  EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.aborts_injected")->value());
+  EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.delays_injected")->value());
+  EXPECT_EQ(0UL, test_server_->counter("http.config_test.fault.response_rl_injected")->value());
+  EXPECT_EQ(0UL, test_server_->gauge("http.config_test.fault.active_faults")->value());
+  EXPECT_EQ(1UL, test_server_->counter("http.config_test.fault.superman.aborts_dry_run")->value());
+  EXPECT_EQ(nullptr, test_server_->counter("http.config_test.fault.superman.aborts_injected"));
+
+  const std::string& hcm_prefix = Config::TagNames::get().HTTP_CONN_MANAGER_PREFIX;
+  expectStat("http.config_test.fault.superman.aborts_dry_run", "http.fault.aborts_dry_run",
+             {{hcm_prefix, "config_test"},
+              {Config::TagNames::get().FAULT_DOWNSTREAM_CLUSTER, "superman"}});
+}
+
 // Fault integration tests that run with HTTP/2 only, used for fully testing trailers.
 class FaultIntegrationTestHttp2 : public FaultIntegrationTest {};
 INSTANTIATE_TEST_SUITE_P(Protocols, FaultIntegrationTestHttp2,