From b8bbd269e0a4fa8950d864bd3ab25deed870fd91 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:55:46 -0400 Subject: [PATCH 01/10] Define Feature Flagging configuration source contract --- .../OpenFeatureProviderSmokeTest.groovy | 1 + .../datadog/trace/api/ConfigDefaults.java | 4 + .../api/config/FeatureFlaggingConfig.java | 11 +++ .../main/java/datadog/trace/api/Config.java | 78 +++++++++++++++++++ .../datadog/trace/api/ConfigTest.groovy | 32 ++++++++ metadata/supported-configurations.json | 32 ++++++++ .../featureflag/FeatureFlaggingSystem.java | 2 +- .../FeatureFlaggingSystemTest.java | 4 + ...e.java => ConfigurationSourceService.java} | 2 +- .../featureflag/RemoteConfigServiceImpl.java | 2 +- 10 files changed, 165 insertions(+), 3 deletions(-) rename products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/{RemoteConfigService.java => ConfigurationSourceService.java} (60%) diff --git a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy index cb4c641d667..ca80ca47f8b 100644 --- a/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy +++ b/dd-smoke-tests/openfeature/src/test/groovy/datadog/smoketest/springboot/OpenFeatureProviderSmokeTest.groovy @@ -42,6 +42,7 @@ class OpenFeatureProviderSmokeTest extends AbstractServerSmokeTest { command.addAll(['-jar', springBootShadowJar, "--server.port=${httpPort}".toString()]) final builder = new ProcessBuilder(command).directory(new File(buildDirectory)) builder.environment().put('DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED', 'true') + builder.environment().put('DD_FEATURE_FLAGS_CONFIGURATION_SOURCE', 'remote_config') return builder } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 1aba2cb276b..532e9f47d71 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -49,6 +49,10 @@ public final class ConfigDefaults { static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; + public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java index 28151f88864..05d53a0c2ad 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/FeatureFlaggingConfig.java @@ -3,4 +3,15 @@ public class FeatureFlaggingConfig { public static final String FLAGGING_PROVIDER_ENABLED = "experimental.flagging.provider.enabled"; + + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE = + "feature.flags.configuration.source"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL = + "feature.flags.configuration.source.agentless.base.url"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS = + "feature.flags.configuration.source.agentless.poll.interval.seconds"; + public static final String FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS = + "feature.flags.configuration.source.agentless.request.timeout.seconds"; + + private FeatureFlaggingConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 70ff04932e7..87f4866f5a5 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -86,6 +86,9 @@ import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_BODY_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_ELASTICSEARCH_PARAMS_ENABLED; import static datadog.trace.api.ConfigDefaults.DEFAULT_EXPERIMENTATAL_JEE_SPLIT_BY_DEPLOYMENT; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_CLIENT_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_GRPC_SERVER_ERROR_STATUSES; import static datadog.trace.api.ConfigDefaults.DEFAULT_HEALTH_METRICS_ENABLED; @@ -361,6 +364,10 @@ import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_EXCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_INCLUDES; import static datadog.trace.api.config.DebuggerConfig.THIRD_PARTY_SHADING_IDENTIFIERS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_LEVEL; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_QUEUE_SIZE; import static datadog.trace.api.config.GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL; @@ -1213,6 +1220,11 @@ public static String getHostName() { private final int remoteConfigMaxExtraServices; + private final String featureFlaggingConfigurationSource; + private final String featureFlaggingConfigurationSourceAgentlessBaseUrl; + private final int featureFlaggingConfigurationSourcePollIntervalSeconds; + private final int featureFlaggingConfigurationSourceRequestTimeoutSeconds; + private final boolean dbmInjectSqlBaseHash; private final String dbmPropagationMode; private final boolean dbmTracePreparedStatements; @@ -2837,6 +2849,40 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getInteger( REMOTE_CONFIG_MAX_EXTRA_SERVICES, DEFAULT_REMOTE_CONFIG_MAX_EXTRA_SERVICES); + featureFlaggingConfigurationSource = + normalizeFeatureFlaggingConfigurationSource( + configProvider.getString( + FEATURE_FLAGS_CONFIGURATION_SOURCE, DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE)); + featureFlaggingConfigurationSourceAgentlessBaseUrl = + configProvider.getStringNotEmpty( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL, null); + int configuredFeatureFlaggingPollIntervalSeconds = + configProvider.getInteger( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS); + if (configuredFeatureFlaggingPollIntervalSeconds <= 0) { + log.warn( + "Invalid Feature Flagging agentless poll interval: {}. The value must be positive", + configuredFeatureFlaggingPollIntervalSeconds); + configuredFeatureFlaggingPollIntervalSeconds = + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS; + } + featureFlaggingConfigurationSourcePollIntervalSeconds = + configuredFeatureFlaggingPollIntervalSeconds; + int configuredFeatureFlaggingRequestTimeoutSeconds = + configProvider.getInteger( + FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS); + if (configuredFeatureFlaggingRequestTimeoutSeconds <= 0) { + log.warn( + "Invalid Feature Flagging agentless request timeout: {}. The value must be positive", + configuredFeatureFlaggingRequestTimeoutSeconds); + configuredFeatureFlaggingRequestTimeoutSeconds = + DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS; + } + featureFlaggingConfigurationSourceRequestTimeoutSeconds = + configuredFeatureFlaggingRequestTimeoutSeconds; + dynamicInstrumentationEnabled = configProvider.getBoolean( DYNAMIC_INSTRUMENTATION_ENABLED, DEFAULT_DYNAMIC_INSTRUMENTATION_ENABLED); @@ -3749,6 +3795,14 @@ public boolean isInferredProxyPropagationEnabled() { return traceInferredProxyEnabled; } + private static String normalizeFeatureFlaggingConfigurationSource(final String source) { + if (source == null) { + return DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE; + } + final String normalized = source.trim().toLowerCase(Locale.ROOT); + return normalized.isEmpty() ? DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE : normalized; + } + public boolean isBaggageExtract() { return tracePropagationStylesToExtract.contains(TracePropagationStyle.BAGGAGE); } @@ -4656,6 +4710,22 @@ public int getRemoteConfigMaxExtraServices() { return remoteConfigMaxExtraServices; } + public String getFeatureFlaggingConfigurationSource() { + return featureFlaggingConfigurationSource; + } + + public String getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() { + return featureFlaggingConfigurationSourceAgentlessBaseUrl; + } + + public int getFeatureFlaggingConfigurationSourcePollIntervalSeconds() { + return featureFlaggingConfigurationSourcePollIntervalSeconds; + } + + public int getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds() { + return featureFlaggingConfigurationSourceRequestTimeoutSeconds; + } + public boolean isDynamicInstrumentationEnabled() { return dynamicInstrumentationEnabled; } @@ -6488,6 +6558,14 @@ public String toString() { + remoteConfigMaxPayloadSize + ", remoteConfigIntegrityCheckEnabled=" + remoteConfigIntegrityCheckEnabled + + ", featureFlaggingConfigurationSource=" + + featureFlaggingConfigurationSource + + ", featureFlaggingConfigurationSourceAgentlessBaseUrl=" + + featureFlaggingConfigurationSourceAgentlessBaseUrl + + ", featureFlaggingConfigurationSourcePollIntervalSeconds=" + + featureFlaggingConfigurationSourcePollIntervalSeconds + + ", featureFlaggingConfigurationSourceRequestTimeoutSeconds=" + + featureFlaggingConfigurationSourceRequestTimeoutSeconds + ", debuggerEnabled=" + dynamicInstrumentationEnabled + ", debuggerUploadTimeout=" diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 7db955e49ae..953c5ced8f3 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -2,6 +2,8 @@ package datadog.trace.api import static datadog.trace.api.ConfigDefaults.DEFAULT_HTTP_CLIENT_ERROR_STATUSES import static datadog.trace.api.ConfigDefaults.DEFAULT_HTTP_SERVER_ERROR_STATUSES +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS +import static datadog.trace.api.ConfigDefaults.DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.ConfigDefaults.DEFAULT_PARTIAL_FLUSH_MIN_SPANS import static datadog.trace.api.ConfigDefaults.DEFAULT_SERVICE_NAME import static datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_LONG_RUNNING_FLUSH_INTERVAL @@ -55,6 +57,8 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_METRICS_CONFIGS @@ -3478,4 +3482,32 @@ class ConfigTest extends DDSpecification { "1" | true "0" | false } + + def "agentless feature flag timing uses positive configured values"() { + setup: + Properties properties = new Properties() + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "60") + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "4") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSourcePollIntervalSeconds == 60 + config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == 4 + } + + def "agentless feature flag timing falls back for non-positive values"() { + setup: + Properties properties = new Properties() + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS, "0") + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS, "-1") + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSourcePollIntervalSeconds == DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS + config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS + } } diff --git a/metadata/supported-configurations.json b/metadata/supported-configurations.json index 733f3736cbb..74b6ba24df8 100644 --- a/metadata/supported-configurations.json +++ b/metadata/supported-configurations.json @@ -1497,6 +1497,38 @@ "aliases": [] } ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE": [ + { + "version": "A", + "type": "string", + "default": "agentless", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL": [ + { + "version": "A", + "type": "string", + "default": null, + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS": [ + { + "version": "A", + "type": "int", + "default": "30", + "aliases": [] + } + ], + "DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS": [ + { + "version": "A", + "type": "int", + "default": "2", + "aliases": [] + } + ], "DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED": [ { "version": "A", diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 02689767bad..2011d31e83d 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -9,7 +9,7 @@ public class FeatureFlaggingSystem { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureFlaggingSystem.class); - private static volatile RemoteConfigService CONFIG_SERVICE; + private static volatile ConfigurationSourceService CONFIG_SERVICE; private static volatile ExposureWriter EXPOSURE_WRITER; private FeatureFlaggingSystem() {} diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 970c2cf16e3..5b088d99b9d 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -1,5 +1,6 @@ package com.datadog.featureflag; +import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -23,6 +24,8 @@ class FeatureFlaggingSystemTest { @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") void testFeatureFlagSystemInitialization() { ConfigurationPoller poller = mock(ConfigurationPoller.class); DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); @@ -48,6 +51,7 @@ void testFeatureFlagSystemInitialization() { } @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") void testThatRemoteConfigIsRequired() { SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java similarity index 60% rename from products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java rename to products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java index 5f84a78f7d4..83495545717 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigService.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/ConfigurationSourceService.java @@ -2,7 +2,7 @@ import java.io.Closeable; -public interface RemoteConfigService extends Closeable { +public interface ConfigurationSourceService extends Closeable { void init(); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index ec28f684a96..a5a9b2d7f36 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -32,7 +32,7 @@ import okio.Okio; public class RemoteConfigServiceImpl - implements RemoteConfigService, ConfigurationChangesTypedListener { + implements ConfigurationSourceService, ConfigurationChangesTypedListener { private final ConfigurationPoller configurationPoller; From 4baefdbf6ec8f4e690e10d07f7ace15475ae6043 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:57:34 -0400 Subject: [PATCH 02/10] Add Datadog-managed agentless UFC polling --- .../trace/util/AgentThreadFactory.java | 3 +- .../feature-flagging-lib/build.gradle.kts | 1 + .../AgentlessConfigurationSource.java | 358 ++++++++ .../AgentlessConfigurationSourceTest.java | 805 ++++++++++++++++++ 4 files changed, 1166 insertions(+), 1 deletion(-) create mode 100644 products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java create mode 100644 products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index 752adb8899d..e9ef282cc7d 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -66,7 +66,8 @@ public enum AgentThread { LLMOBS_EVALS_PROCESSOR("dd-llmobs-evals-processor"), - FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"); + FEATURE_FLAG_EXPOSURE_PROCESSOR("dd-ffe-exposure-processor"), + FEATURE_FLAG_CONFIGURATION_POLLER("dd-feature-flagging-http-poller"); public final String threadName; diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 3291e239d40..548f5e26d2b 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { api(libs.moshi) api(libs.jctools) api(project(":communication")) + implementation(project(":internal-api")) api(project(":products:feature-flagging:feature-flagging-bootstrap")) api(project(":utils:queue-utils")) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java new file mode 100644 index 00000000000..b3c58631d3a --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -0,0 +1,358 @@ +package com.datadog.featureflag; + +import static datadog.communication.http.OkHttpUtils.prepareRequest; +import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import datadog.trace.util.AgentThreadFactory; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.DoubleSupplier; +import okhttp3.Call; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class AgentlessConfigurationSource implements ConfigurationSourceService { + private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessConfigurationSource.class); + + // TODO before merge: confirm the final backend route with the server-distribution API owners. + private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = + "/api/v2/feature-flagging/config/server-distribution"; + private static final int MAX_ATTEMPTS = 3; + private static final long FIRST_RETRY_MIN_MILLIS = 2_000; + private static final long FIRST_RETRY_MAX_MILLIS = 10_000; + private static final long SECOND_RETRY_MIN_MILLIS = 5_000; + private static final long SECOND_RETRY_MAX_MILLIS = 30_000; + private static final double RETRY_JITTER = 0.2; + + private final HttpUrl endpoint; + private final Config config; + private final long pollIntervalMillis; + private final UfcHttpClient client; + private final ScheduledExecutorService executor; + private final RetrySleeper retrySleeper; + private final DoubleSupplier jitter; + private final Object lifecycleLock = new Object(); + private final AtomicBoolean polling = new AtomicBoolean(); + private volatile boolean closed; + private volatile ScheduledFuture scheduledPoll; + private volatile String etag; + + AgentlessConfigurationSource(final Config config) { + this(config, endpoint(config)); + } + + private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint) { + this( + endpoint, + config, + millis(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()), + new OkHttpUfcHttpClient( + OkHttpUtils.buildHttpClient( + endpoint, + millis(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()))), + Executors.newSingleThreadScheduledExecutor( + new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), + TimeUnit.MILLISECONDS::sleep, + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor) { + this( + endpoint, + config, + pollIntervalMillis, + client, + executor, + TimeUnit.MILLISECONDS::sleep, + () -> 1.0); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter) { + this.endpoint = endpoint; + this.config = config; + this.pollIntervalMillis = pollIntervalMillis; + this.client = client; + this.executor = executor; + this.retrySleeper = retrySleeper; + this.jitter = jitter; + } + + @Override + public void init() { + synchronized (lifecycleLock) { + if (closed || scheduledPoll != null) { + return; + } + scheduledPoll = + executor.scheduleWithFixedDelay( + this::pollOnceSafely, 0, pollIntervalMillis, TimeUnit.MILLISECONDS); + } + } + + boolean pollOnce() { + if (closed || !polling.compareAndSet(false, true)) { + return false; + } + try { + return fetchAndApply(); + } finally { + polling.set(false); + } + } + + @Override + public void close() { + final ScheduledFuture poll; + synchronized (lifecycleLock) { + if (closed) { + return; + } + closed = true; + poll = scheduledPoll; + scheduledPoll = null; + } + if (poll != null) { + poll.cancel(true); + } + client.cancel(); + executor.shutdownNow(); + } + + private void pollOnceSafely() { + try { + pollOnce(); + } catch (final RuntimeException e) { + LOGGER.debug("Unexpected error while polling Feature Flagging HTTP configuration source", e); + } + } + + private boolean fetchAndApply() { + for (int attempt = 1; ; attempt++) { + try { + final UfcHttpResponse response = client.fetch(endpoint, config, etag); + if (closed) { + return false; + } + if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { + if (!waitBeforeRetry(attempt)) { + return false; + } + continue; + } + synchronized (lifecycleLock) { + return !closed && apply(response); + } + } catch (final IOException e) { + if (closed) { + return false; + } + if (attempt == MAX_ATTEMPTS) { + LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); + return false; + } + if (!waitBeforeRetry(attempt)) { + return false; + } + } + } + } + + private boolean waitBeforeRetry(final int attempt) { + if (closed) { + return false; + } + try { + retrySleeper.sleep(retryDelayMillis(pollIntervalMillis, attempt, jitter.getAsDouble())); + return !closed; + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + private boolean apply(final UfcHttpResponse response) { + if (response.status == HttpURLConnection.HTTP_NOT_MODIFIED) { + return true; + } + if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED + || response.status == HttpURLConnection.HTTP_FORBIDDEN + || response.status != HttpURLConnection.HTTP_OK + || response.body == null) { + return false; + } + final ServerConfiguration configuration; + try { + configuration = + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( + response.body); + } catch (final IOException | RuntimeException e) { + LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); + return false; + } + if (configuration == null) { + return false; + } + FeatureFlaggingGateway.dispatch(configuration); + updateEtag(response.etag); + return true; + } + + private static boolean isRetryableStatus(final int status) { + return status == HttpURLConnection.HTTP_CLIENT_TIMEOUT + || status == 429 + || (status >= 500 && status <= 599); + } + + private void updateEtag(final String nextEtag) { + if (nextEtag != null && !nextEtag.trim().isEmpty()) { + etag = nextEtag; + } + } + + static HttpUrl endpoint(final Config config) { + final String endpoint = datadogApiServerDistributionEndpoint(config); + final HttpUrl parsed = HttpUrl.parse(endpoint); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); + } + return parsed; + } + + private static String datadogApiServerDistributionEndpoint(final Config config) { + final StringBuilder endpoint = + new StringBuilder("https://api.") + .append(config.getSite().toLowerCase(Locale.ROOT)) + .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); + final String env = config.getEnv(); + if (env != null && !env.isEmpty()) { + endpoint.append("?dd_env=").append(urlEncode(env)); + } + return endpoint.toString(); + } + + private static String urlEncode(final String value) { + try { + return URLEncoder.encode(value, "UTF-8"); + } catch (final IOException e) { + throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + } + } + + static long millis(final int seconds) { + return TimeUnit.SECONDS.toMillis(seconds); + } + + static long retryDelayMillis( + final long pollIntervalMillis, final int attempt, final double jitter) { + final long baseDelay; + if (attempt == 1) { + baseDelay = clamp(pollIntervalMillis / 6, FIRST_RETRY_MIN_MILLIS, FIRST_RETRY_MAX_MILLIS); + } else if (attempt == 2) { + baseDelay = clamp(pollIntervalMillis / 3, SECOND_RETRY_MIN_MILLIS, SECOND_RETRY_MAX_MILLIS); + } else { + throw new IllegalArgumentException("Unsupported Feature Flagging retry attempt: " + attempt); + } + return Math.max(1, Math.round(baseDelay * jitter)); + } + + private static long clamp(final long value, final long minimum, final long maximum) { + return Math.max(minimum, Math.min(maximum, value)); + } + + interface UfcHttpClient { + UfcHttpResponse fetch(HttpUrl endpoint, Config config, String etag) throws IOException; + + void cancel(); + } + + interface RetrySleeper { + void sleep(long delayMillis) throws InterruptedException; + } + + static final class UfcHttpResponse { + final int status; + final String etag; + final byte[] body; + + UfcHttpResponse(final int status, final String etag, final byte[] body) { + this.status = status; + this.etag = etag; + this.body = body; + } + } + + static final class OkHttpUfcHttpClient implements UfcHttpClient { + private final OkHttpClient httpClient; + private final AtomicReference activeCall = new AtomicReference<>(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + + OkHttpUfcHttpClient(final OkHttpClient httpClient) { + this.httpClient = httpClient; + } + + @Override + public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final String etag) + throws IOException { + final Map headers = new HashMap<>(); + if (etag != null) { + headers.put("If-None-Match", etag); + } + final Request request = prepareRequest(endpoint, headers, config, true).get().build(); + final Call call = httpClient.newCall(request); + if (!activeCall.compareAndSet(null, call)) { + throw new IllegalStateException("Feature Flagging HTTP request already in flight"); + } + if (cancelled.get()) { + call.cancel(); + } + try (Response response = call.execute()) { + final ResponseBody responseBody = response.body(); + return new UfcHttpResponse(response.code(), response.header("ETag"), responseBody.bytes()); + } finally { + activeCall.compareAndSet(call, null); + } + } + + @Override + public void cancel() { + cancelled.set(true); + final Call call = activeCall.get(); + if (call != null) { + call.cancel(); + } + } + } +} diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java new file mode 100644 index 00000000000..4d5e0651c8e --- /dev/null +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -0,0 +1,805 @@ +package com.datadog.featureflag; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; + +import datadog.communication.http.OkHttpUtils; +import datadog.trace.agent.test.server.http.JavaTestHttpServer; +import datadog.trace.api.Config; +import datadog.trace.api.featureflag.FeatureFlaggingGateway; +import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class AgentlessConfigurationSourceTest { + private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/server-distribution"; + + @Mock private FeatureFlaggingGateway.ConfigListener listener; + + @AfterEach + void cleanup() { + FeatureFlaggingGateway.removeConfigListener(listener); + FeatureFlaggingGateway.dispatch((ServerConfiguration) null); + } + + @Test + void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { + final Config config = config("datad0g.com", "staging env"); + + assertEquals( + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void derivesDatadogApiServerDistributionEndpointWithoutEnv() { + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", "")).toString()); + assertEquals( + "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); + } + + @Test + void rejectsInvalidDatadogApiServerDistributionEndpoint() { + assertThrows( + IllegalArgumentException.class, + () -> AgentlessConfigurationSource.endpoint(config("datadoghq.com:bad", ""))); + } + + @Test + void defaultConstructorBuildsHttpClientFromConfig() { + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource(config("datad0g.com", "staging")); + + service.close(); + } + + @Test + void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .addHeader("ETag", "etag-b") + .send(emptyConfig()))))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), "etag-a"); + + assertEquals(HttpURLConnection.HTTP_OK, response.status); + assertEquals("etag-b", response.etag); + assertEquals(emptyConfig(), new String(response.body, UTF_8)); + assertEquals("test-api-key", server.getLastRequest().getHeader("DD-API-KEY")); + assertEquals("etag-a", server.getLastRequest().getHeader("If-None-Match")); + assertEquals("java", server.getLastRequest().getHeader("Datadog-Meta-Lang")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> + api.getResponse() + .status(HttpURLConnection.HTTP_NO_CONTENT) + .send())))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null); + + assertEquals(HttpURLConnection.HTTP_NO_CONTENT, response.status); + assertNull(response.etag); + assertEquals(0, response.body.length); + assertNull(server.getLastRequest().getHeader("If-None-Match")); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientCancellationInterruptsInFlightRequest() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> { + requestStarted.countDown(); + assertTrue(releaseRequest.await(1, TimeUnit.SECONDS)); + api.getResponse().send(emptyConfig()); + })))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + + try { + final Future response = + runner.submit( + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + assertThrows( + IllegalStateException.class, + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + + client.cancel(); + + final ExecutionException failure = + assertThrows(ExecutionException.class, () -> response.get(1, TimeUnit.SECONDS)); + assertInstanceOf(IOException.class, failure.getCause()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientCancellationBeforeFetchPreventsRequest() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> h.get(CONFIG_PATH, api -> api.getResponse().send(emptyConfig()))))) { + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + client.cancel(); + + assertThrows( + IOException.class, + () -> + client.fetch( + HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)), config(), null)); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void realHttpClientTimesOutDelayedResponse() throws Exception { + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> + s.handlers( + h -> + h.get( + CONFIG_PATH, + api -> { + TimeUnit.MILLISECONDS.sleep(500); + api.getResponse().send(emptyConfig()); + })))) { + final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); + final OkHttpClient httpClient = OkHttpUtils.buildHttpClient(endpoint, 50); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + try { + assertThrows(IOException.class, () -> client.fetch(endpoint, config(), null)); + } finally { + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + + @Test + void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("test-api-key", client.requests.get(0).apiKey); + assertNull(client.requests.get(0).etag); + } + + @Test + void ignoresBlankEtag() throws Exception { + final FakeClient client = + new FakeClient(response(200, " ", emptyConfig()), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { + final FakeClient client = + new FakeClient(response(200, "etag-a", emptyConfig()), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + } + + @Test + void coldNotModifiedDoesNotEstablishEtag() throws Exception { + final FakeClient client = + new FakeClient(response(304, "etag-cold", null), response(200, "etag-warm", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void failedGatewayDispatchDoesNotAdvanceEtag() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + final FeatureFlaggingGateway.ConfigListener failingListener = + configuration -> { + throw new IllegalStateException("listener rejected configuration"); + }; + FeatureFlaggingGateway.addConfigListener(failingListener); + + try { + assertThrows(IllegalStateException.class, service::pollOnce); + FeatureFlaggingGateway.removeConfigListener(failingListener); + + assertTrue(service.pollOnce()); + + assertNull(client.requests.get(1).etag); + } finally { + FeatureFlaggingGateway.removeConfigListener(failingListener); + } + } + + @Test + void keepsLastKnownGoodOnAuthFailureAndMalformedPayload() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-good", emptyConfig()), + response(401, null, null), + response(200, null, "{not-json}"), + response(200, null, "{\"flags\":[]}")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(listener).accept(any(ServerConfiguration.class)); + assertEquals("etag-good", client.requests.get(1).etag); + assertEquals("etag-good", client.requests.get(2).etag); + assertEquals("etag-good", client.requests.get(3).etag); + } + + @Test + void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { + final FakeClient client = + new FakeClient( + response(403, null, null), + response(404, null, null), + response(600, null, null), + response(200, null, null), + response(200, null, "null")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verifyNoInteractions(listener); + } + + @Test + void retriesTimeoutBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesClientTimeoutAndRateLimitStatusBeforeApplyingConfig() throws Exception { + final FakeClient client = + new FakeClient( + response(408, null, null), + response(200, "etag-a", emptyConfig()), + response(429, null, null), + response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertEquals(4, client.calls.get()); + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + } + + @Test + void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { + final FakeClient client = new FakeClient(response(500, null, null), response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(2, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void givesUpAfterRetryableFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + response(503, null, null), response(503, null, null), response(503, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void givesUpAfterIoFailuresAreExhausted() throws Exception { + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source"), + new SocketTimeoutException("slow HTTP configuration source")); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertFalse(service.pollOnce()); + + assertEquals(3, client.calls.get()); + verifyNoInteractions(listener); + } + + @Test + void usesIntervalAwareRetryBackoff() throws Exception { + final List delays = new ArrayList<>(); + final FakeClient client = + new FakeClient( + response(503, null, null), + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client, delays::add, () -> 1.0); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + assertEquals(java.util.Arrays.asList(5_000L, 10_000L), delays); + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void clampsAndJittersRetryBackoff() { + assertEquals(2_000, AgentlessConfigurationSource.retryDelayMillis(1_000, 1, 1.0)); + assertEquals(5_000, AgentlessConfigurationSource.retryDelayMillis(1_000, 2, 1.0)); + assertEquals(10_000, AgentlessConfigurationSource.retryDelayMillis(600_000, 1, 1.0)); + assertEquals(30_000, AgentlessConfigurationSource.retryDelayMillis(600_000, 2, 1.0)); + assertEquals(6_000, AgentlessConfigurationSource.retryDelayMillis(30_000, 1, 1.2)); + assertThrows( + IllegalArgumentException.class, + () -> AgentlessConfigurationSource.retryDelayMillis(30_000, 3, 1.0)); + } + + @Test + void rejectsOverlappingPolls() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newFixedThreadPool(2); + + try { + final Future first = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + final Future second = runner.submit(service::pollOnce); + + assertFalse(second.get(1, TimeUnit.SECONDS)); + releaseRequest.countDown(); + assertTrue(first.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void initSchedulesPollAndCloseCancelsFuture() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 60_000, + client, + Executors.newSingleThreadScheduledExecutor()); + FeatureFlaggingGateway.addConfigListener(listener); + + service.init(); + awaitCalls(client, 1); + service.close(); + + verify(listener).accept(any(ServerConfiguration.class)); + } + + @Test + void repeatedInitStartsOnlyOnePoller() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.init(); + service.init(); + awaitCalls(client, 1); + service.close(); + + assertEquals(1, client.calls.get()); + } + + @Test + void closeCancelsInFlightRequestAndIgnoresLateSuccess() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + final Future poll = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertFalse(poll.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.cancelCalls.get()); + assertEquals(1, client.calls.get()); + verifyNoInteractions(listener); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closeDuringIoFailurePreventsRetry() throws Exception { + final CountDownLatch requestStarted = new CountDownLatch(1); + final CountDownLatch releaseRequest = new CountDownLatch(1); + final FakeClient client = + new FakeClient(new SocketTimeoutException("slow HTTP configuration source")); + client.block(requestStarted, releaseRequest); + final AgentlessConfigurationSource service = service(client); + final ExecutorService runner = Executors.newSingleThreadExecutor(); + + try { + final Future poll = runner.submit(service::pollOnce); + assertTrue(requestStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertFalse(poll.get(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } finally { + releaseRequest.countDown(); + runner.shutdownNow(); + } + } + + @Test + void closeInterruptsRetryBackoff() throws Exception { + final CountDownLatch backoffStarted = new CountDownLatch(1); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final FakeClient client = + new FakeClient( + new SocketTimeoutException("slow HTTP configuration source"), + response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> { + backoffStarted.countDown(); + TimeUnit.MINUTES.sleep(1); + }, + () -> 1.0); + + service.init(); + assertTrue(backoffStarted.await(1, TimeUnit.SECONDS)); + + service.close(); + + assertTrue(executor.awaitTermination(1, TimeUnit.SECONDS)); + assertEquals(1, client.calls.get()); + } + + @Test + void closePreventsFurtherPolls() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.close(); + + assertFalse(service.pollOnce()); + assertEquals(0, client.calls.get()); + } + + @Test + void initAfterCloseDoesNotSchedulePoll() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); + final AgentlessConfigurationSource service = service(client); + + service.close(); + service.init(); + + assertEquals(0, client.calls.get()); + } + + @Nested + class SystemTestParity { + @Test + void preservesSystemTestSourceTransitionsAndLastKnownGoodState() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), + response(304, "etag-must-not-replace-a", null), + response(509, null, null), + response(200, "etag-b", emptyConfig()), + response(200, "etag-c", "{not-json}"), + response(401, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + assertEquals("etag-a", client.requests.get(1).etag); + assertEquals("etag-a", client.requests.get(2).etag); + assertEquals("etag-a", client.requests.get(3).etag); + assertEquals("etag-b", client.requests.get(4).etag); + assertEquals("etag-b", client.requests.get(5).etag); + } + } + + private static AgentlessConfigurationSource service(final FakeClient client) { + return service(client, delay -> {}, () -> 1.0); + } + + private static AgentlessConfigurationSource service( + final FakeClient client, + final AgentlessConfigurationSource.RetrySleeper retrySleeper, + final java.util.function.DoubleSupplier jitter) { + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + return new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + retrySleeper, + jitter); + } + + private static Config config() { + return config("datadoghq.com", ""); + } + + private static Config config(final String site, final String env) { + final Config config = mock(Config.class); + lenient() + .when(config.getFeatureFlaggingConfigurationSourcePollIntervalSeconds()) + .thenReturn(30); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) + .thenReturn(2); + lenient().when(config.getApiKey()).thenReturn("test-api-key"); + lenient().when(config.getSite()).thenReturn(site); + lenient().when(config.getEnv()).thenReturn(env); + return config; + } + + private static AgentlessConfigurationSource.UfcHttpResponse response( + final int status, final String etag, final String body) { + return new AgentlessConfigurationSource.UfcHttpResponse( + status, etag, body == null ? null : body.getBytes(UTF_8)); + } + + private static String emptyConfig() { + return "{" + + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Test\"}," + + "\"flags\":{}" + + "}"; + } + + private static void awaitCalls(final FakeClient client, final int count) throws Exception { + for (int i = 0; i < 100; i++) { + if (client.calls.get() >= count) { + return; + } + TimeUnit.MILLISECONDS.sleep(10); + } + assertEquals(count, client.calls.get()); + } + + private static final class FakeClient implements AgentlessConfigurationSource.UfcHttpClient { + private final AtomicInteger calls = new AtomicInteger(); + private final AtomicInteger cancelCalls = new AtomicInteger(); + private final List requests = new ArrayList<>(); + private final BlockingQueue responses = new LinkedBlockingQueue<>(); + private CountDownLatch requestStarted; + private CountDownLatch releaseRequest; + + private FakeClient(final Object... responses) { + for (final Object response : responses) { + this.responses.add(response); + } + } + + private void block(final CountDownLatch requestStarted, final CountDownLatch releaseRequest) { + this.requestStarted = requestStarted; + this.releaseRequest = releaseRequest; + } + + @Override + public AgentlessConfigurationSource.UfcHttpResponse fetch( + final HttpUrl endpoint, final Config config, final String etag) throws IOException { + calls.incrementAndGet(); + requests.add(new Request(config.getApiKey(), etag)); + if (requestStarted != null) { + requestStarted.countDown(); + } + if (releaseRequest != null) { + await(releaseRequest); + } + final Object response = responses.remove(); + if (response instanceof IOException) { + throw (IOException) response; + } + if (response instanceof RuntimeException) { + throw (RuntimeException) response; + } + return (AgentlessConfigurationSource.UfcHttpResponse) response; + } + + @Override + public void cancel() { + cancelCalls.incrementAndGet(); + if (releaseRequest != null) { + releaseRequest.countDown(); + } + } + + private static void await(final CountDownLatch latch) throws IOException { + try { + if (!latch.await(1, TimeUnit.SECONDS)) { + throw new SocketTimeoutException("test request did not release"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + } + + private static final class Request { + private final String apiKey; + private final String etag; + + private Request(final String apiKey, final String etag) { + this.apiKey = apiKey; + this.etag = etag; + } + } +} From d378350cf1916ddd20a4cdde14393af9a70001de Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Fri, 10 Jul 2026 23:58:40 -0400 Subject: [PATCH 03/10] Support custom agentless UFC endpoints --- .../AgentlessConfigurationSource.java | 22 +++++++++++- .../AgentlessConfigurationSourceTest.java | 36 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index b3c58631d3a..07deeac0ffc 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -242,7 +242,11 @@ private void updateEtag(final String nextEtag) { } static HttpUrl endpoint(final Config config) { - final String endpoint = datadogApiServerDistributionEndpoint(config); + final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl(); + final String endpoint = + configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredBaseUrl(configuredBaseUrl); final HttpUrl parsed = HttpUrl.parse(endpoint); if (parsed == null) { throw new IllegalArgumentException( @@ -251,6 +255,22 @@ static HttpUrl endpoint(final Config config) { return parsed; } + private static String endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { + final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim()); + if (parsed == null) { + throw new IllegalArgumentException( + "Invalid Feature Flagging HTTP configuration source URL: " + configuredBaseUrl); + } + if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) { + return parsed + .newBuilder() + .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) + .build() + .toString(); + } + return parsed.toString(); + } + private static String datadogApiServerDistributionEndpoint(final Config config) { final StringBuilder endpoint = new StringBuilder("https://api.") diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 4d5e0651c8e..c79de87502d 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -74,6 +74,41 @@ void derivesDatadogApiServerDistributionEndpointWithoutEnv() { AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); } + @Test + void appendsServerDistributionPathToConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080"); + + assertEquals( + "http://mock-backend:8080/api/v2/feature-flagging/config/server-distribution", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void usesConfiguredAgentlessEndpointWithPathUnchanged() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://mock-backend:8080/custom/ufc?tenant=test"); + + assertEquals( + "http://mock-backend:8080/custom/ufc?tenant=test", + AgentlessConfigurationSource.endpoint(config).toString()); + } + + @Test + void rejectsInvalidConfiguredAgentlessBaseUrl() { + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("not a URL"); + + assertThrows( + IllegalArgumentException.class, () -> AgentlessConfigurationSource.endpoint(config)); + } + @Test void rejectsInvalidDatadogApiServerDistributionEndpoint() { assertThrows( @@ -702,6 +737,7 @@ private static Config config(final String site, final String env) { lenient() .when(config.getFeatureFlaggingConfigurationSourceRequestTimeoutSeconds()) .thenReturn(2); + lenient().when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()).thenReturn(null); lenient().when(config.getApiKey()).thenReturn("test-api-key"); lenient().when(config.getSite()).thenReturn(site); lenient().when(config.getEnv()).thenReturn(env); From f9e70e67c4f16185dbf09bbb458c5572684fb2b2 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Sat, 11 Jul 2026 00:01:22 -0400 Subject: [PATCH 04/10] Select the Feature Flagging configuration source --- .../java/datadog/trace/bootstrap/Agent.java | 17 ++++ .../AgentFeatureFlaggingLifecycleTest.java | 48 ++++++++++ .../featureflag/FeatureFlaggingSystem.java | 94 ++++++++++++++---- .../FeatureFlaggingSystemTest.java | 95 +++++++++++++++++++ .../feature-flagging-api/README.md | 10 +- 5 files changed, 246 insertions(+), 18 deletions(-) create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 876a07f1fd1..056e47147f9 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -528,6 +528,9 @@ public static void shutdown(final boolean sync) { if (flareEnabled) { stopFlarePoller(); } + if (featureFlaggingEnabled) { + shutdownFeatureFlagging(AGENT_CLASSLOADER); + } if (agentlessLogSubmissionEnabled) { shutdownLogsIntake(); @@ -1179,6 +1182,20 @@ private static void maybeStartFeatureFlagging(final Class scoClass, final Obj } } + static void shutdownFeatureFlagging(final ClassLoader agentClassLoader) { + if (agentClassLoader == null) { + return; + } + try { + final Class ffSysClass = + agentClassLoader.loadClass("com.datadog.featureflag.FeatureFlaggingSystem"); + final Method stopMethod = ffSysClass.getMethod("stop"); + stopMethod.invoke(null); + } catch (final Throwable e) { + log.warn("Unable to stop Feature Flagging subsystem", e); + } + } + private static void maybeInstallLogsIntake(Class scoClass, Object sco) { if (agentlessLogSubmissionEnabled || appLogsCollectionEnabled) { StaticEventLogger.begin("Logs Intake"); diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java new file mode 100644 index 00000000000..75e9987c4da --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/AgentFeatureFlaggingLifecycleTest.java @@ -0,0 +1,48 @@ +package datadog.trace.bootstrap; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class AgentFeatureFlaggingLifecycleTest { + + @BeforeEach + void reset() { + FakeFeatureFlaggingSystem.stopCalls.set(0); + } + + @Test + void shutdownInvokesFeatureFlaggingSystemStopThroughAgentClassLoader() { + final ClassLoader classLoader = + new ClassLoader(null) { + @Override + public Class loadClass(final String name) throws ClassNotFoundException { + if ("com.datadog.featureflag.FeatureFlaggingSystem".equals(name)) { + return FakeFeatureFlaggingSystem.class; + } + return super.loadClass(name); + } + }; + + Agent.shutdownFeatureFlagging(classLoader); + + assertEquals(1, FakeFeatureFlaggingSystem.stopCalls.get()); + } + + @Test + void shutdownIsNoopBeforeAgentClassLoaderExists() { + Agent.shutdownFeatureFlagging(null); + + assertEquals(0, FakeFeatureFlaggingSystem.stopCalls.get()); + } + + public static final class FakeFeatureFlaggingSystem { + private static final AtomicInteger stopCalls = new AtomicInteger(); + + public static void stop() { + stopCalls.incrementAndGet(); + } + } +} diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 2011d31e83d..f52ef257450 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -14,31 +14,93 @@ public class FeatureFlaggingSystem { private FeatureFlaggingSystem() {} - public static void start(final SharedCommunicationObjects sco) { + public static synchronized void start(final SharedCommunicationObjects sco) { + if (CONFIG_SERVICE != null || EXPOSURE_WRITER != null) { + LOGGER.debug("Feature Flagging system already started"); + return; + } LOGGER.debug("Feature Flagging system starting"); final Config config = Config.get(); + final ConfigurationSourceService configService = createConfigurationSourceService(sco, config); + final ExposureWriter exposureWriter = new ExposureWriterImpl(sco, config); + initialize(configService, exposureWriter); + + LOGGER.debug("Feature Flagging system started"); + } - if (!config.isRemoteConfigEnabled()) { - throw new IllegalStateException("Feature Flagging system started without RC"); + static void initialize( + final ConfigurationSourceService configService, final ExposureWriter exposureWriter) { + try { + if (configService != null) { + configService.init(); + } + exposureWriter.init(); + CONFIG_SERVICE = configService; + EXPOSURE_WRITER = exposureWriter; + } catch (final RuntimeException | Error e) { + exposureWriter.close(); + if (configService != null) { + configService.close(); + } + throw e; } - CONFIG_SERVICE = new RemoteConfigServiceImpl(sco, config); - CONFIG_SERVICE.init(); + } - EXPOSURE_WRITER = new ExposureWriterImpl(sco, config); - EXPOSURE_WRITER.init(); + static ConfigurationSourceService createConfigurationSourceService( + final SharedCommunicationObjects sco, final Config config) { + final ConfigurationSource configurationSource = + ConfigurationSource.from(config.getFeatureFlaggingConfigurationSource()); - LOGGER.debug("Feature Flagging system started"); + if (configurationSource == ConfigurationSource.REMOTE_CONFIG) { + if (!config.isRemoteConfigEnabled()) { + throw new IllegalStateException("Feature Flagging system started without RC"); + } + return new RemoteConfigServiceImpl(sco, config); + } + if (configurationSource == ConfigurationSource.AGENTLESS) { + return new AgentlessConfigurationSource(config); + } + LOGGER.debug( + "Feature Flagging offline configuration source selected; no config service started"); + return null; } - public static void stop() { - if (EXPOSURE_WRITER != null) { - EXPOSURE_WRITER.close(); - EXPOSURE_WRITER = null; - } - if (CONFIG_SERVICE != null) { - CONFIG_SERVICE.close(); - CONFIG_SERVICE = null; + public static synchronized void stop() { + final ExposureWriter exposureWriter = EXPOSURE_WRITER; + final ConfigurationSourceService configService = CONFIG_SERVICE; + EXPOSURE_WRITER = null; + CONFIG_SERVICE = null; + try { + if (exposureWriter != null) { + exposureWriter.close(); + } + } finally { + if (configService != null) { + configService.close(); + } } LOGGER.debug("Feature Flagging system stopped"); } + + private enum ConfigurationSource { + AGENTLESS("agentless"), + REMOTE_CONFIG("remote_config"), + OFFLINE("offline"); + + private final String value; + + ConfigurationSource(final String value) { + this.value = value; + } + + private static ConfigurationSource from(final String value) { + for (final ConfigurationSource source : values()) { + if (source.value.equals(value)) { + return source; + } + } + throw new IllegalArgumentException( + "Unsupported Feature Flagging configuration source: " + value); + } + } } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 5b088d99b9d..82ce01e4929 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -2,9 +2,13 @@ import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE; import static datadog.trace.api.config.RemoteConfigConfig.REMOTE_CONFIGURATION_ENABLED; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -37,12 +41,14 @@ void testFeatureFlagSystemInitialization() { sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + FeatureFlaggingSystem.start(sharedCommunicationObjects); FeatureFlaggingSystem.start(sharedCommunicationObjects); verify(poller).addCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); verify(poller).addListener(eq(Product.FFE_FLAGS), any(ConfigurationDeserializer.class), any()); verify(poller).start(); + FeatureFlaggingSystem.stop(); FeatureFlaggingSystem.stop(); verify(poller).removeCapabilities(Capabilities.CAPABILITY_FFE_FLAG_CONFIGURATION_RULES); @@ -64,4 +70,93 @@ void testThatRemoteConfigIsRequired() { FeatureFlaggingSystem.stop(); } } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "agentless") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "false") + void agentlessConfigurationSourceUsesHttpServiceWithoutRemoteConfig() { + assertInstanceOf( + AgentlessConfigurationSource.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "remote_config") + @WithConfig(key = REMOTE_CONFIGURATION_ENABLED, value = "true") + void explicitRemoteConfigUsesRemoteConfigService() { + SharedCommunicationObjects sharedCommunicationObjects = sharedCommunicationObjects(); + when(sharedCommunicationObjects.configurationPoller(any(Config.class))) + .thenReturn(mock(ConfigurationPoller.class)); + + assertInstanceOf( + RemoteConfigServiceImpl.class, + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects, Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "invalid") + void invalidConfigurationSourceFailsBeforeStartingNetworkSource() { + assertThrows( + IllegalArgumentException.class, + () -> + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void offlineConfigurationSourceDoesNotStartNetworkSource() { + assertNull( + FeatureFlaggingSystem.createConfigurationSourceService( + sharedCommunicationObjects(), Config.get())); + } + + @Test + @WithConfig(key = FEATURE_FLAGS_CONFIGURATION_SOURCE, value = "offline") + void startWithOfflineConfigurationSourceSkipsConfigService() { + try { + assertDoesNotThrow(() -> FeatureFlaggingSystem.start(sharedCommunicationObjects())); + } finally { + FeatureFlaggingSystem.stop(); + } + } + + @Test + void initializationFailureClosesConfigurationSourceAndExposureWriter() { + ConfigurationSourceService configService = mock(ConfigurationSourceService.class); + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + + assertThrows( + IllegalStateException.class, + () -> FeatureFlaggingSystem.initialize(configService, exposureWriter)); + + verify(configService).init(); + verify(configService).close(); + verify(exposureWriter).close(); + } + + @Test + void initializationFailureWithoutConfigurationSourceClosesExposureWriter() { + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + + assertThrows( + IllegalStateException.class, () -> FeatureFlaggingSystem.initialize(null, exposureWriter)); + + verify(exposureWriter).close(); + } + + private static SharedCommunicationObjects sharedCommunicationObjects() { + DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); + when(discovery.supportsEvpProxy()).thenReturn(true); + when(discovery.getEvpProxyEndpoint()).thenReturn("/evp_proxy/"); + SharedCommunicationObjects sharedCommunicationObjects = mock(SharedCommunicationObjects.class); + when(sharedCommunicationObjects.featuresDiscovery(any(Config.class))).thenReturn(discovery); + sharedCommunicationObjects.agentUrl = HttpUrl.get("http://localhost"); + sharedCommunicationObjects.agentHttpClient = new OkHttpClient.Builder().build(); + return sharedCommunicationObjects; + } } diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index 47733020559..ca9dc9e7d0a 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -85,5 +85,11 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc ## Requirements - Java 11+ -- Datadog Agent with Remote Configuration enabled -- `DD_EXPERIMENTAL_FLAGGING_PROVIDER_ENABLED=true` +- `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless + backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a + different HTTP backend while keeping agentless delivery semantics. A bare + host uses the standard server-distribution path; a URL with a path is used as + the exact UFC endpoint. `remote_config` uses the existing Agent Remote + Configuration path. `offline` is reserved for startup-provided UFC bytes; + until those bytes are implemented, no network source starts and evaluations + use defaults. From 17a02ad627bef6028356acc8485039458c55660d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Mon, 13 Jul 2026 08:15:22 -0400 Subject: [PATCH 05/10] Warn on agentless authentication failures --- .../feature-flagging-lib/build.gradle.kts | 1 + .../AgentlessConfigurationSource.java | 40 ++++++++++++++++--- .../AgentlessConfigurationSourceTest.java | 34 ++++++++++++++++ 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 548f5e26d2b..d1fb611fe5e 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { api(project(":communication")) implementation(project(":internal-api")) api(project(":products:feature-flagging:feature-flagging-bootstrap")) + implementation(project(":utils:logging-utils")) api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 07deeac0ffc..0b3ce685564 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -4,6 +4,7 @@ import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; import datadog.communication.http.OkHttpUtils; +import datadog.logging.RatelimitedLogger; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; import datadog.trace.api.featureflag.ufc.v1.ServerConfiguration; @@ -38,6 +39,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; + private static final int MINUTES_BETWEEN_AUTH_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; private static final long FIRST_RETRY_MAX_MILLIS = 10_000; private static final long SECOND_RETRY_MIN_MILLIS = 5_000; @@ -51,6 +53,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private final ScheduledExecutorService executor; private final RetrySleeper retrySleeper; private final DoubleSupplier jitter; + private final RatelimitedLogger ratelimitedLogger; private final Object lifecycleLock = new Object(); private final AtomicBoolean polling = new AtomicBoolean(); private volatile boolean closed; @@ -73,7 +76,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint Executors.newSingleThreadScheduledExecutor( new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), TimeUnit.MILLISECONDS::sleep, - () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER)); + () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER), + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -89,7 +93,8 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint client, executor, TimeUnit.MILLISECONDS::sleep, - () -> 1.0); + () -> 1.0, + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -100,6 +105,26 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint final ScheduledExecutorService executor, final RetrySleeper retrySleeper, final DoubleSupplier jitter) { + this( + endpoint, + config, + pollIntervalMillis, + client, + executor, + retrySleeper, + jitter, + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + } + + AgentlessConfigurationSource( + final HttpUrl endpoint, + final Config config, + final long pollIntervalMillis, + final UfcHttpClient client, + final ScheduledExecutorService executor, + final RetrySleeper retrySleeper, + final DoubleSupplier jitter, + final RatelimitedLogger ratelimitedLogger) { this.endpoint = endpoint; this.config = config; this.pollIntervalMillis = pollIntervalMillis; @@ -107,6 +132,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint this.executor = executor; this.retrySleeper = retrySleeper; this.jitter = jitter; + this.ratelimitedLogger = ratelimitedLogger; } @Override @@ -207,9 +233,13 @@ private boolean apply(final UfcHttpResponse response) { return true; } if (response.status == HttpURLConnection.HTTP_UNAUTHORIZED - || response.status == HttpURLConnection.HTTP_FORBIDDEN - || response.status != HttpURLConnection.HTTP_OK - || response.body == null) { + || response.status == HttpURLConnection.HTTP_FORBIDDEN) { + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + response.status); + return false; + } + if (response.status != HttpURLConnection.HTTP_OK || response.body == null) { return false; } final ServerConfiguration configuration; diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index c79de87502d..0ecc790b3e4 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import datadog.communication.http.OkHttpUtils; +import datadog.logging.RatelimitedLogger; import datadog.trace.agent.test.server.http.JavaTestHttpServer; import datadog.trace.api.Config; import datadog.trace.api.featureflag.FeatureFlaggingGateway; @@ -413,6 +414,39 @@ void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { verifyNoInteractions(listener); } + @Test + void warnsRateLimitedOnUnauthorizedAndForbidden() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); + final FakeClient client = new FakeClient(response(401, null, null), response(403, null, null)); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); + + try { + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + HttpURLConnection.HTTP_UNAUTHORIZED); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint returned HTTP {}; verify DD_API_KEY is configured and valid", + HttpURLConnection.HTTP_FORBIDDEN); + } finally { + service.close(); + } + } + @Test void retriesTimeoutBeforeApplyingConfig() throws Exception { final FakeClient client = From 505753954704653f1b65dc579da8832fa4444408 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 14 Jul 2026 22:24:28 -0400 Subject: [PATCH 06/10] fix(ffe): handle nullable agentless response bodies --- .../datadog/trace/api/ConfigDefaults.java | 7 ++--- .../AgentlessConfigurationSource.java | 16 ++++++---- .../AgentlessConfigurationSourceTest.java | 30 +++++++++++++++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 532e9f47d71..0c008f41094 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -45,14 +45,13 @@ public final class ConfigDefaults { public static final String DEFAULT_SERVLET_ROOT_CONTEXT_SERVICE_NAME = "root-servlet"; public static final String DEFAULT_AGENT_WRITER_TYPE = "DDAgentWriter"; public static final boolean DEFAULT_STARTUP_LOGS_ENABLED = true; - - static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; - static final String DEFAULT_SITE = "datadoghq.com"; - public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; + static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; + static final String DEFAULT_SITE = "datadoghq.com"; + static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; static final boolean DEFAULT_TRACE_ENABLED = true; diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index 0b3ce685564..be381893ccf 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.DoubleSupplier; +import javax.annotation.Nullable; import okhttp3.Call; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; @@ -354,10 +355,10 @@ interface RetrySleeper { static final class UfcHttpResponse { final int status; - final String etag; - final byte[] body; + @Nullable final String etag; + @Nullable final byte[] body; - UfcHttpResponse(final int status, final String etag, final byte[] body) { + UfcHttpResponse(final int status, @Nullable final String etag, @Nullable final byte[] body) { this.status = status; this.etag = etag; this.body = body; @@ -388,9 +389,12 @@ public UfcHttpResponse fetch(final HttpUrl endpoint, final Config config, final if (cancelled.get()) { call.cancel(); } - try (Response response = call.execute()) { - final ResponseBody responseBody = response.body(); - return new UfcHttpResponse(response.code(), response.header("ETag"), responseBody.bytes()); + try { + final Response response = call.execute(); + try (ResponseBody responseBody = response.body()) { + final byte[] body = responseBody != null ? responseBody.bytes() : null; + return new UfcHttpResponse(response.code(), response.header("ETag"), body); + } } finally { activeCall.compareAndSet(call, null); } diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 0ecc790b3e4..4d8714c0e67 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; @@ -35,8 +36,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.Call; import okhttp3.HttpUrl; import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Response; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -191,6 +195,32 @@ void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { } } + @Test + void httpClientAdapterPreservesMissingResponseBody() throws Exception { + final OkHttpClient httpClient = mock(OkHttpClient.class); + final Call call = mock(Call.class); + final HttpUrl endpoint = HttpUrl.get("http://localhost"); + final okhttp3.Request request = new okhttp3.Request.Builder().url(endpoint).build(); + final Response okHttpResponse = + new Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(HttpURLConnection.HTTP_OK) + .message("OK") + .build(); + when(httpClient.newCall(any())).thenReturn(call); + when(call.execute()).thenReturn(okHttpResponse); + final AgentlessConfigurationSource.OkHttpUfcHttpClient client = + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient); + + final AgentlessConfigurationSource.UfcHttpResponse response = + client.fetch(endpoint, config(), null); + + assertEquals(HttpURLConnection.HTTP_OK, response.status); + assertNull(response.etag); + assertNull(response.body); + } + @Test void realHttpClientCancellationInterruptsInFlightRequest() throws Exception { final CountDownLatch requestStarted = new CountDownLatch(1); From ffbf97023f38675007e899bd54b0bd56307e039d Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 14:36:02 -0400 Subject: [PATCH 07/10] Address agentless poller review feedback --- .../feature-flagging-api/README.md | 7 +- .../AgentlessConfigurationSource.java | 28 +++- .../AgentlessConfigurationSourceTest.java | 155 ++++++++++++++++-- 3 files changed, 169 insertions(+), 21 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index ca9dc9e7d0a..ae861d07afd 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -89,7 +89,12 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a different HTTP backend while keeping agentless delivery semantics. A bare host uses the standard server-distribution path; a URL with a path is used as - the exact UFC endpoint. `remote_config` uses the existing Agent Remote + the exact UFC endpoint. Configured URLs are opaque: the SDK does not add the + Datadog-managed `dd_env` query parameter, so custom backends must include any + required tenant or environment scope in the configured URL. The derived + Datadog-managed endpoint is intended for supported commercial sites; use an + explicit base URL elsewhere. Agentless responses do not have an SDK-imposed + payload-size limit. `remote_config` uses the existing Agent Remote Configuration path. `offline` is reserved for startup-provided UFC bytes; until those bytes are implemented, no network source starts and evaluations use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index be381893ccf..e9f0786d76a 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -40,7 +40,7 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = "/api/v2/feature-flagging/config/server-distribution"; private static final int MAX_ATTEMPTS = 3; - private static final int MINUTES_BETWEEN_AUTH_WARNINGS = 5; + private static final int MINUTES_BETWEEN_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; private static final long FIRST_RETRY_MAX_MILLIS = 10_000; private static final long SECOND_RETRY_MIN_MILLIS = 5_000; @@ -78,7 +78,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint new AgentThreadFactory(FEATURE_FLAG_CONFIGURATION_POLLER)), TimeUnit.MILLISECONDS::sleep, () -> ThreadLocalRandom.current().nextDouble(1 - RETRY_JITTER, 1 + RETRY_JITTER), - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -95,7 +95,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint executor, TimeUnit.MILLISECONDS::sleep, () -> 1.0, - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -114,7 +114,7 @@ private AgentlessConfigurationSource(final Config config, final HttpUrl endpoint executor, retrySleeper, jitter, - new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_AUTH_WARNINGS, TimeUnit.MINUTES)); + new RatelimitedLogger(LOGGER, MINUTES_BETWEEN_WARNINGS, TimeUnit.MINUTES)); } AgentlessConfigurationSource( @@ -192,11 +192,18 @@ private boolean fetchAndApply() { if (closed) { return false; } - if (isRetryableStatus(response.status) && attempt < MAX_ATTEMPTS) { - if (!waitBeforeRetry(attempt)) { - return false; + if (isRetryableStatus(response.status)) { + if (attempt < MAX_ATTEMPTS) { + if (!waitBeforeRetry(attempt)) { + return false; + } + continue; } - continue; + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", + MAX_ATTEMPTS, + response.status); + return false; } synchronized (lifecycleLock) { return !closed && apply(response); @@ -206,7 +213,10 @@ private boolean fetchAndApply() { return false; } if (attempt == MAX_ATTEMPTS) { - LOGGER.debug("Feature Flagging HTTP configuration source request failed", e); + ratelimitedLogger.warn( + "Feature Flagging agentless endpoint request failed after {} attempts", + MAX_ATTEMPTS, + e); return false; } if (!waitBeforeRetry(attempt)) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 4d8714c0e67..a5b6dc84bfb 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -45,6 +45,7 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -163,6 +164,41 @@ void realHttpClientSendsAgentlessHeadersAndReadsResponse() throws Exception { } } + @Test + void downloadsAndAppliesLargeUfcWithoutPayloadLimit() throws Exception { + final int flagCount = 5_000; + final String largeConfig = largeConfig(flagCount); + assertTrue(largeConfig.getBytes(UTF_8).length > 500_000); + + try (JavaTestHttpServer server = + JavaTestHttpServer.httpServer( + s -> s.handlers(h -> h.get(CONFIG_PATH, api -> api.getResponse().send(largeConfig))))) { + final HttpUrl endpoint = HttpUrl.get(server.getAddress().resolve(CONFIG_PATH)); + final OkHttpClient httpClient = new OkHttpClient.Builder().build(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + endpoint, + config(), + 30_000, + new AgentlessConfigurationSource.OkHttpUfcHttpClient(httpClient), + Executors.newSingleThreadScheduledExecutor()); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); + FeatureFlaggingGateway.addConfigListener(listener); + + try { + assertTrue(service.pollOnce()); + + verify(listener).accept(configuration.capture()); + assertEquals(flagCount, configuration.getValue().flags.size()); + } finally { + service.close(); + httpClient.dispatcher().executorService().shutdownNow(); + httpClient.connectionPool().evictAll(); + } + } + } + @Test void realHttpClientAllowsMissingEtagAndEmptyResponseBody() throws Exception { try (JavaTestHttpServer server = @@ -524,33 +560,73 @@ void retriesServerErrorThenKeepsColdStateOnNotModified() throws Exception { } @Test - void givesUpAfterRetryableFailuresAreExhausted() throws Exception { + void warnsRateLimitedAfterRetryableFailuresAreExhausted() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); final FakeClient client = new FakeClient( response(503, null, null), response(503, null, null), response(503, null, null)); - final AgentlessConfigurationSource service = service(client); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); - assertFalse(service.pollOnce()); + try { + assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); - verifyNoInteractions(listener); + assertEquals(3, client.calls.get()); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint failed after {} attempts with HTTP {}", 3, 503); + verifyNoInteractions(listener); + } finally { + service.close(); + } } @Test - void givesUpAfterIoFailuresAreExhausted() throws Exception { + void warnsRateLimitedAfterIoFailuresAreExhausted() throws Exception { + final RatelimitedLogger ratelimitedLogger = mock(RatelimitedLogger.class); + final SocketTimeoutException finalFailure = + new SocketTimeoutException("slow HTTP configuration source"); final FakeClient client = new FakeClient( new SocketTimeoutException("slow HTTP configuration source"), new SocketTimeoutException("slow HTTP configuration source"), - new SocketTimeoutException("slow HTTP configuration source")); - final AgentlessConfigurationSource service = service(client); + finalFailure); + final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 30_000, + client, + executor, + delay -> {}, + () -> 1.0, + ratelimitedLogger); FeatureFlaggingGateway.addConfigListener(listener); - assertFalse(service.pollOnce()); + try { + assertFalse(service.pollOnce()); - assertEquals(3, client.calls.get()); - verifyNoInteractions(listener); + assertEquals(3, client.calls.get()); + verify(ratelimitedLogger) + .warn( + "Feature Flagging agentless endpoint request failed after {} attempts", + 3, + finalFailure); + verifyNoInteractions(listener); + } finally { + service.close(); + } } @Test @@ -638,6 +714,39 @@ void repeatedInitStartsOnlyOnePoller() throws Exception { assertEquals(1, client.calls.get()); } + @Test + void scheduledPollContinuesAfterListenerRuntimeException() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), response(200, "etag-b", emptyConfig())); + final AgentlessConfigurationSource service = + new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config(), + 10, + client, + Executors.newSingleThreadScheduledExecutor()); + final AtomicInteger listenerCalls = new AtomicInteger(); + final FeatureFlaggingGateway.ConfigListener flakyListener = + configuration -> { + if (listenerCalls.incrementAndGet() == 1) { + throw new IllegalStateException("listener rejected first configuration"); + } + }; + FeatureFlaggingGateway.addConfigListener(flakyListener); + + try { + service.init(); + awaitCalls(client, 2); + + assertEquals(2, listenerCalls.get()); + assertNull(client.requests.get(1).etag); + } finally { + service.close(); + FeatureFlaggingGateway.removeConfigListener(flakyListener); + } + } + @Test void closeCancelsInFlightRequestAndIgnoresLateSuccess() throws Exception { final CountDownLatch requestStarted = new CountDownLatch(1); @@ -823,6 +932,30 @@ private static String emptyConfig() { + "}"; } + private static String largeConfig(final int flagCount) { + final StringBuilder json = + new StringBuilder( + "{\"createdAt\":\"2024-04-17T19:40:53.716Z\"," + + "\"format\":\"SERVER\"," + + "\"environment\":{\"name\":\"Large Test\"}," + + "\"flags\":{"); + for (int index = 0; index < flagCount; index++) { + if (index > 0) { + json.append(','); + } + final String flagKey = "large-flag-" + index; + json.append('"') + .append(flagKey) + .append("\":{\"key\":\"") + .append(flagKey) + .append( + "\",\"enabled\":true,\"variationType\":\"STRING\"," + + "\"variations\":{\"on\":{\"key\":\"on\",\"value\":\"on\"}}," + + "\"allocations\":[]}"); + } + return json.append("}}").toString(); + } + private static void awaitCalls(final FakeClient client, final int count) throws Exception { for (int i = 0; i < 100; i++) { if (client.calls.get() >= count) { From 3b9b97c470124442a93f3f40e186d7483bfc2dcd Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 14:40:14 -0400 Subject: [PATCH 08/10] Fix feature flag config test imports --- .../src/test/groovy/datadog/trace/api/ConfigTest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index 953c5ced8f3..a9046475a7c 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -57,8 +57,8 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS -import static datadog.trace.api.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_ENABLED import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_METRICS_CONFIGS From eaaea2762128decffb6fa20fc7b1956d17671362 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 15:52:39 -0400 Subject: [PATCH 09/10] Address additional agentless source review feedback --- .../datadog/trace/api/ConfigDefaults.java | 6 +-- .../datadog/trace/api/ConfigTest.groovy | 22 ++++++++ .../featureflag/FeatureFlaggingSystem.java | 9 ++-- .../FeatureFlaggingSystemTest.java | 14 ++++++ .../feature-flagging-lib/build.gradle.kts | 3 -- .../AgentlessConfigurationSource.java | 50 ++++++------------- .../AgentlessConfigurationSourceTest.java | 21 +++++++- 7 files changed, 81 insertions(+), 44 deletions(-) diff --git a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java index 0c008f41094..327a1555846 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java @@ -45,12 +45,12 @@ public final class ConfigDefaults { public static final String DEFAULT_SERVLET_ROOT_CONTEXT_SERVICE_NAME = "root-servlet"; public static final String DEFAULT_AGENT_WRITER_TYPE = "DDAgentWriter"; public static final boolean DEFAULT_STARTUP_LOGS_ENABLED = true; - public static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; - public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; - public static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; static final boolean DEFAULT_INJECT_DATADOG_ATTRIBUTE = true; static final String DEFAULT_SITE = "datadoghq.com"; + static final String DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE = "agentless"; + static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_POLL_INTERVAL_SECONDS = 30; + static final int DEFAULT_FEATURE_FLAGGING_CONFIGURATION_SOURCE_REQUEST_TIMEOUT_SECONDS = 2; static final boolean DEFAULT_CODE_ORIGIN_FOR_SPANS_INTERFACE_SUPPORT = false; static final int DEFAULT_CODE_ORIGIN_MAX_USER_FRAMES = 8; diff --git a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy index a9046475a7c..63b442f9a48 100644 --- a/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy +++ b/internal-api/src/test/groovy/datadog/trace/api/ConfigTest.groovy @@ -57,6 +57,7 @@ import static datadog.trace.api.config.GeneralConfig.TAGS import static datadog.trace.api.config.GeneralConfig.TRACER_METRICS_IGNORED_RESOURCES import static datadog.trace.api.config.GeneralConfig.TRACE_OTEL_SEMANTICS_ENABLED import static datadog.trace.api.config.GeneralConfig.VERSION +import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_POLL_INTERVAL_SECONDS import static datadog.trace.api.featureflag.config.FeatureFlaggingConfig.FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_REQUEST_TIMEOUT_SECONDS import static datadog.trace.api.config.JmxFetchConfig.JMX_FETCH_CHECK_PERIOD @@ -3497,6 +3498,27 @@ class ConfigTest extends DDSpecification { config.featureFlaggingConfigurationSourceRequestTimeoutSeconds == 4 } + def "feature flag configuration source normalizes #value to #expected"() { + setup: + Properties properties = new Properties() + if (value != null) { + properties.setProperty(FEATURE_FLAGS_CONFIGURATION_SOURCE, value) + } + + when: + def config = new Config(ConfigProvider.withPropertiesOverride(properties)) + + then: + config.featureFlaggingConfigurationSource == expected + + where: + value | expected + null | "agentless" + "" | "agentless" + " " | "agentless" + " ReMoTe_ConFiG " | "remote_config" + } + def "agentless feature flag timing falls back for non-positive values"() { setup: Properties properties = new Properties() diff --git a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java index 6d8fd4cb720..5c9cb20540e 100644 --- a/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java +++ b/products/feature-flagging/feature-flagging-agent/src/main/java/com/datadog/featureflag/FeatureFlaggingSystem.java @@ -47,9 +47,12 @@ static void initialize( CONFIG_SERVICE = configService; EXPOSURE_WRITER = exposureWriter; } catch (final RuntimeException | Error e) { - exposureWriter.close(); - if (configService != null) { - configService.close(); + try { + exposureWriter.close(); + } finally { + if (configService != null) { + configService.close(); + } } throw e; } diff --git a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java index 4b241fe3cb2..f6fc7516c0b 100644 --- a/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java +++ b/products/feature-flagging/feature-flagging-agent/src/test/java/com/datadog/featureflag/FeatureFlaggingSystemTest.java @@ -149,6 +149,20 @@ void initializationFailureWithoutConfigurationSourceClosesExposureWriter() { verify(exposureWriter).close(); } + @Test + void initializationFailureClosesConfigurationSourceWhenExposureWriterCloseFails() { + ConfigurationSourceService configService = mock(ConfigurationSourceService.class); + ExposureWriter exposureWriter = mock(ExposureWriter.class); + doThrow(new IllegalStateException("exposure init failed")).when(exposureWriter).init(); + doThrow(new IllegalArgumentException("exposure close failed")).when(exposureWriter).close(); + + assertThrows( + IllegalArgumentException.class, + () -> FeatureFlaggingSystem.initialize(configService, exposureWriter)); + + verify(configService).close(); + } + private static SharedCommunicationObjects sharedCommunicationObjects() { DDAgentFeaturesDiscovery discovery = mock(DDAgentFeaturesDiscovery.class); when(discovery.supportsEvpProxy()).thenReturn(true); diff --git a/products/feature-flagging/feature-flagging-lib/build.gradle.kts b/products/feature-flagging/feature-flagging-lib/build.gradle.kts index 82faa899f3e..425e217e822 100644 --- a/products/feature-flagging/feature-flagging-lib/build.gradle.kts +++ b/products/feature-flagging/feature-flagging-lib/build.gradle.kts @@ -24,14 +24,11 @@ dependencies { api(project(":utils:queue-utils")) compileOnly(project(":dd-trace-core")) // shading does not work with this one - // Span-enrichment write tier: TraceInterceptor / GlobalTracer / AgentTracer / AgentSpan. - compileOnly(project(":internal-api")) // Platform JSON writer for the ffe_* tag values. compileOnly(project(":components:json")) testImplementation(libs.bundles.junit5) testImplementation(libs.bundles.mockito) - testImplementation(project(":internal-api")) testImplementation(project(":utils:test-utils")) testImplementation(project(":dd-java-agent:testing")) } diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index e9f0786d76a..ba0d712d88c 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -2,6 +2,7 @@ import static datadog.communication.http.OkHttpUtils.prepareRequest; import static datadog.trace.util.AgentThreadFactory.AgentThread.FEATURE_FLAG_CONFIGURATION_POLLER; +import static datadog.trace.util.Strings.isBlank; import datadog.communication.http.OkHttpUtils; import datadog.logging.RatelimitedLogger; @@ -11,9 +12,7 @@ import datadog.trace.util.AgentThreadFactory; import java.io.IOException; import java.net.HttpURLConnection; -import java.net.URLEncoder; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -277,26 +276,17 @@ private static boolean isRetryableStatus(final int status) { } private void updateEtag(final String nextEtag) { - if (nextEtag != null && !nextEtag.trim().isEmpty()) { - etag = nextEtag; - } + etag = isBlank(nextEtag) ? null : nextEtag; } static HttpUrl endpoint(final Config config) { final String configuredBaseUrl = config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl(); - final String endpoint = - configuredBaseUrl == null - ? datadogApiServerDistributionEndpoint(config) - : endpointFromConfiguredBaseUrl(configuredBaseUrl); - final HttpUrl parsed = HttpUrl.parse(endpoint); - if (parsed == null) { - throw new IllegalArgumentException( - "Invalid Feature Flagging HTTP configuration source URL: " + endpoint); - } - return parsed; + return configuredBaseUrl == null + ? datadogApiServerDistributionEndpoint(config) + : endpointFromConfiguredBaseUrl(configuredBaseUrl); } - private static String endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { + private static HttpUrl endpointFromConfiguredBaseUrl(final String configuredBaseUrl) { final HttpUrl parsed = HttpUrl.parse(configuredBaseUrl.trim()); if (parsed == null) { throw new IllegalArgumentException( @@ -306,30 +296,22 @@ private static String endpointFromConfiguredBaseUrl(final String configuredBaseU return parsed .newBuilder() .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) - .build() - .toString(); + .build(); } - return parsed.toString(); + return parsed; } - private static String datadogApiServerDistributionEndpoint(final Config config) { - final StringBuilder endpoint = - new StringBuilder("https://api.") - .append(config.getSite().toLowerCase(Locale.ROOT)) - .append(DATADOG_API_SERVER_DISTRIBUTION_PATH); + private static HttpUrl datadogApiServerDistributionEndpoint(final Config config) { + final HttpUrl.Builder endpoint = + new HttpUrl.Builder() + .scheme("https") + .host("api." + config.getSite()) + .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)); final String env = config.getEnv(); if (env != null && !env.isEmpty()) { - endpoint.append("?dd_env=").append(urlEncode(env)); - } - return endpoint.toString(); - } - - private static String urlEncode(final String value) { - try { - return URLEncoder.encode(value, "UTF-8"); - } catch (final IOException e) { - throw new IllegalArgumentException("Unable to encode Feature Flagging environment", e); + endpoint.addQueryParameter("dd_env", env); } + return endpoint.build(); } static long millis(final int seconds) { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index a5b6dc84bfb..07557135c25 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -66,7 +66,7 @@ void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { final Config config = config("datad0g.com", "staging env"); assertEquals( - "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging+env", + "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging%20env", AgentlessConfigurationSource.endpoint(config).toString()); } @@ -385,6 +385,25 @@ void ignoresBlankEtag() throws Exception { verify(listener).accept(any(ServerConfiguration.class)); } + @Test + void successfulResponseWithoutEtagClearsPreviousEtag() throws Exception { + final FakeClient client = + new FakeClient( + response(200, "etag-a", emptyConfig()), + response(200, null, emptyConfig()), + response(304, null, null)); + final AgentlessConfigurationSource service = service(client); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + assertTrue(service.pollOnce()); + + assertEquals("etag-a", client.requests.get(1).etag); + assertNull(client.requests.get(2).etag); + verify(listener, times(2)).accept(any(ServerConfiguration.class)); + } + @Test void usesEtagAndSkipsDispatchOnUnchangedConfig() throws Exception { final FakeClient client = From 76917d61f00ab1ae9b5fb2bcd32f2dee2cb587c7 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 15 Jul 2026 16:30:56 -0400 Subject: [PATCH 10/10] Support the UFC CDN response contract --- .../feature-flagging-api/README.md | 11 ++- .../AgentlessConfigurationSource.java | 16 ++-- .../featureflag/RemoteConfigServiceImpl.java | 30 ++++++ .../AgentlessConfigurationSourceTest.java | 93 +++++++++++++++---- 4 files changed, 120 insertions(+), 30 deletions(-) diff --git a/products/feature-flagging/feature-flagging-api/README.md b/products/feature-flagging/feature-flagging-api/README.md index ae861d07afd..74bbccca2e2 100644 --- a/products/feature-flagging/feature-flagging-api/README.md +++ b/products/feature-flagging/feature-flagging-api/README.md @@ -88,13 +88,16 @@ OTEL_EXPORTER_OTLP_PROTOCOL=grpc - `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE=agentless` uses the Datadog agentless backend. Set `DD_FEATURE_FLAGS_CONFIGURATION_SOURCE_AGENTLESS_BASE_URL` to a different HTTP backend while keeping agentless delivery semantics. A bare - host uses the standard server-distribution path; a URL with a path is used as + host uses the standard rules-based server path; a URL with a path is used as the exact UFC endpoint. Configured URLs are opaque: the SDK does not add the Datadog-managed `dd_env` query parameter, so custom backends must include any required tenant or environment scope in the configured URL. The derived - Datadog-managed endpoint is intended for supported commercial sites; use an - explicit base URL elsewhere. Agentless responses do not have an SDK-imposed - payload-size limit. `remote_config` uses the existing Agent Remote + Datadog-managed endpoint is + `https://ufc-server.ff-cdn./api/v2/feature-flagging/config/rules-based/server` + and expects UFC under the JSON:API `data.attributes` response member. It is + intended for supported commercial sites; use an explicit base URL elsewhere. + Agentless responses do not have an SDK-imposed payload-size limit. + `remote_config` uses the existing Agent Remote Configuration path. `offline` is reserved for startup-provided UFC bytes; until those bytes are implemented, no network source starts and evaluations use defaults. diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java index ba0d712d88c..506fcddc9fa 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/AgentlessConfigurationSource.java @@ -35,9 +35,8 @@ final class AgentlessConfigurationSource implements ConfigurationSourceService { private static final Logger LOGGER = LoggerFactory.getLogger(AgentlessConfigurationSource.class); - // TODO before merge: confirm the final backend route with the server-distribution API owners. - private static final String DATADOG_API_SERVER_DISTRIBUTION_PATH = - "/api/v2/feature-flagging/config/server-distribution"; + private static final String DATADOG_UFC_RULES_BASED_SERVER_PATH = + "/api/v2/feature-flagging/config/rules-based/server"; private static final int MAX_ATTEMPTS = 3; private static final int MINUTES_BETWEEN_WARNINGS = 5; private static final long FIRST_RETRY_MIN_MILLIS = 2_000; @@ -255,8 +254,9 @@ private boolean apply(final UfcHttpResponse response) { final ServerConfiguration configuration; try { configuration = - RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserialize( - response.body); + RemoteConfigServiceImpl.UniversalFlagConfigDeserializer.INSTANCE.deserializeApiResponse( + response.body, + config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl() != null); } catch (final IOException | RuntimeException e) { LOGGER.debug("Feature Flagging HTTP configuration source returned malformed UFC payload", e); return false; @@ -295,7 +295,7 @@ private static HttpUrl endpointFromConfiguredBaseUrl(final String configuredBase if ("/".equals(parsed.encodedPath()) || parsed.encodedPath().isEmpty()) { return parsed .newBuilder() - .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)) + .addPathSegments(DATADOG_UFC_RULES_BASED_SERVER_PATH.substring(1)) .build(); } return parsed; @@ -305,8 +305,8 @@ private static HttpUrl datadogApiServerDistributionEndpoint(final Config config) final HttpUrl.Builder endpoint = new HttpUrl.Builder() .scheme("https") - .host("api." + config.getSite()) - .addPathSegments(DATADOG_API_SERVER_DISTRIBUTION_PATH.substring(1)); + .host("ufc-server.ff-cdn." + config.getSite()) + .addPathSegments(DATADOG_UFC_RULES_BASED_SERVER_PATH.substring(1)); final String env = config.getEnv(); if (env != null && !env.isEmpty()) { endpoint.addQueryParameter("dd_env", env); diff --git a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java index a5a9b2d7f36..12672aa74c1 100644 --- a/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java +++ b/products/feature-flagging/feature-flagging-lib/src/main/java/com/datadog/featureflag/RemoteConfigServiceImpl.java @@ -66,17 +66,47 @@ public void accept( static class UniversalFlagConfigDeserializer implements ConfigurationDeserializer { + private static final String UNIVERSAL_FLAG_CONFIGURATION_TYPE = "universal-flag-configuration"; static final UniversalFlagConfigDeserializer INSTANCE = new UniversalFlagConfigDeserializer(); private static final Moshi MOSHI = new Moshi.Builder().add(Date.class, new DateAdapter()).add(FlagMapAdapter.FACTORY).build(); private static final JsonAdapter V1_ADAPTER = MOSHI.adapter(ServerConfiguration.class); + private static final Type API_RESPONSE_TYPE = + Types.newParameterizedType(Map.class, String.class, Object.class); + private static final JsonAdapter> API_RESPONSE_ADAPTER = + MOSHI.adapter(API_RESPONSE_TYPE); @Override public ServerConfiguration deserialize(final byte[] content) throws IOException { return V1_ADAPTER.fromJson(Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); } + + @Nullable + ServerConfiguration deserializeApiResponse( + final byte[] content, final boolean allowRawConfiguration) throws IOException { + final Map response = + API_RESPONSE_ADAPTER.fromJson( + Okio.buffer(Okio.source(new ByteArrayInputStream(content)))); + if (response != null && response.containsKey("data")) { + final Object data = response.get("data"); + if (!(data instanceof Map)) { + return null; + } + final Map dataAttributes = (Map) data; + return UNIVERSAL_FLAG_CONFIGURATION_TYPE.equals(dataAttributes.get("type")) + ? validConfiguration(V1_ADAPTER.fromJsonValue(dataAttributes.get("attributes"))) + : null; + } + return allowRawConfiguration ? validConfiguration(deserialize(content)) : null; + } + + @Nullable + private static ServerConfiguration validConfiguration( + @Nullable final ServerConfiguration configuration) { + return configuration != null && configuration.flags != null ? configuration : null; + } } static class FlagMapAdapter extends JsonAdapter> { diff --git a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java index 07557135c25..c450c2e2191 100644 --- a/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java +++ b/products/feature-flagging/feature-flagging-lib/src/test/java/com/datadog/featureflag/AgentlessConfigurationSourceTest.java @@ -51,7 +51,7 @@ @ExtendWith(MockitoExtension.class) class AgentlessConfigurationSourceTest { - private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/server-distribution"; + private static final String CONFIG_PATH = "/api/v2/feature-flagging/config/rules-based/server"; @Mock private FeatureFlaggingGateway.ConfigListener listener; @@ -62,33 +62,33 @@ void cleanup() { } @Test - void derivesDatadogApiServerDistributionEndpointFromSiteAndEnv() { + void derivesDatadogUfcCdnEndpointFromSiteAndEnv() { final Config config = config("datad0g.com", "staging env"); assertEquals( - "https://api.datad0g.com/api/v2/feature-flagging/config/server-distribution?dd_env=staging%20env", + "https://ufc-server.ff-cdn.datad0g.com/api/v2/feature-flagging/config/rules-based/server?dd_env=staging%20env", AgentlessConfigurationSource.endpoint(config).toString()); } @Test - void derivesDatadogApiServerDistributionEndpointWithoutEnv() { + void derivesDatadogUfcCdnEndpointWithoutEnv() { assertEquals( - "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config("datadoghq.com", "")).toString()); assertEquals( - "https://api.datadoghq.com/api/v2/feature-flagging/config/server-distribution", + "https://ufc-server.ff-cdn.datadoghq.com/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config("datadoghq.com", null)).toString()); } @Test - void appendsServerDistributionPathToConfiguredAgentlessBaseUrl() { + void appendsRulesBasedServerPathToConfiguredAgentlessBaseUrl() { final Config config = config(); lenient() .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) .thenReturn("http://mock-backend:8080"); assertEquals( - "http://mock-backend:8080/api/v2/feature-flagging/config/server-distribution", + "http://mock-backend:8080/api/v2/feature-flagging/config/rules-based/server", AgentlessConfigurationSource.endpoint(config).toString()); } @@ -116,7 +116,7 @@ void rejectsInvalidConfiguredAgentlessBaseUrl() { } @Test - void rejectsInvalidDatadogApiServerDistributionEndpoint() { + void rejectsInvalidDatadogUfcCdnEndpoint() { assertThrows( IllegalArgumentException.class, () -> AgentlessConfigurationSource.endpoint(config("datadoghq.com:bad", ""))); @@ -359,18 +359,43 @@ void realHttpClientTimesOutDelayedResponse() throws Exception { } @Test - void appliesAcceptedUfcThroughGatewayAndSendsApiKey() throws Exception { + void appliesAcceptedJsonApiUfcThroughGatewayAndSendsApiKey() throws Exception { final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfig())); final AgentlessConfigurationSource service = service(client); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); FeatureFlaggingGateway.addConfigListener(listener); assertTrue(service.pollOnce()); - verify(listener).accept(any(ServerConfiguration.class)); + verify(listener).accept(configuration.capture()); + assertEquals("2026-07-15T19:57:07.219869778Z", configuration.getValue().createdAt); + assertNull(configuration.getValue().format); + assertEquals("Staging", configuration.getValue().environment.name); + assertTrue(configuration.getValue().flags.isEmpty()); assertEquals("test-api-key", client.requests.get(0).apiKey); assertNull(client.requests.get(0).etag); } + @Test + void appliesRawUfcFromConfiguredCompatibilityEndpoint() throws Exception { + final FakeClient client = new FakeClient(response(200, "etag-a", emptyConfigAttributes())); + final Config config = config(); + lenient() + .when(config.getFeatureFlaggingConfigurationSourceAgentlessBaseUrl()) + .thenReturn("http://compatibility-backend/custom/ufc"); + final AgentlessConfigurationSource service = service(client, config); + final ArgumentCaptor configuration = + ArgumentCaptor.forClass(ServerConfiguration.class); + FeatureFlaggingGateway.addConfigListener(listener); + + assertTrue(service.pollOnce()); + + verify(listener).accept(configuration.capture()); + assertEquals("Staging", configuration.getValue().environment.name); + assertTrue(configuration.getValue().flags.isEmpty()); + } + @Test void ignoresBlankEtag() throws Exception { final FakeClient client = @@ -486,10 +511,19 @@ void rejectsForbiddenNonOkMissingBodyAndNullConfiguration() throws Exception { response(404, null, null), response(600, null, null), response(200, null, null), - response(200, null, "null")); + response(200, null, "null"), + response(200, null, jsonApiResponse("other-configuration", emptyConfigAttributes())), + response(200, null, "{\"data\":null}"), + response( + 200, null, "{\"data\":{\"id\":\"1\",\"type\":\"universal-flag-configuration\"}}"), + response(200, null, emptyConfigAttributes())); final AgentlessConfigurationSource service = service(client); FeatureFlaggingGateway.addConfigListener(listener); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); + assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); assertFalse(service.pollOnce()); @@ -902,6 +936,16 @@ private static AgentlessConfigurationSource service(final FakeClient client) { return service(client, delay -> {}, () -> 1.0); } + private static AgentlessConfigurationSource service( + final FakeClient client, final Config config) { + return new AgentlessConfigurationSource( + HttpUrl.get("http://localhost" + CONFIG_PATH), + config, + 30_000, + client, + Executors.newSingleThreadScheduledExecutor()); + } + private static AgentlessConfigurationSource service( final FakeClient client, final AgentlessConfigurationSource.RetrySleeper retrySleeper, @@ -943,19 +987,32 @@ private static AgentlessConfigurationSource.UfcHttpResponse response( } private static String emptyConfig() { + return jsonApiResponse("universal-flag-configuration", emptyConfigAttributes()); + } + + private static String emptyConfigAttributes() { return "{" - + "\"createdAt\":\"2024-04-17T19:40:53.716Z\"," - + "\"format\":\"SERVER\"," - + "\"environment\":{\"name\":\"Test\"}," + + "\"createdAt\":\"2026-07-15T19:57:07.219869778Z\"," + + "\"environment\":{\"name\":\"Staging\"}," + "\"flags\":{}" + "}"; } + private static String jsonApiResponse(final String type, final String attributes) { + return "{\"data\":{" + + "\"id\":\"1\"," + + "\"type\":\"" + + type + + "\"," + + "\"attributes\":" + + attributes + + "}}"; + } + private static String largeConfig(final int flagCount) { final StringBuilder json = new StringBuilder( - "{\"createdAt\":\"2024-04-17T19:40:53.716Z\"," - + "\"format\":\"SERVER\"," + "{\"createdAt\":\"2026-07-15T19:57:07.219869778Z\"," + "\"environment\":{\"name\":\"Large Test\"}," + "\"flags\":{"); for (int index = 0; index < flagCount; index++) { @@ -972,7 +1029,7 @@ private static String largeConfig(final int flagCount) { + "\"variations\":{\"on\":{\"key\":\"on\",\"value\":\"on\"}}," + "\"allocations\":[]}"); } - return json.append("}}").toString(); + return jsonApiResponse("universal-flag-configuration", json.append("}}").toString()); } private static void awaitCalls(final FakeClient client, final int count) throws Exception {