From 41525ffeb889790faf2a337a0b5004c69abe1c7d Mon Sep 17 00:00:00 2001 From: Gauri Kalra Date: Fri, 30 Oct 2026 07:50:20 +0000 Subject: [PATCH] fix(gax): use TLS channel credentials for DirectPath over Interconnect --- .../storage/StorageOptionsBuilderTest.java | 31 +- .../gax-java/gax-grpc/BUILD.bazel | 1 + .../InstantiatingGrpcChannelProvider.java | 96 +++- .../InstantiatingGrpcChannelProviderTest.java | 467 +++++++++++++++++- 4 files changed, 561 insertions(+), 34 deletions(-) diff --git a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageOptionsBuilderTest.java b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageOptionsBuilderTest.java index 87136e5c69af..b8dbb440504c 100644 --- a/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageOptionsBuilderTest.java +++ b/java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageOptionsBuilderTest.java @@ -71,10 +71,12 @@ public void grpc() throws Exception { @Test public void grpc_attemptDirectPathXdsOverInterconnect() throws Exception { - com.google.auth.Credentials mockCreds = com.google.cloud.NoCredentials.getInstance(); + com.google.auth.Credentials credentials = + com.google.auth.oauth2.GoogleCredentials.create( + new com.google.auth.oauth2.AccessToken("token", null)); GrpcStorageOptions options = GrpcStorageOptions.grpc() - .setCredentials(mockCreds) + .setCredentials(credentials) .setAttemptDirectPathXdsOverInterconnect(true) .build(); @@ -88,8 +90,11 @@ public void grpc_attemptDirectPathXdsOverInterconnect() throws Exception { com.google.api.gax.rpc.TransportChannelProvider tcp = settings.getTransportChannelProvider(); assertThat(tcp).isInstanceOf(com.google.api.gax.grpc.InstantiatingGrpcChannelProvider.class); + // ClientContext attaches the settings' credentials to the channel provider when the client is + // created; DirectPath over Interconnect requires them. com.google.api.gax.grpc.InstantiatingGrpcChannelProvider provider = - (com.google.api.gax.grpc.InstantiatingGrpcChannelProvider) tcp; + (com.google.api.gax.grpc.InstantiatingGrpcChannelProvider) + tcp.withCredentials(settings.getCredentialsProvider().getCredentials()); assertThat(provider.getEndpoint()).isEqualTo("storage.googleapis.com:443"); assertThat(provider.isDirectPathXdsEnabled()).isTrue(); @@ -100,6 +105,26 @@ public void grpc_attemptDirectPathXdsOverInterconnect() throws Exception { } } + @Test + public void grpc_attemptDirectPathXdsOverInterconnect_noCredentials_fallsBackToCloudPath() + throws Exception { + GrpcStorageOptions options = + GrpcStorageOptions.grpc() + .setCredentials(com.google.cloud.NoCredentials.getInstance()) + .setAttemptDirectPathXdsOverInterconnect(true) + .build(); + com.google.storage.v2.StorageSettings settings = options.getStorageSettings(); + + // NoCredentials resolves to no credentials, so ClientContext leaves the channel provider + // without any, and the channel uses CloudPath. + assertThat(settings.getCredentialsProvider().getCredentials()).isNull(); + com.google.api.gax.grpc.InstantiatingGrpcChannelProvider provider = + (com.google.api.gax.grpc.InstantiatingGrpcChannelProvider) + settings.getTransportChannelProvider(); + assertThat(provider.isDirectPathXdsEnabled()).isTrue(); + assertThat(provider.canUseDirectPath()).isFalse(); + } + @Test public void grpc_fallbackFromDirectPathXdsOverInterconnect_nonGduUniverse() throws Exception { GrpcStorageOptions options = diff --git a/sdk-platform-java/gax-java/gax-grpc/BUILD.bazel b/sdk-platform-java/gax-java/gax-grpc/BUILD.bazel index 52bec965145b..f615eb9e3294 100644 --- a/sdk-platform-java/gax-java/gax-grpc/BUILD.bazel +++ b/sdk-platform-java/gax-java/gax-grpc/BUILD.bazel @@ -37,6 +37,7 @@ _COMPILE_DEPS = [ _TEST_COMPILE_DEPS = [ "@org_junit_jupiter_junit_jupiter_api//jar", + "@org_junit_jupiter_junit_jupiter_params//jar", "@org_mockito_mockito_core//jar", "@org_mockito_mockito_junit_jupiter//jar", "@com_google_truth_truth//jar", diff --git a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index a6db148b992d..aa8e2f52ad0f 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -408,9 +408,10 @@ public TransportChannel getTransportChannel() throws IOException { private TransportChannel createChannel() throws IOException { // Only track the workload certificate for rotation when the pool's channels actually present - // it, mirroring the credential selection in createSingleChannel(): DirectPath channels use - // GoogleDefaultChannelCredentials (ALTS) rather than the client certificate, and without an - // mtlsProvider there is no client certificate KeyStore (S2A or plain TLS is used instead). + // it, mirroring the credential selection in createChannelBuilder(): DirectPath channels use + // createDirectPathChannelCredentials() (ALTS, or TLS over Interconnect) rather than the client + // certificate, and without an mtlsProvider there is no client certificate KeyStore (S2A or + // plain TLS is used instead). String workloadCertPath = !this.canUseDirectPath() && mtlsProvider != null ? certificateBasedAccess.getWorkloadCertPath() @@ -483,13 +484,32 @@ public boolean isDirectPathXdsEnabled() { // builder or createSingleChannel, only in getTransportChannel which creates the first channel // for a client. private void logDirectPathMisconfig() { - Level level = isOnComputeEngine() ? Level.WARNING : Level.FINE; + // Off GCE, DirectPath is normally unavailable, so misconfigurations are only logged at FINE. + // DirectPath over Interconnect is meant to run off GCE, so its misconfigurations are warnings. + Level level = + isOnComputeEngine() || isAttemptDirectPathXdsOverInterconnect() + ? Level.WARNING + : Level.FINE; if (!isDirectPathEnabled()) { + // Checked before the xDS cases: clients such as Storage set the xDS option together with + // attemptDirectPath, so with the kill switch set the xDS message would wrongly report + // attemptDirectPath as unset. + if (isAttemptDirectPathXdsOverInterconnect()) { + LOG.log( + level, + "DirectPath over Interconnect is misconfigured. The attemptDirectPathXdsOverInterconnect" + + " option or the " + + DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT + + " environment variable was set, but DirectPath is not enabled for this client." + + " Please set the attemptDirectPath option as well, and make sure " + + DIRECT_PATH_ENV_DISABLE_DIRECT_PATH + + " is not set to true."); + } // Direct Path xDS can be enabled two ways: via environment variable or via builder. // Case 1: Direct Path is only enabled via xDS env var. We will _warn_ the user that this is // a misconfiguration if they intended to set the env var. - if (isDirectPathXdsEnabledViaEnv()) { + else if (isDirectPathXdsEnabledViaEnv()) { LOG.log( level, "Env var " @@ -517,7 +537,12 @@ else if (isDirectPathXdsEnabledViaBuilderOption()) { + " soon require xDS to be enabled. Please set the attemptDirectPathXds option."); } // Case 4: credential is not correctly set - if (!isCredentialDirectPathCompatible()) { + if (isAttemptDirectPathXdsOverInterconnect() && needsCredentials()) { + LOG.log( + level, + "DirectPath over Interconnect is misconfigured. Credentials are required; without them" + + " the channel falls back to CloudPath."); + } else if (!isCredentialDirectPathCompatible()) { LOG.log( level, "DirectPath is misconfigured. Please make sure the credential is an instance of " @@ -535,14 +560,14 @@ else if (isDirectPathXdsEnabledViaBuilderOption()) { @VisibleForTesting boolean isCredentialDirectPathCompatible() { - // xDS over Interconnect is designed to work on-premise using arbitrary service credentials. - if (isAttemptDirectPathXdsOverInterconnect()) { - return true; - } // DirectPath requires a call credential during gRPC channel construction. if (needsCredentials()) { return false; } + // xDS over Interconnect is designed to work on-premise using arbitrary service credentials. + if (isAttemptDirectPathXdsOverInterconnect()) { + return true; + } if (allowNonDefaultServiceAccount != null && allowNonDefaultServiceAccount) { return true; } @@ -752,6 +777,36 @@ ChannelCredentials createS2ASecuredChannelCredentials() { return s2aChannelCredentials; } + /** + * Returns the channel credentials for a DirectPath channel. + * + *

Over Interconnect every subchannel negotiates TLS. {@link GoogleDefaultChannelCredentials} + * would select ALTS for every xDS cluster whose name is not prefixed {@code google_cfe_}, which + * includes the regional Interconnect clusters, and ALTS is unavailable outside Google Cloud. The + * call credentials are attached to the channel credentials because those are the only credentials + * the RLS control channel inherits from the parent channel. ALTS hard-bound tokens ({@code + * altsCallCredentials}) are not requested because they require an ALTS connection. + */ + @VisibleForTesting + ChannelCredentials createDirectPathChannelCredentials() { + if (isAttemptDirectPathXdsOverInterconnect()) { + return CompositeChannelCredentials.create( + TlsChannelCredentials.create(), + MoreCallCredentials.from( + Preconditions.checkNotNull( + credentials, "DirectPath over Interconnect requires credentials"))); + } + GoogleDefaultChannelCredentials.Builder credsBuilder = + GoogleDefaultChannelCredentials.newBuilder(); + if (credentials != null) { + credsBuilder.callCredentials(MoreCallCredentials.from(credentials)); + } + if (altsCallCredentials != null) { + credsBuilder.altsCallCredentials(altsCallCredentials); + } + return credsBuilder.build(); + } + @InternalApi("For internal use by google-cloud-java clients only") public ManagedChannelBuilder createChannelBuilder() throws IOException { int colon = endpoint.lastIndexOf(':'); @@ -766,15 +821,7 @@ public ManagedChannelBuilder createChannelBuilder() throws IOException { // Check DirectPath traffic. boolean useDirectPathXds = false; if (canUseDirectPath()) { - GoogleDefaultChannelCredentials.Builder credsBuilder = - GoogleDefaultChannelCredentials.newBuilder(); - if (credentials != null) { - credsBuilder.callCredentials(MoreCallCredentials.from(credentials)); - } - if (altsCallCredentials != null) { - credsBuilder.altsCallCredentials(altsCallCredentials); - } - ChannelCredentials channelCreds = credsBuilder.build(); + ChannelCredentials channelCreds = createDirectPathChannelCredentials(); useDirectPathXds = isDirectPathXdsEnabled(); if (useDirectPathXds) { // google-c2p: CloudToProd(C2P) Directpath. This scheme is defined in @@ -931,7 +978,9 @@ private void removeApiKeyCredentialDuplicateHeaders() { @InternalApi public boolean canUseDirectPath() { if (isAttemptDirectPathXdsOverInterconnect()) { - return isDirectPathEnabled() && canUseDirectPathWithUniverseDomain(); + return isDirectPathEnabled() + && isCredentialDirectPathCompatible() + && canUseDirectPathWithUniverseDomain(); } return isDirectPathEnabled() && isCredentialDirectPathCompatible() @@ -1373,6 +1422,13 @@ public Builder setAttemptDirectPathXds() { /** * Use DirectPath xDS over Interconnect. Bypasses GCP GCE environment checks. * + *

DirectPath must also be enabled with {@link #setAttemptDirectPath(boolean)}; otherwise the + * channel uses CloudPath and a warning is logged. Credentials are required, as for standard + * DirectPath; without them the channel falls back to CloudPath. The channel negotiates TLS + * rather than ALTS, which is unavailable outside Google Cloud. Application Default Credentials + * must also be available, because the google-c2p name resolver uses them for its connection to + * the control plane. + * *

This can also be enabled or disabled via the {@code * GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT} environment variable ({@code "true"} * or {@code "false"}), which takes precedence over this builder option. diff --git a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java index 1612744a43ee..2089a77c0f19 100644 --- a/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java +++ b/sdk-platform-java/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java @@ -63,12 +63,17 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.truth.Truth; +import io.grpc.CallCredentials; +import io.grpc.ChannelCredentials; +import io.grpc.CompositeChannelCredentials; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.TlsChannelCredentials; import io.grpc.alts.ComputeEngineChannelBuilder; +import io.grpc.auth.MoreCallCredentials; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.security.GeneralSecurityException; import java.time.Duration; import java.util.ArrayList; @@ -90,6 +95,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -1053,8 +1060,9 @@ public void getTransportChannel_storageTarget_withInterconnect() } @Test - public void getTransportChannel_storageTarget_withInterconnectAndNullCredentials() - throws IOException, InterruptedException { + public void + getTransportChannel_storageTarget_withInterconnectAndNullCredentials_fallsBackToCloudPath() + throws IOException, InterruptedException { System.setProperty("os.name", "Not Linux"); EnvironmentProvider envProvider = mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); @@ -1068,6 +1076,9 @@ public void getTransportChannel_storageTarget_withInterconnectAndNullCredentials return channelBuilder; }; + // Interconnect accepts any service credentials but, like standard DirectPath, requires some: + // the channel credentials carry the call credentials, so without them the channel uses + // CloudPath. InstantiatingGrpcChannelProvider.Builder builder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) @@ -1087,12 +1098,146 @@ public void getTransportChannel_storageTarget_withInterconnectAndNullCredentials .withHeaders(Collections.emptyMap()) .withEndpoint("storage.googleapis.com:443"); + Truth.assertThat(configuredProvider.isCredentialDirectPathCompatible()).isFalse(); + Truth.assertThat(configuredProvider.canUseDirectPath()).isFalse(); + TransportChannel transportChannel = configuredProvider.getTransportChannel(); transportChannel.shutdownNow(); transportChannel.awaitTermination(5, TimeUnit.SECONDS); - Truth.assertThat(capturedTarget.get()) - .contains("google-c2p:///storage-direct.googleapis.com?force-xds"); + Truth.assertThat(capturedTarget.get()).isEqualTo("storage.googleapis.com:443"); + } + + private static final String INTERCONNECT_CREDENTIALS_REQUIRED_MESSAGE = + "DirectPath over Interconnect is misconfigured. Credentials are required; without them the" + + " channel falls back to CloudPath."; + private static final String INTERCONNECT_DIRECT_PATH_NOT_ENABLED_MESSAGE = + "DirectPath over Interconnect is misconfigured. The attemptDirectPathXdsOverInterconnect" + + " option or the " + + InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT + + " environment variable was set, but DirectPath is not enabled for this client. Please" + + " set the attemptDirectPath option as well, and make sure " + + InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH + + " is not set to true."; + + /** + * Builds the provider's transport channel off GCE and returns the misconfiguration log records, + * at every level, that building it produced. + */ + private FakeLogHandler logDirectPathMisconfigOffGce( + InstantiatingGrpcChannelProvider.Builder builder) throws Exception { + System.setProperty("os.name", "Not Linux"); + FakeLogHandler logHandler = new FakeLogHandler(); + InstantiatingGrpcChannelProvider.LOG.setLevel(Level.FINE); + InstantiatingGrpcChannelProvider.LOG.addHandler(logHandler); + try { + createAndCloseTransportChannel( + builder.setCertificateBasedAccess(certificateBasedAccess).build()); + } finally { + InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); + } + return logHandler; + } + + private static EnvironmentProvider envProviderWithDirectPathEnabled() { + EnvironmentProvider envProvider = + mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); + when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) + .thenReturn("false"); + return envProvider; + } + + @Test + void logDirectPathMisconfig_interconnectWithoutCredentials_warnsCredentialsRequired() + throws Exception { + FakeLogHandler logHandler = + logDirectPathMisconfigOffGce( + createChannelProviderBuilderForDirectPathLogTests() + .setAttemptDirectPath(true) + .setAttemptDirectPathXdsOverInterconnect(true) + .setEnvProvider(envProviderWithDirectPathEnabled())); + + // Interconnect runs off GCE, so its misconfigurations are warnings there, unlike standard + // DirectPath's. + assertThat(logHandler.levelsOf(INTERCONNECT_CREDENTIALS_REQUIRED_MESSAGE)) + .containsExactly(Level.WARNING); + // The standard DirectPath message would wrongly ask for ComputeEngineCredentials. + assertThat(logHandler.getAllMessages()) + .doesNotContain( + "DirectPath is misconfigured. Please make sure the credential is an instance of " + + ComputeEngineCredentials.class.getName() + + " ."); + } + + @Test + void logDirectPathMisconfig_interconnectWithoutAttemptDirectPath_warns() throws Exception { + FakeLogHandler logHandler = + logDirectPathMisconfigOffGce( + createChannelProviderBuilderForDirectPathLogTests() + .setAttemptDirectPathXdsOverInterconnect(true) + .setEnvProvider(envProviderWithDirectPathEnabled())); + + assertThat(logHandler.levelsOf(INTERCONNECT_DIRECT_PATH_NOT_ENABLED_MESSAGE)) + .containsExactly(Level.WARNING); + } + + @Test + void logDirectPathMisconfig_interconnectEnvVarWithoutAttemptDirectPath_warns() throws Exception { + EnvironmentProvider envProvider = envProviderWithDirectPathEnabled(); + when(envProvider.getenv( + InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT)) + .thenReturn("true"); + + FakeLogHandler logHandler = + logDirectPathMisconfigOffGce( + createChannelProviderBuilderForDirectPathLogTests().setEnvProvider(envProvider)); + + assertThat(logHandler.levelsOf(INTERCONNECT_DIRECT_PATH_NOT_ENABLED_MESSAGE)) + .containsExactly(Level.WARNING); + } + + @Test + void logDirectPathMisconfig_interconnectWithDirectPathKillSwitch_warns() throws Exception { + EnvironmentProvider envProvider = + mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); + when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) + .thenReturn("true"); + + // Storage, Bigtable and Spanner also set the xDS option alongside attemptDirectPath. + FakeLogHandler logHandler = + logDirectPathMisconfigOffGce( + createChannelProviderBuilderForDirectPathLogTests() + .setAttemptDirectPath(true) + .setAttemptDirectPathXds() + .setAttemptDirectPathXdsOverInterconnect(true) + .setEnvProvider(envProvider)); + + // The kill switch disables DirectPath as a whole; the message names it as a possible cause. + assertThat(logHandler.levelsOf(INTERCONNECT_DIRECT_PATH_NOT_ENABLED_MESSAGE)) + .containsExactly(Level.WARNING); + // The xDS message would wrongly report that attemptDirectPath was not set. + assertThat(logHandler.getAllMessages()) + .doesNotContain( + "DirectPath is misconfigured. The DirectPath XDS option was set, but the" + + " attemptDirectPath option was not. Please set both the attemptDirectPath and" + + " attemptDirectPathXds options."); + } + + @Test + void logDirectPathMisconfig_offGceWithoutInterconnect_logsAtFine() throws Exception { + // Standard DirectPath keeps its off-GCE level; only Interconnect misconfigurations warn. + FakeLogHandler logHandler = + logDirectPathMisconfigOffGce( + createChannelProviderBuilderForDirectPathLogTests() + .setAttemptDirectPathXds() + .setEnvProvider(envProviderWithDirectPathEnabled())); + + assertThat( + logHandler.levelsOf( + "DirectPath is misconfigured. The DirectPath XDS option was set, but the" + + " attemptDirectPath option was not. Please set both the attemptDirectPath and" + + " attemptDirectPathXds options.")) + .containsExactly(Level.FINE); } @Test @@ -1516,6 +1661,21 @@ void testSettingBackgroundExecutor() { } private static String extractTargetFromChannelBuilder(ManagedChannelBuilder channelBuilder) { + Object target = extractDelegateField(channelBuilder, "target"); + return target != null ? (String) target : channelBuilder.toString(); + } + + private static ChannelCredentials extractChannelCredentialsFromChannelBuilder( + ManagedChannelBuilder channelBuilder) { + return (ChannelCredentials) extractDelegateField(channelBuilder, "channelCredentials"); + } + + /** + * Reads {@code fieldName} from the ManagedChannelImplBuilder that the Netty builder delegates to, + * or returns null when either the delegate or the field cannot be found. + */ + private static @Nullable Object extractDelegateField( + ManagedChannelBuilder channelBuilder, String fieldName) { try { Class nettyBuilderClass = channelBuilder.getClass(); java.lang.reflect.Field delegateField = null; @@ -1534,23 +1694,23 @@ private static String extractTargetFromChannelBuilder(ManagedChannelBuilder c delegateField.setAccessible(true); Object delegate = delegateField.get(channelBuilder); Class delegateClass = delegate.getClass(); - java.lang.reflect.Field targetField = null; - while (delegateClass != null && targetField == null) { + Field field = null; + while (delegateClass != null && field == null) { try { - targetField = delegateClass.getDeclaredField("target"); + field = delegateClass.getDeclaredField(fieldName); } catch (NoSuchFieldException e) { delegateClass = delegateClass.getSuperclass(); } } - if (targetField != null) { - targetField.setAccessible(true); - return (String) targetField.get(delegate); + if (field != null) { + field.setAccessible(true); + return field.get(delegate); } } } catch (Exception e) { throw new RuntimeException(e); } - return channelBuilder.toString(); + return null; } @Test @@ -1624,6 +1784,7 @@ void canUseDirectPath_interconnectEnabledButDirectPathDisabled_fallsBackToCloudP mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); InstantiatingGrpcChannelProvider provider = InstantiatingGrpcChannelProvider.newBuilder() @@ -1633,6 +1794,7 @@ void canUseDirectPath_interconnectEnabledButDirectPathDisabled_fallsBackToCloudP .setExecutor(mock(Executor.class)) .setEndpoint(DEFAULT_ENDPOINT) .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1655,6 +1817,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); InstantiatingGrpcChannelProvider provider = InstantiatingGrpcChannelProvider.newBuilder() @@ -1664,6 +1827,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac .setExecutor(mock(Executor.class)) .setEndpoint("storage-direct.some-other-universe.com:443") .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1688,6 +1852,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); InstantiatingGrpcChannelProvider provider = InstantiatingGrpcChannelProvider.newBuilder() @@ -1697,6 +1862,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac .setExecutor(mock(Executor.class)) .setEndpoint("storage.googleapis.com:443") .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1720,6 +1886,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); InstantiatingGrpcChannelProvider provider = InstantiatingGrpcChannelProvider.newBuilder() @@ -1729,6 +1896,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac .setExecutor(mock(Executor.class)) .setEndpoint("storage-preprod.sandbox.googleapis.com:443") .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1753,6 +1921,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); // Test with bigtable.googleapis.com:443 (rewritten to bigtable-direct) InstantiatingGrpcChannelProvider provider = @@ -1763,6 +1932,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac .setExecutor(mock(Executor.class)) .setEndpoint("bigtable.googleapis.com:443") .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1786,6 +1956,7 @@ void canUseDirectPath_interconnectAndDirectPathEnabledButNonGduUniverse_fallsBac .setExecutor(mock(Executor.class)) .setEndpoint("bigtable-direct.googleapis.com:443") .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setEnvProvider(envProvider) .build(); @@ -1845,6 +2016,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_bypassesGceCheckAndRewritesTa mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); when(envProvider.getenv( InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT)) .thenReturn("true"); @@ -1853,6 +2025,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_bypassesGceCheckAndRewritesTa InstantiatingGrpcChannelProvider.Builder unsetBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setHeaderProvider(mock(HeaderProvider.class, withSettings().withoutAnnotations())) .setExecutor(mock(Executor.class)) @@ -1880,6 +2053,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_bypassesGceCheckAndRewritesTa InstantiatingGrpcChannelProvider.Builder falseBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setAttemptDirectPathXdsOverInterconnect(false) .setHeaderProvider(mock(HeaderProvider.class, withSettings().withoutAnnotations())) @@ -1911,6 +2085,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_bypassesGceCheckAndRewritesTa mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); when(envProvider.getenv( InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT)) .thenReturn("false"); @@ -1918,6 +2093,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_bypassesGceCheckAndRewritesTa InstantiatingGrpcChannelProvider.Builder builder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setAttemptDirectPathXdsOverInterconnect(true) .setHeaderProvider(mock(HeaderProvider.class, withSettings().withoutAnnotations())) @@ -1952,9 +2128,11 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_respectsDirectPathKillSwitch( // Case 1: setAttemptDirectPath(false) when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); InstantiatingGrpcChannelProvider.Builder disabledOptionBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(false) .setEndpoint("storage.googleapis.com:443") .setEnvProvider(envProvider); @@ -1968,6 +2146,7 @@ void canUseDirectPath_interconnectEnabledViaEnvVar_respectsDirectPathKillSwitch( InstantiatingGrpcChannelProvider.Builder disabledEnvBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setEndpoint("storage.googleapis.com:443") .setEnvProvider(envProvider); @@ -1983,6 +2162,7 @@ void canUseDirectPath_interconnectInvalidOrEmptyEnvVar_fallsBackToBuilderOption( mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) .thenReturn("false"); + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); for (String envVal : new String[] {"", " ", "invalid", "1", "0"}) { when(envProvider.getenv( @@ -1992,6 +2172,7 @@ void canUseDirectPath_interconnectInvalidOrEmptyEnvVar_fallsBackToBuilderOption( InstantiatingGrpcChannelProvider.Builder enabledBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setAttemptDirectPathXdsOverInterconnect(true) .setEndpoint("storage.googleapis.com:443") @@ -2003,6 +2184,7 @@ void canUseDirectPath_interconnectInvalidOrEmptyEnvVar_fallsBackToBuilderOption( InstantiatingGrpcChannelProvider.Builder disabledBuilder = InstantiatingGrpcChannelProvider.newBuilder() .setCertificateBasedAccess(certificateBasedAccess) + .setCredentials(credentials) .setAttemptDirectPath(true) .setAttemptDirectPathXdsOverInterconnect(false) .setEndpoint("storage.googleapis.com:443") @@ -2013,6 +2195,262 @@ void canUseDirectPath_interconnectInvalidOrEmptyEnvVar_fallsBackToBuilderOption( } } + /** + * Returns a builder for a DirectPath-enabled provider as configured off GCE, with the + * Interconnect environment variable set to {@code interconnectEnvValue} (unset when null). + */ + private InstantiatingGrpcChannelProvider.Builder directPathProviderBuilder( + @Nullable Credentials credentials, @Nullable String interconnectEnvValue) { + System.setProperty("os.name", "Not Linux"); + EnvironmentProvider envProvider = + mock(EnvironmentProvider.class, withSettings().withoutAnnotations()); + when(envProvider.getenv(InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_DISABLE_DIRECT_PATH)) + .thenReturn("false"); + when(envProvider.getenv( + InstantiatingGrpcChannelProvider.DIRECT_PATH_ENV_ENABLE_XDS_OVER_INTERCONNECT)) + .thenReturn(interconnectEnvValue); + return InstantiatingGrpcChannelProvider.newBuilder() + .setCertificateBasedAccess(certificateBasedAccess) + .setAttemptDirectPath(true) + .setCredentials(credentials) + .setEndpoint("storage.googleapis.com:443") + .setEnvProvider(envProvider) + .setHeaderProvider(mock(HeaderProvider.class, withSettings().withoutAnnotations())); + } + + /** Returns an off-GCE provider with DirectPath over Interconnect enabled by builder option. */ + private InstantiatingGrpcChannelProvider interconnectProvider( + @Nullable Credentials credentials, @Nullable String interconnectEnvValue) { + return new InstantiatingGrpcChannelProvider( + directPathProviderBuilder(credentials, interconnectEnvValue) + .setAttemptDirectPathXdsOverInterconnect(true), + "not-gce-product-name"); + } + + /** + * Returns the channel credentials wrapped by a DirectPath channel's {@link + * CompositeChannelCredentials}. GoogleDefaultChannelCredentials and the Interconnect credentials + * both attach the call credentials this way, so the inner type tells them apart. + */ + private static ChannelCredentials innerChannelCredentials(ChannelCredentials channelCredentials) { + assertThat(channelCredentials).isInstanceOf(CompositeChannelCredentials.class); + return ((CompositeChannelCredentials) channelCredentials).getChannelCredentials(); + } + + private static CallCredentials callCredentialsOf(ChannelCredentials channelCredentials) { + assertThat(channelCredentials).isInstanceOf(CompositeChannelCredentials.class); + return ((CompositeChannelCredentials) channelCredentials).getCallCredentials(); + } + + /** + * Returns the {@link Credentials} that {@link MoreCallCredentials#from} wrapped into {@code + * callCredentials}, read from the wrapper's {@code creds} field. The ALTS hard-bound token + * credential is wrapped the same way, so only the identity of the wrapped instance tells which + * credential a channel carries. + */ + private static Credentials credentialsWrappedBy(CallCredentials callCredentials) + throws ReflectiveOperationException { + assertThat(callCredentials) + .isInstanceOf( + MoreCallCredentials.from(mock(Credentials.class, withSettings().withoutAnnotations())) + .getClass()); + Field credsField = callCredentials.getClass().getDeclaredField("creds"); + credsField.setAccessible(true); + return (Credentials) credsField.get(callCredentials); + } + + @Test + void createDirectPathChannelCredentials_interconnect_usesTlsWithCallCredentials() + throws Exception { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + InstantiatingGrpcChannelProvider provider = + interconnectProvider(credentials, /* interconnectEnvValue= */ null); + + ChannelCredentials channelCredentials = provider.createDirectPathChannelCredentials(); + + assertThat(innerChannelCredentials(channelCredentials)) + .isInstanceOf(TlsChannelCredentials.class); + // The call credentials stay on the channel credentials so the RLS control channel inherits + // them. + assertThat(credentialsWrappedBy(callCredentialsOf(channelCredentials))) + .isSameInstanceAs(credentials); + } + + // The builder option is unset; the variable is trimmed and matched case-insensitively. + @ParameterizedTest + @ValueSource(strings = {"true", "TRUE", " True "}) + void createDirectPathChannelCredentials_interconnectEnvVar_usesTls(String envValue) { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + InstantiatingGrpcChannelProvider provider = + new InstantiatingGrpcChannelProvider( + directPathProviderBuilder(credentials, envValue), "not-gce-product-name"); + + ChannelCredentials channelCredentials = provider.createDirectPathChannelCredentials(); + + assertThat(innerChannelCredentials(channelCredentials)) + .isInstanceOf(TlsChannelCredentials.class); + } + + @Test + void createDirectPathChannelCredentials_interconnectInvalidEnvVar_fallsBackToBuilderOption() { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + InstantiatingGrpcChannelProvider enabledProvider = interconnectProvider(credentials, "yes"); + InstantiatingGrpcChannelProvider disabledProvider = + new InstantiatingGrpcChannelProvider( + directPathProviderBuilder(credentials, "yes") + .setAttemptDirectPathXdsOverInterconnect(false), + "not-gce-product-name"); + + ChannelCredentials enabledCredentials = enabledProvider.createDirectPathChannelCredentials(); + ChannelCredentials disabledCredentials = disabledProvider.createDirectPathChannelCredentials(); + + assertThat(innerChannelCredentials(enabledCredentials)) + .isInstanceOf(TlsChannelCredentials.class); + assertThat(innerChannelCredentials(disabledCredentials)) + .isNotInstanceOf(TlsChannelCredentials.class); + } + + @Test + void createDirectPathChannelCredentials_interconnectEnvFalse_usesGoogleDefault() { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + // The environment variable overrides the builder option, so this is standard DirectPath. + InstantiatingGrpcChannelProvider provider = interconnectProvider(credentials, "false"); + + ChannelCredentials channelCredentials = provider.createDirectPathChannelCredentials(); + + assertThat(innerChannelCredentials(channelCredentials)) + .isNotInstanceOf(TlsChannelCredentials.class); + } + + @Test + void createDirectPathChannelCredentials_interconnectIgnoresAltsCallCredentials() + throws Exception { + when(computeEngineCredentials.toBuilder()).thenReturn(ComputeEngineCredentials.newBuilder()); + InstantiatingGrpcChannelProvider.Builder builder = + directPathProviderBuilder(computeEngineCredentials, /* interconnectEnvValue= */ null) + .setAttemptDirectPathXdsOverInterconnect(true) + .setAllowHardBoundTokenTypes(Collections.singletonList(HardBoundTokenTypes.ALTS)); + assertThat(builder.isDirectPathBoundTokenEnabled()).isTrue(); + // build() is what creates altsCallCredentials, from a copy of the credentials. + InstantiatingGrpcChannelProvider provider = builder.build(); + + ChannelCredentials channelCredentials = provider.createDirectPathChannelCredentials(); + + assertThat(innerChannelCredentials(channelCredentials)) + .isInstanceOf(TlsChannelCredentials.class); + // Hard-bound tokens need an ALTS connection, so the channel carries the original credentials + // alone: neither the hard-bound copy nor GoogleDefaultChannelCredentials' pair of both. + assertThat(credentialsWrappedBy(callCredentialsOf(channelCredentials))) + .isSameInstanceAs(computeEngineCredentials); + } + + @Test + void createDirectPathChannelCredentials_directPath_usesGoogleDefault() throws Exception { + when(computeEngineCredentials.toBuilder()).thenReturn(ComputeEngineCredentials.newBuilder()); + InstantiatingGrpcChannelProvider withoutBoundTokens = + directPathProviderBuilder(computeEngineCredentials, /* interconnectEnvValue= */ null) + .build(); + InstantiatingGrpcChannelProvider withBoundTokens = + directPathProviderBuilder(computeEngineCredentials, /* interconnectEnvValue= */ null) + .setAllowHardBoundTokenTypes(Collections.singletonList(HardBoundTokenTypes.ALTS)) + .build(); + + ChannelCredentials withoutBoundTokensCredentials = + withoutBoundTokens.createDirectPathChannelCredentials(); + ChannelCredentials withBoundTokensCredentials = + withBoundTokens.createDirectPathChannelCredentials(); + + // GoogleDefaultChannelCredentials wraps its own TLS/ALTS negotiator, not TlsChannelCredentials. + assertThat(innerChannelCredentials(withoutBoundTokensCredentials)) + .isNotInstanceOf(TlsChannelCredentials.class); + assertThat(credentialsWrappedBy(callCredentialsOf(withoutBoundTokensCredentials))) + .isSameInstanceAs(computeEngineCredentials); + assertThat(innerChannelCredentials(withBoundTokensCredentials)) + .isNotInstanceOf(TlsChannelCredentials.class); + // With ALTS hard-bound tokens, GoogleDefaultChannelCredentials pairs both call credentials in + // a CallCredentials type of its own. + assertThat(callCredentialsOf(withBoundTokensCredentials)) + .isNotInstanceOf(MoreCallCredentials.from(computeEngineCredentials).getClass()); + } + + @Test + void createChannelBuilder_interconnectWithMtlsProvider_usesTlsWithoutClientCert() + throws Exception { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, withSettings().withoutAnnotations()); + when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + when(mtlsCertificateBasedAccess.getWorkloadCertPath()).thenReturn("/path/to/workload/cert.pem"); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + InstantiatingGrpcChannelProvider provider = + new InstantiatingGrpcChannelProvider( + directPathProviderBuilder(credentials, /* interconnectEnvValue= */ null) + .setAttemptDirectPathXdsOverInterconnect(true) + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setMtlsProvider(mtlsProvider), + "not-gce-product-name"); + + ManagedChannelBuilder channelBuilder = provider.createChannelBuilder(); + ChannelCredentials inner = + innerChannelCredentials(extractChannelCredentialsFromChannelBuilder(channelBuilder)); + + // DirectPath channels never present the client certificate, Interconnect included; both + // accessors are null unless TlsChannelCredentials was built with a client identity. + assertThat(inner).isInstanceOf(TlsChannelCredentials.class); + assertThat(((TlsChannelCredentials) inner).getKeyManagers()).isNull(); + assertThat(((TlsChannelCredentials) inner).getCertificateChain()).isNull(); + // Consequently the pool does not track the workload certificate for rotation either. + TransportChannel transportChannel = provider.getTransportChannel(); + try { + ChannelPool pool = (ChannelPool) ((GrpcTransportChannel) transportChannel).getChannel(); + assertThat(pool.getWorkloadCertPath()).isNull(); + } finally { + transportChannel.shutdownNow(); + } + } + + @Test + void createChannelBuilder_interconnectWithoutCredentials_usesCloudPathClientCert() + throws Exception { + CertificateBasedAccess mtlsCertificateBasedAccess = + mock(CertificateBasedAccess.class, withSettings().withoutAnnotations()); + when(mtlsCertificateBasedAccess.useMtlsClientCertificate()).thenReturn(true); + MtlsProvider mtlsProvider = + new FakeMtlsProvider(FakeMtlsProvider.createTestMtlsKeyStore(), "", false); + InstantiatingGrpcChannelProvider provider = + new InstantiatingGrpcChannelProvider( + directPathProviderBuilder(/* credentials= */ null, /* interconnectEnvValue= */ null) + .setAttemptDirectPathXdsOverInterconnect(true) + .setCertificateBasedAccess(mtlsCertificateBasedAccess) + .setMtlsProvider(mtlsProvider), + "not-gce-product-name"); + + ManagedChannelBuilder channelBuilder = provider.createChannelBuilder(); + + // Without credentials the channel is CloudPath, which presents the client certificate. + assertThat(extractTargetFromChannelBuilder(channelBuilder)) + .isEqualTo("storage.googleapis.com:443"); + ChannelCredentials channelCredentials = + extractChannelCredentialsFromChannelBuilder(channelBuilder); + assertThat(channelCredentials).isInstanceOf(TlsChannelCredentials.class); + assertThat(((TlsChannelCredentials) channelCredentials).getKeyManagers()).isNotNull(); + } + + @Test + void createChannelBuilder_interconnect_usesTlsChannelCredentials() throws IOException { + Credentials credentials = mock(Credentials.class, withSettings().withoutAnnotations()); + InstantiatingGrpcChannelProvider provider = + interconnectProvider(credentials, /* interconnectEnvValue= */ null); + + ManagedChannelBuilder channelBuilder = provider.createChannelBuilder(); + + assertThat(extractTargetFromChannelBuilder(channelBuilder)) + .isEqualTo("google-c2p:///storage-direct.googleapis.com?force-xds"); + assertThat(innerChannelCredentials(extractChannelCredentialsFromChannelBuilder(channelBuilder))) + .isInstanceOf(TlsChannelCredentials.class); + } + @Test void createChannel_whenDirectPathEnabled_ignoresWorkloadCertPath() throws Exception { System.setProperty("os.name", "Linux"); @@ -2165,6 +2603,13 @@ public void close() throws SecurityException {} List getAllMessages() { return records.stream().map(LogRecord::getMessage).collect(Collectors.toList()); } + + List levelsOf(String message) { + return records.stream() + .filter(record -> record.getMessage().equals(message)) + .map(LogRecord::getLevel) + .collect(Collectors.toList()); + } } private static String extractAuthorityFromChannelBuilder(