From 4afefe58390305b3f6cc3116d986d804012ea7d8 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 25 Aug 2026 02:16:08 +0000 Subject: [PATCH 1/2] test(datalabeling): configure retries and 24h resource cleanup in ITSystemTest Configures custom GAX RetrySettings for UNAVAILABLE and DEADLINE_EXCEEDED on create operations in ITSystemTest to prevent flaky 502 Bad Gateway failures. Adds 24-hour stale resource sweep and unique timestamped display names to prevent leaking test resources across retries and crashed runs. Fixes #14176 --- .../cloud/datalabeling/it/ITSystemTest.java | 148 ++++++++++++++++-- 1 file changed, 135 insertions(+), 13 deletions(-) diff --git a/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java b/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java index bc2f0a45fcbb..b4a1cd4daef4 100644 --- a/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java +++ b/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java @@ -18,20 +18,28 @@ import static org.junit.Assert.assertEquals; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.StatusCode; import com.google.cloud.ServiceOptions; import com.google.cloud.datalabeling.v1beta1.AnnotationSpec; import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSet; import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSetName; import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient; +import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings; import com.google.cloud.datalabeling.v1beta1.Dataset; import com.google.cloud.datalabeling.v1beta1.DatasetName; import com.google.cloud.datalabeling.v1beta1.ProjectName; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -45,8 +53,12 @@ public class ITSystemTest { private static final Logger LOGGER = Logger.getLogger(ITSystemTest.class.getName()); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String DATASET_DISPLAY_NAME = "test_dataset_" + UUID.randomUUID(); - private static final String ANNOTATION_SPEC_SET = "test_annotationSpecSet_" + UUID.randomUUID(); + private static final String DATASET_PREFIX = "test_dataset_"; + private static final String ANNOTATION_SPEC_SET_PREFIX = "test_annotationSpecSet_"; + private static final String RUN_ID = + System.currentTimeMillis() + "_" + UUID.randomUUID().toString().substring(0, 8); + private static final String DATASET_DISPLAY_NAME = DATASET_PREFIX + RUN_ID; + private static final String ANNOTATION_SPEC_SET = ANNOTATION_SPEC_SET_PREFIX + RUN_ID; private static final String LABEL_1 = "label_1"; private static final String LABEL_2 = "label_2"; private static final String DESCRIPTION = "test_description"; @@ -59,7 +71,43 @@ public class ITSystemTest { @BeforeClass public static void beforeClass() throws Exception { - client = DataLabelingServiceClient.create(); + DataLabelingServiceSettings.Builder settingsBuilder = DataLabelingServiceSettings.newBuilder(); + + // Resource creation RPCs (createDataset, createAnnotationSpecSet) are non-idempotent by default + // and generated with no retryable codes under AIP-194. However, live integration tests running + // against backend services occasionally encounter transient 502 Bad Gateway / UNAVAILABLE + // proxy errors from GFE/Envoy during cold starts or backend resets. We configure custom + // RetrySettings with backoff for test resilience, paired with unique display names and + // comprehensive project sweeps in @AfterClass and @BeforeClass to avoid leaking resources. + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setInitialRetryDelayDuration(Duration.ofSeconds(2)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelayDuration(Duration.ofSeconds(10)) + .setInitialRpcTimeoutDuration(Duration.ofSeconds(30)) + .setMaxRpcTimeoutDuration(Duration.ofSeconds(60)) + .setTotalTimeoutDuration(Duration.ofMinutes(2)) + .build(); + Set retryableCodes = + ImmutableSet.of(StatusCode.Code.UNAVAILABLE, StatusCode.Code.DEADLINE_EXCEEDED); + + settingsBuilder + .createDatasetSettings() + .setRetryableCodes(retryableCodes) + .setRetrySettings(retrySettings); + + settingsBuilder + .createAnnotationSpecSetSettings() + .setRetryableCodes(retryableCodes) + .setRetrySettings(retrySettings); + + client = DataLabelingServiceClient.create(settingsBuilder.build()); + + // Clean up any stale resources older than 24 hours from previous crashed or interrupted test + // runs + Instant cutoff = Instant.now().minus(Duration.ofHours(24)); + cleanUpDatasets(client, cutoff); + cleanUpAnnotationSpecSets(client, cutoff); /** create Dataset */ Dataset dataSetResponse = client.createDataset(PARENT, DATASET); @@ -92,18 +140,92 @@ public static void beforeClass() throws Exception { } @AfterClass - public static void afterClass() { - if (annotationSpecSetId != null) { - String annotationSpecSetName = AnnotationSpecSetName.format(PROJECT_ID, annotationSpecSetId); - client.deleteAnnotationSpecSet(annotationSpecSetName); - LOGGER.info("AnnotationSpecSet deleted successfully."); + public static void afterClass() throws Exception { + if (client != null) { + // Clean up current run resources as well as any resources older than 24 hours + Instant cutoff = Instant.now().minus(Duration.ofHours(24)); + cleanUpDatasets(client, cutoff); + cleanUpAnnotationSpecSets(client, cutoff); + + client.close(); + client.awaitTermination(10, TimeUnit.SECONDS); + } + } + + /** + * Sweeps datasets in the GCP project. Deletes resources created by the current test run (matching + * {@code DATASET_DISPLAY_NAME}) as well as stale datasets older than the 24-hour cutoff (or + * legacy datasets without valid creation timestamps) from previous crashed or aborted test runs. + */ + private static void cleanUpDatasets(DataLabelingServiceClient client, Instant cutoff) { + try { + for (Dataset dataset : client.listDatasets(PARENT, "").iterateAll()) { + boolean isCurrentRun = DATASET_DISPLAY_NAME.equals(dataset.getDisplayName()); + boolean isStale = false; + if (dataset.getDisplayName().startsWith(DATASET_PREFIX)) { + if (dataset.hasCreateTime()) { + Instant createTime = + Instant.ofEpochSecond( + dataset.getCreateTime().getSeconds(), dataset.getCreateTime().getNanos()); + isStale = createTime.isBefore(cutoff); + } else { + // If creation timestamp is missing, assume it is a legacy resource and mark for cleanup + isStale = true; + } + } + if (isCurrentRun || isStale) { + try { + client.deleteDataset(dataset.getName()); + } catch (Exception e) { + LOGGER.warning("Failed to delete Dataset " + dataset.getName() + ": " + e.getMessage()); + } + } + } + } catch (Exception e) { + LOGGER.warning("Failed to sweep datasets: " + e.getMessage()); } - if (dataSetId != null) { - String dataSet = DatasetName.format(PROJECT_ID, dataSetId); - client.deleteDataset(dataSet); - LOGGER.info("Dataset deleted successfully."); + } + + /** + * Sweeps annotation spec sets in the GCP project. Deletes resources created by the current test + * run (matching {@code ANNOTATION_SPEC_SET}) as well as stale annotation spec sets older than the + * 24-hour cutoff (or legacy sets that do not conform to the timestamped naming format) from + * previous crashed or aborted test runs. + */ + private static void cleanUpAnnotationSpecSets(DataLabelingServiceClient client, Instant cutoff) { + try { + for (AnnotationSpecSet specSet : client.listAnnotationSpecSets(PARENT, "").iterateAll()) { + boolean isCurrentRun = ANNOTATION_SPEC_SET.equals(specSet.getDisplayName()); + boolean isStale = false; + if (specSet.getDisplayName().startsWith(ANNOTATION_SPEC_SET_PREFIX)) { + String[] parts = specSet.getDisplayName().split("_"); + // Format: test_annotationSpecSet__ + if (parts.length >= 3) { + try { + long timestamp = Long.parseLong(parts[2]); + isStale = Instant.ofEpochMilli(timestamp).isBefore(cutoff); + } catch (NumberFormatException e) { + // If the timestamp part is non-numeric, assume legacy resource and mark for cleanup + isStale = true; + } + } else { + // If the name does not match the timestamped format, assume legacy resource and mark + // for cleanup + isStale = true; + } + } + if (isCurrentRun || isStale) { + try { + client.deleteAnnotationSpecSet(specSet.getName()); + } catch (Exception e) { + LOGGER.warning( + "Failed to delete AnnotationSpecSet " + specSet.getName() + ": " + e.getMessage()); + } + } + } + } catch (Exception e) { + LOGGER.warning("Failed to sweep annotationSpecSets: " + e.getMessage()); } - client.close(); } @Test From 024dcec424d871967ff6930a04ef270f89ab3745 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 25 Aug 2026 14:45:47 +0000 Subject: [PATCH 2/2] test(datalabeling): increase retry timeouts and total timeout duration in ITSystemTest --- .../com/google/cloud/datalabeling/it/ITSystemTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java b/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java index b4a1cd4daef4..8ef8bdcdad08 100644 --- a/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java +++ b/java-datalabeling/google-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java @@ -83,10 +83,11 @@ public static void beforeClass() throws Exception { RetrySettings.newBuilder() .setInitialRetryDelayDuration(Duration.ofSeconds(2)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelayDuration(Duration.ofSeconds(10)) - .setInitialRpcTimeoutDuration(Duration.ofSeconds(30)) - .setMaxRpcTimeoutDuration(Duration.ofSeconds(60)) - .setTotalTimeoutDuration(Duration.ofMinutes(2)) + .setMaxRetryDelayDuration(Duration.ofSeconds(30)) + .setInitialRpcTimeoutDuration(Duration.ofSeconds(60)) + .setRpcTimeoutMultiplier(1.5) + .setMaxRpcTimeoutDuration(Duration.ofSeconds(120)) + .setTotalTimeoutDuration(Duration.ofMinutes(5)) .build(); Set retryableCodes = ImmutableSet.of(StatusCode.Code.UNAVAILABLE, StatusCode.Code.DEADLINE_EXCEEDED);