Skip to content

test(datalabeling): configure retries and 24h resource cleanup in ITSystemTest #14183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lqiu96 wants to merge 3 commits into from
Closed

test(datalabeling): configure retries and 24h resource cleanup in ITSystemTest #14183

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4afefe5
test(datalabeling): configure retries and 24h resource cleanup in ITS…
lqiu96 Aug 25, 2026
024dcec
test(datalabeling): increase retry timeouts and total timeout duratio…
lqiu96 Aug 25, 2026
2e9285e
Merge branch 'main' into fix_issue_14176_investigation
lqiu96 Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 136 additions & 13 deletions ...oogle-cloud-datalabeling/src/test/java/com/google/cloud/datalabeling/it/ITSystemTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,28 @@

import static org.junit.Assert.assertEquals;

import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.rpc.StatusCode;
import com.google.cloud.ServiceOptions;
import com.google.cloud.datalabeling.v1beta1.AnnotationSpec;
import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSet;
import com.google.cloud.datalabeling.v1beta1.AnnotationSpecSetName;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceClient;
import com.google.cloud.datalabeling.v1beta1.DataLabelingServiceSettings;
import com.google.cloud.datalabeling.v1beta1.Dataset;
import com.google.cloud.datalabeling.v1beta1.DatasetName;
import com.google.cloud.datalabeling.v1beta1.ProjectName;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;
import org.junit.AfterClass;
import org.junit.BeforeClass;
Expand All @@ -45,8 +53,12 @@ public class ITSystemTest {

private static final Logger LOGGER = Logger.getLogger(ITSystemTest.class.getName());
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
private static final String DATASET_DISPLAY_NAME = "test_dataset_" + UUID.randomUUID();
private static final String ANNOTATION_SPEC_SET = "test_annotationSpecSet_" + UUID.randomUUID();
private static final String DATASET_PREFIX = "test_dataset_";
private static final String ANNOTATION_SPEC_SET_PREFIX = "test_annotationSpecSet_";
private static final String RUN_ID =
System.currentTimeMillis() + "_" + UUID.randomUUID().toString().substring(0, 8);
private static final String DATASET_DISPLAY_NAME = DATASET_PREFIX + RUN_ID;
private static final String ANNOTATION_SPEC_SET = ANNOTATION_SPEC_SET_PREFIX + RUN_ID;
private static final String LABEL_1 = "label_1";
private static final String LABEL_2 = "label_2";
private static final String DESCRIPTION = "test_description";
Expand All @@ -59,7 +71,44 @@ public class ITSystemTest {

@BeforeClass
public static void beforeClass() throws Exception {
client = DataLabelingServiceClient.create();
DataLabelingServiceSettings.Builder settingsBuilder = DataLabelingServiceSettings.newBuilder();

// Resource creation RPCs (createDataset, createAnnotationSpecSet) are non-idempotent by default
// and generated with no retryable codes under AIP-194. However, live integration tests running
// against backend services occasionally encounter transient 502 Bad Gateway / UNAVAILABLE
// proxy errors from GFE/Envoy during cold starts or backend resets. We configure custom
// RetrySettings with backoff for test resilience, paired with unique display names and
// comprehensive project sweeps in @AfterClass and @BeforeClass to avoid leaking resources.
RetrySettings retrySettings =
RetrySettings.newBuilder()
.setInitialRetryDelayDuration(Duration.ofSeconds(2))
.setRetryDelayMultiplier(1.5)
.setMaxRetryDelayDuration(Duration.ofSeconds(30))
.setInitialRpcTimeoutDuration(Duration.ofSeconds(60))
.setRpcTimeoutMultiplier(1.5)
.setMaxRpcTimeoutDuration(Duration.ofSeconds(120))
.setTotalTimeoutDuration(Duration.ofMinutes(5))
.build();
Set<StatusCode.Code> retryableCodes =
ImmutableSet.of(StatusCode.Code.UNAVAILABLE, StatusCode.Code.DEADLINE_EXCEEDED);

settingsBuilder
.createDatasetSettings()
.setRetryableCodes(retryableCodes)
.setRetrySettings(retrySettings);

settingsBuilder
.createAnnotationSpecSetSettings()
.setRetryableCodes(retryableCodes)
.setRetrySettings(retrySettings);

client = DataLabelingServiceClient.create(settingsBuilder.build());

// Clean up any stale resources older than 24 hours from previous crashed or interrupted test
// runs
Instant cutoff = Instant.now().minus(Duration.ofHours(24));
cleanUpDatasets(client, cutoff);
cleanUpAnnotationSpecSets(client, cutoff);

/** create Dataset */
Dataset dataSetResponse = client.createDataset(PARENT, DATASET);
Expand Down Expand Up @@ -92,18 +141,92 @@ public static void beforeClass() throws Exception {
}

@AfterClass
public static void afterClass() {
if (annotationSpecSetId != null) {
String annotationSpecSetName = AnnotationSpecSetName.format(PROJECT_ID, annotationSpecSetId);
client.deleteAnnotationSpecSet(annotationSpecSetName);
LOGGER.info("AnnotationSpecSet deleted successfully.");
public static void afterClass() throws Exception {
if (client != null) {
// Clean up current run resources as well as any resources older than 24 hours
Instant cutoff = Instant.now().minus(Duration.ofHours(24));
cleanUpDatasets(client, cutoff);
cleanUpAnnotationSpecSets(client, cutoff);

client.close();
client.awaitTermination(10, TimeUnit.SECONDS);
}
}

/**
* Sweeps datasets in the GCP project. Deletes resources created by the current test run (matching
* {@code DATASET_DISPLAY_NAME}) as well as stale datasets older than the 24-hour cutoff (or
* legacy datasets without valid creation timestamps) from previous crashed or aborted test runs.
*/
private static void cleanUpDatasets(DataLabelingServiceClient client, Instant cutoff) {
try {
for (Dataset dataset : client.listDatasets(PARENT, "").iterateAll()) {
boolean isCurrentRun = DATASET_DISPLAY_NAME.equals(dataset.getDisplayName());
boolean isStale = false;
if (dataset.getDisplayName().startsWith(DATASET_PREFIX)) {
if (dataset.hasCreateTime()) {
Instant createTime =
Instant.ofEpochSecond(
dataset.getCreateTime().getSeconds(), dataset.getCreateTime().getNanos());
isStale = createTime.isBefore(cutoff);
} else {
// If creation timestamp is missing, assume it is a legacy resource and mark for cleanup
isStale = true;
}
}
if (isCurrentRun || isStale) {
try {
client.deleteDataset(dataset.getName());
} catch (Exception e) {
LOGGER.warning("Failed to delete Dataset " + dataset.getName() + ": " + e.getMessage());
}
}
}
} catch (Exception e) {
LOGGER.warning("Failed to sweep datasets: " + e.getMessage());
}
if (dataSetId != null) {
String dataSet = DatasetName.format(PROJECT_ID, dataSetId);
client.deleteDataset(dataSet);
LOGGER.info("Dataset deleted successfully.");
}

/**
* Sweeps annotation spec sets in the GCP project. Deletes resources created by the current test
* run (matching {@code ANNOTATION_SPEC_SET}) as well as stale annotation spec sets older than the
* 24-hour cutoff (or legacy sets that do not conform to the timestamped naming format) from
* previous crashed or aborted test runs.
*/
private static void cleanUpAnnotationSpecSets(DataLabelingServiceClient client, Instant cutoff) {
try {
for (AnnotationSpecSet specSet : client.listAnnotationSpecSets(PARENT, "").iterateAll()) {
boolean isCurrentRun = ANNOTATION_SPEC_SET.equals(specSet.getDisplayName());
boolean isStale = false;
if (specSet.getDisplayName().startsWith(ANNOTATION_SPEC_SET_PREFIX)) {
String[] parts = specSet.getDisplayName().split("_");
// Format: test_annotationSpecSet_<timestamp>_<uuid>
if (parts.length >= 3) {
try {
long timestamp = Long.parseLong(parts[2]);
isStale = Instant.ofEpochMilli(timestamp).isBefore(cutoff);
} catch (NumberFormatException e) {
// If the timestamp part is non-numeric, assume legacy resource and mark for cleanup
isStale = true;
}
} else {
// If the name does not match the timestamped format, assume legacy resource and mark
// for cleanup
isStale = true;
}
}
if (isCurrentRun || isStale) {
try {
client.deleteAnnotationSpecSet(specSet.getName());
} catch (Exception e) {
LOGGER.warning(
"Failed to delete AnnotationSpecSet " + specSet.getName() + ": " + e.getMessage());
}
}
}
} catch (Exception e) {
LOGGER.warning("Failed to sweep annotationSpecSets: " + e.getMessage());
}
client.close();
}

@Test
Expand Down
Loading