From 7c00831e659efbfac754e2ddf464be7416446592 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 15:38:17 +0800 Subject: [PATCH 01/10] [rest] Add database branch and tag APIs --- docs/static/rest-catalog-open-api.yaml | 293 ++++++++++++++++++ .../apache/paimon/rest/DatabaseReference.java | 89 ++++++ .../paimon/rest/DatabaseReferenceType.java | 35 +++ .../org/apache/paimon/rest/HttpClient.java | 53 +++- .../apache/paimon/rest/HttpClientUtils.java | 5 + .../java/org/apache/paimon/rest/RESTApi.java | 114 +++++++ .../paimon/rest/RESTCatalogOptions.java | 8 + .../org/apache/paimon/rest/ResourcePaths.java | 13 + .../ListDatabaseReferencesResponse.java | 72 +++++ .../SingleDatabaseReferenceResponse.java | 51 +++ .../rest/RESTApiDatabaseReferenceTest.java | 282 +++++++++++++++++ 11 files changed, 1009 insertions(+), 6 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 02ee6df595be..24964106bc00 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -203,6 +203,256 @@ paths: $ref: '#/components/responses/DatabaseNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees: + get: + tags: + - database-reference + summary: List database references + operationId: listDatabaseReferences + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: type + in: query + required: false + schema: + type: string + enum: [ "branch", "tag" ] + - name: maxResults + in: query + required: false + schema: + type: integer + format: int32 + - name: pageToken + in: query + required: false + schema: + type: string + responses: + "200": + description: Database branches and immutable tags. + content: + application/json: + schema: + $ref: '#/components/schemas/ListDatabaseReferencesResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + post: + tags: + - database-reference + summary: Create database reference + operationId: createDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: query + required: true + schema: + type: string + - name: type + in: query + required: true + schema: + type: string + enum: [ "branch", "tag" ] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReference' + responses: + "200": + description: Created branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "409": + description: Reference already exists. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{name}: + get: + tags: + - database-reference + summary: Get database reference + operationId: getDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: Named branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database or reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + put: + tags: + - database-reference + summary: Fast-forward database branch from immutable tag + operationId: fastForwardDatabaseBranch + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + - name: mode + in: query + required: true + schema: + type: string + const: "FAST_FORWARD" + - name: type + in: query + required: false + schema: + type: string + const: "branch" + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReference' + responses: + "200": + description: Updated target branch. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database, target branch, or source tag does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Target is not an ancestor of the source tag. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + delete: + tags: + - database-reference + summary: Delete database reference + operationId: deleteDatabaseReference + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + - name: type + in: query + required: false + schema: + type: string + enum: [ "branch", "tag" ] + responses: + "200": + description: Deleted branch or immutable tag. + content: + application/json: + schema: + $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database or reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Reference type does not match or the default branch is protected. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/register: post: tags: @@ -269,6 +519,7 @@ paths: in: query schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -461,6 +712,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -595,6 +847,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' requestBody: content: application/json: @@ -810,6 +1063,7 @@ paths: required: true schema: type: string + - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -2406,6 +2660,15 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: + parameters: + DatabaseReferenceHeader: + name: Paimon-Reference + in: header + required: false + description: Database-level branch or immutable tag. Required for a versioned database. + schema: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" ############################# # Reusable Response Objects # ############################# @@ -3753,6 +4016,36 @@ components: $ref: '#/components/schemas/Identifier' nextPageToken: type: string + DatabaseReference: + type: object + required: + - type + - name + properties: + type: + type: string + enum: [ "BRANCH", "TAG" ] + name: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + SingleDatabaseReferenceResponse: + type: object + required: + - reference + properties: + reference: + $ref: '#/components/schemas/DatabaseReference' + ListDatabaseReferencesResponse: + type: object + required: + - references + properties: + references: + type: array + items: + $ref: '#/components/schemas/DatabaseReference' + nextPageToken: + type: string ConfigResponse: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java new file mode 100644 index 000000000000..416dff1749ce --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** A named database-level branch or immutable tag. */ +@Experimental +public class DatabaseReference implements RESTRequest { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_NAME = "name"; + private static final String NAME_PATTERN = "[A-Za-z0-9][A-Za-z0-9._-]{0,127}"; + + @JsonProperty(FIELD_TYPE) + private final DatabaseReferenceType type; + + @JsonProperty(FIELD_NAME) + private final String name; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE, FIELD_NAME}) + public DatabaseReference( + @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, + @JsonProperty(FIELD_NAME) String name) { + checkArgument(type != null, "Reference type must not be null"); + checkArgument( + name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + this.type = type; + this.name = name; + } + + @JsonGetter(FIELD_TYPE) + public DatabaseReferenceType getType() { + return type; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DatabaseReference)) { + return false; + } + DatabaseReference that = (DatabaseReference) o; + return type == that.type && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(type, name); + } + + @Override + public String toString() { + return type + ":" + name; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java new file mode 100644 index 000000000000..55b3e7813a62 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReferenceType.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +import java.util.Locale; + +/** Types of database-level references supported by the REST catalog. */ +@Experimental +public enum DatabaseReferenceType { + BRANCH, + TAG; + + /** Lowercase form used by the trees query parameters. */ + public String queryValue() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 8205dfe21295..e30ecab35f3f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -32,6 +32,7 @@ import org.apache.hc.client5.http.classic.methods.HttpDelete; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.ContentType; @@ -97,12 +98,21 @@ public T post( RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, null)); + return post(path, Collections.emptyMap(), body, responseType, restAuthFunction); + } + + public T post( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, queryParams)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPost.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, "POST", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, queryParams, "POST", encodedBody, restAuthFunction); httpPost.setHeaders(authHeaders); // A POST the server cannot absorb twice is sent exactly once, whatever the status says. return exec( @@ -113,22 +123,53 @@ public T post( : null); } + public T put( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, queryParams)); + String encodedBody = RESTUtil.encodedBody(body); + if (encodedBody != null) { + httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); + } + Header[] authHeaders = getHeaders(path, queryParams, "PUT", encodedBody, restAuthFunction); + httpPut.setHeaders(authHeaders); + return exec( + httpPut, + responseType, + body != null && !body.isRetrySafe() + ? ExponentialHttpRequestRetryStrategy.retryUnsafeContext() + : null); + } + @Override public T delete(String path, RESTAuthFunction restAuthFunction) { - return delete(path, null, restAuthFunction); + return delete(path, Collections.emptyMap(), null, null, restAuthFunction); } @Override public T delete( String path, RESTRequest body, RESTAuthFunction restAuthFunction) { - HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, null)); + return delete(path, Collections.emptyMap(), body, null, restAuthFunction); + } + + public T delete( + String path, + Map queryParams, + RESTRequest body, + Class responseType, + RESTAuthFunction restAuthFunction) { + HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, queryParams)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpDelete.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, "DELETE", encodedBody, restAuthFunction); + Header[] authHeaders = + getHeaders(path, queryParams, "DELETE", encodedBody, restAuthFunction); httpDelete.setHeaders(authHeaders); - return exec(httpDelete, null); + return exec(httpDelete, responseType); } @VisibleForTesting diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java index 1444c8118d14..94d0ba80f723 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java @@ -26,6 +26,7 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpHead; import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -238,6 +239,10 @@ public static HttpPost newHttpPost(String uri) { return newRequest(uri, HttpPost::new); } + public static HttpPut newHttpPut(String uri) { + return newRequest(uri, HttpPut::new); + } + public static HttpDelete newHttpDelete(String uri) { return newRequest(uri, HttpDelete::new); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 8246cb153335..64b5aeab8873 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -90,6 +90,7 @@ import org.apache.paimon.rest.responses.GetViewResponse; import org.apache.paimon.rest.responses.ListBranchesResponse; import org.apache.paimon.rest.responses.ListConsumersResponse; +import org.apache.paimon.rest.responses.ListDatabaseReferencesResponse; import org.apache.paimon.rest.responses.ListDatabasesResponse; import org.apache.paimon.rest.responses.ListFunctionDetailsResponse; import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; @@ -109,6 +110,7 @@ import org.apache.paimon.rest.responses.ListViewsGloballyResponse; import org.apache.paimon.rest.responses.ListViewsResponse; import org.apache.paimon.rest.responses.PagedResponse; +import org.apache.paimon.rest.responses.SingleDatabaseReferenceResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -194,6 +196,11 @@ public class RESTApi { public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; public static final String TAG_NAME_PREFIX = "tagNamePrefix"; + private static final String REFERENCE_NAME = "name"; + private static final String REFERENCE_TYPE = "type"; + private static final String REFERENCE_UPDATE_MODE = "mode"; + private static final String FAST_FORWARD = "FAST_FORWARD"; + public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; public static final ObjectMapper OBJECT_MAPPER = JsonSerdeUtil.OBJECT_MAPPER_INSTANCE; @@ -360,6 +367,113 @@ public void alterDatabase(String name, List removals, Map listDatabaseReferences( + String databaseName, @Nullable DatabaseReferenceType type) { + return listDataFromPageApi( + queryParams -> { + if (type != null) { + queryParams.put(REFERENCE_TYPE, type.queryValue()); + } + return client.get( + resourcePaths.databaseTrees(databaseName), + queryParams, + ListDatabaseReferencesResponse.class, + restAuthFunction); + }); + } + + /** List one page of database-level branches and immutable tags. */ + @Experimental + public PagedList listDatabaseReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken) { + Map queryParams = buildPagedQueryParams(maxResults, pageToken); + if (type != null) { + queryParams.put(REFERENCE_TYPE, type.queryValue()); + } + ListDatabaseReferencesResponse response = + client.get( + resourcePaths.databaseTrees(databaseName), + queryParams, + ListDatabaseReferencesResponse.class, + restAuthFunction); + List references = response.getReferences(); + return new PagedList<>( + references == null ? emptyList() : references, response.getNextPageToken()); + } + + /** Get one database-level branch or immutable tag. */ + @Experimental + public DatabaseReference getDatabaseReference(String databaseName, String referenceName) { + SingleDatabaseReferenceResponse response = + client.get( + resourcePaths.databaseTree(databaseName, referenceName), + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Create a database-level branch or immutable tag from an existing reference. */ + @Experimental + public DatabaseReference createDatabaseReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source) { + Map queryParams = Maps.newHashMap(); + queryParams.put(REFERENCE_NAME, referenceName); + queryParams.put(REFERENCE_TYPE, type.queryValue()); + SingleDatabaseReferenceResponse response = + client.post( + resourcePaths.databaseTrees(databaseName), + queryParams, + source, + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Fast-forward a database-level branch to an immutable tag. */ + @Experimental + public DatabaseReference fastForwardDatabaseBranch( + String databaseName, String targetBranch, String sourceTag) { + Map queryParams = Maps.newHashMap(); + queryParams.put(REFERENCE_UPDATE_MODE, FAST_FORWARD); + queryParams.put(REFERENCE_TYPE, DatabaseReferenceType.BRANCH.queryValue()); + SingleDatabaseReferenceResponse response = + client.put( + resourcePaths.databaseTree(databaseName, targetBranch), + queryParams, + new DatabaseReference(DatabaseReferenceType.TAG, sourceTag), + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + + /** Delete one database-level branch or immutable tag. */ + @Experimental + public DatabaseReference deleteDatabaseReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType) { + Map queryParams = Maps.newHashMap(); + if (expectedType != null) { + queryParams.put(REFERENCE_TYPE, expectedType.queryValue()); + } + SingleDatabaseReferenceResponse response = + client.delete( + resourcePaths.databaseTree(databaseName, referenceName), + queryParams, + null, + SingleDatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + /** * List tables for a database. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index e7da1f6827dc..3629907dc65f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -98,6 +98,14 @@ public class RESTCatalogOptions { .withDescription( "The user agent of http client connecting to REST Catalog server."); + /** Database-level branch or immutable tag carried by table API requests. */ + public static final ConfigOption DATABASE_REFERENCE = + ConfigOptions.key("header.Paimon-Reference") + .stringType() + .noDefaultValue() + .withDescription( + "Database-level branch or immutable tag used by REST table requests."); + public static final ConfigOption DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint") .stringType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index f3065708da9c..862f78860200 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -36,6 +36,7 @@ public class ResourcePaths { protected static final String PARTITIONS = "partitions"; protected static final String BRANCHES = "branches"; protected static final String TAGS = "tags"; + protected static final String TREES = "trees"; protected static final String SNAPSHOTS = "snapshots"; protected static final String CONSUMERS = "consumers"; protected static final String SCHEMAS = "schemas"; @@ -144,6 +145,18 @@ public String database(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName)); } + /** Database-level branches and immutable tags. */ + @Experimental + public String databaseTrees(String databaseName) { + return SLASH.join(database(databaseName), TREES); + } + + /** One named database-level branch or immutable tag. */ + @Experimental + public String databaseTree(String databaseName, String referenceName) { + return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java new file mode 100644 index 000000000000..3fbbc18e7887 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListDatabaseReferencesResponse.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.List; + +/** Paged response for database-level branches and tags. */ +@Experimental +public class ListDatabaseReferencesResponse implements PagedResponse { + + private static final String FIELD_REFERENCES = "references"; + private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken"; + + @JsonProperty(FIELD_REFERENCES) + private final List references; + + @Nullable + @JsonProperty(FIELD_NEXT_PAGE_TOKEN) + private final String nextPageToken; + + @JsonCreator + @ConstructorProperties({FIELD_REFERENCES, FIELD_NEXT_PAGE_TOKEN}) + public ListDatabaseReferencesResponse( + @JsonProperty(FIELD_REFERENCES) List references, + @Nullable @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) { + this.references = references; + this.nextPageToken = nextPageToken; + } + + @JsonGetter(FIELD_REFERENCES) + public List getReferences() { + return references; + } + + @Nullable + @JsonGetter(FIELD_NEXT_PAGE_TOKEN) + @Override + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return references; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java new file mode 100644 index 000000000000..1bbda053890b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Response containing one database-level reference. */ +@Experimental +public class SingleDatabaseReferenceResponse implements RESTResponse { + + private static final String FIELD_REFERENCE = "reference"; + + @JsonProperty(FIELD_REFERENCE) + private final DatabaseReference reference; + + @JsonCreator + @ConstructorProperties({FIELD_REFERENCE}) + public SingleDatabaseReferenceResponse( + @JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { + this.reference = reference; + } + + @JsonGetter(FIELD_REFERENCE) + public DatabaseReference getReference() { + return reference; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java new file mode 100644 index 000000000000..2ccae1011bce --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.options.Options; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.DATABASE_REFERENCE; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; + +/** HTTP contract tests for database-level branches and immutable tags. */ +class RESTApiDatabaseReferenceTest { + + private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; + + private final Queue replies = new ConcurrentLinkedQueue<>(); + private final List requests = new CopyOnWriteArrayList<>(); + + private HttpServer server; + private RESTApi api; + + @BeforeEach + void setUp() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/v1/", + exchange -> { + requests.add(new Request(exchange)); + Reply reply = replies.poll(); + if (reply == null) { + reply = new Reply(500, "{\"code\":500,\"message\":\"unexpected request\"}"); + } + byte[] data = reply.body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(reply.code, data.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(data); + } finally { + exchange.close(); + } + }); + server.start(); + + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(PREFIX, "catalog/id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + api = new RESTApi(options, false); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void testBranchAndImmutableTagHappyPath() throws Exception { + enqueue( + 200, + "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}," + + "{\"type\":\"TAG\",\"name\":\"train-v1\"}]," + + "\"nextPageToken\":\"next\"}"); + PagedList page = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.TAG, 100, "start token"); + assertThat(page.getElements()) + .containsExactly( + new DatabaseReference(DatabaseReferenceType.BRANCH, "main"), + new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertThat(page.getNextPageToken()).isEqualTo("next"); + assertRequest(0, "GET", TREES_PATH); + assertThat(queryParameters(requests.get(0).query)) + .containsEntry("type", "tag") + .containsEntry("maxResults", "100") + .containsEntry("pageToken", "start token"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + assertThat(api.getDatabaseReference("training db", "main")) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertRequest(1, "GET", TREES_PATH + "/main"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); + DatabaseReference branch = + api.createDatabaseReference( + "training db", + "exp-1", + DatabaseReferenceType.BRANCH, + new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertThat(branch).isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertRequest(2, "POST", TREES_PATH); + assertThat(queryParameters(requests.get(2).query)) + .containsEntry("name", "exp-1") + .containsEntry("type", "branch"); + assertReferenceBody(requests.get(2), "BRANCH", "main"); + + enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); + DatabaseReference tag = + api.createDatabaseReference( + "training db", + "train-v1", + DatabaseReferenceType.TAG, + new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertThat(tag).isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertRequest(3, "POST", TREES_PATH); + assertThat(queryParameters(requests.get(3).query)) + .containsEntry("name", "train-v1") + .containsEntry("type", "tag"); + assertReferenceBody(requests.get(3), "BRANCH", "exp-1"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertRequest(4, "PUT", TREES_PATH + "/main"); + assertThat(queryParameters(requests.get(4).query)) + .containsEntry("mode", "FAST_FORWARD") + .containsEntry("type", "branch"); + assertReferenceBody(requests.get(4), "TAG", "train-v1"); + + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); + assertThat( + api.deleteDatabaseReference( + "training db", "exp-1", DatabaseReferenceType.BRANCH)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertRequest(5, "DELETE", TREES_PATH + "/exp-1"); + assertThat(queryParameters(requests.get(5).query)).containsEntry("type", "branch"); + assertThat(requests.get(5).body).isEmpty(); + } + + @Test + void testListAllReferencesFollowsPages() { + enqueue( + 200, + "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}]," + + "\"nextPageToken\":\"p2\"}"); + enqueue(200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"exp-1\"}]}"); + + assertThat(api.listDatabaseReferences("training db", DatabaseReferenceType.BRANCH)) + .extracting(DatabaseReference::getName) + .containsExactly("main", "exp-1"); + assertThat(requests).hasSize(2); + assertThat(queryParameters(requests.get(0).query)).containsEntry("type", "branch"); + assertThat(queryParameters(requests.get(1).query)) + .containsEntry("type", "branch") + .containsEntry("pageToken", "p2"); + } + + @Test + void testReferenceOptionIsSentToTableApis() { + enqueue(200, "{\"tables\":[]}"); + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(PREFIX, "catalog/id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + options.set(DATABASE_REFERENCE, "train-v1"); + + new RESTApi(options, false).listTables("training db"); + + assertRequest(0, "GET", "/v1/catalog%2Fid/databases/training+db/tables"); + assertThat(requests.get(0).reference).isEqualTo("train-v1"); + } + + private void enqueue(int code, String body) { + replies.add(new Reply(code, body)); + } + + private void assertRequest(int index, String method, String path) { + Request request = requests.get(index); + assertThat(request.method).isEqualTo(method); + assertThat(request.path).isEqualTo(path); + assertThat(request.authorization).isEqualTo("Bearer test-token"); + } + + private static void assertReferenceBody(Request request, String type, String name) + throws Exception { + assertThat(RESTApi.fromJson(request.body, Map.class)) + .containsEntry("type", type) + .containsEntry("name", name) + .hasSize(2); + } + + private static Map queryParameters(String query) { + Map values = new LinkedHashMap<>(); + if (query == null || query.isEmpty()) { + return values; + } + for (String parameter : query.split("&")) { + String[] pair = parameter.split("=", 2); + values.put(decode(pair[0]), decode(pair[1])); + } + return values; + } + + private static String decode(String value) { + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static class Reply { + private final int code; + private final String body; + + private Reply(int code, String body) { + this.code = code; + this.body = body; + } + } + + private static class Request { + private final String method; + private final String path; + private final String query; + private final String body; + private final String authorization; + private final String reference; + + private Request(HttpExchange exchange) throws IOException { + method = exchange.getRequestMethod(); + path = exchange.getRequestURI().getRawPath(); + query = exchange.getRequestURI().getRawQuery(); + body = read(exchange.getRequestBody()); + authorization = exchange.getRequestHeaders().getFirst("Authorization"); + reference = exchange.getRequestHeaders().getFirst("Paimon-Reference"); + } + + private static String read(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = input.read(buffer)) >= 0) { + output.write(buffer, 0, length); + } + return new String(output.toByteArray(), StandardCharsets.UTF_8); + } + } +} From 8532152fd282f420be66e91619ab75b25ece528b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:35:34 +0800 Subject: [PATCH 02/10] [rest] Expose tree management through RESTCatalog --- .../paimon/management/TreeManagement.java | 76 ++++++ .../paimon/rest/RESTTreeManagement.java | 73 ++++++ .../org/apache/paimon/rest/RESTCatalog.java | 6 + .../rest/RESTCatalogTreeManagementTest.java | 231 ++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java new file mode 100644 index 000000000000..29d7c3fe776b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; + +import javax.annotation.Nullable; + +import java.util.List; + +/** Control-plane contract for database-level writable branches and immutable tags. */ +@Experimental +public interface TreeManagement { + + /** Lists all references, following pagination. A null type includes branches and tags. */ + default List listReferences( + String databaseName, @Nullable DatabaseReferenceType type) { + return PagedList.listAllFromPagedApi( + pageToken -> listReferencesPaged(databaseName, type, null, pageToken)); + } + + /** + * Lists one page of references. + * + * @param type reference type to include; null includes branches and tags + * @param maxResults maximum page size; null or zero uses the server default + * @param pageToken opaque continuation token; null for the first page + */ + PagedList listReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken); + + /** Gets a named branch or tag. A missing reference is an error. */ + DatabaseReference getReference(String databaseName, String referenceName); + + /** Creates a branch or immutable tag from an existing reference in the same database. */ + DatabaseReference createReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source); + + /** Fast-forwards a branch to an immutable tag in the same database. */ + DatabaseReference fastForwardBranch(String databaseName, String targetBranch, String sourceTag); + + /** + * Deletes and returns a named reference. A missing reference is an error. + * + * @param expectedType required type of the reference to delete; null omits the type check + */ + DatabaseReference deleteReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType); +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java new file mode 100644 index 000000000000..2be049fb8e02 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.TreeManagement; + +import javax.annotation.Nullable; + +/** REST implementation of tree management, bound to the configured REST catalog prefix. */ +@Experimental +public class RESTTreeManagement implements TreeManagement { + + private final RESTApi api; + + public RESTTreeManagement(RESTApi api) { + this.api = api; + } + + @Override + public PagedList listReferencesPaged( + String databaseName, + @Nullable DatabaseReferenceType type, + @Nullable Integer maxResults, + @Nullable String pageToken) { + return api.listDatabaseReferencesPaged(databaseName, type, maxResults, pageToken); + } + + @Override + public DatabaseReference getReference(String databaseName, String referenceName) { + return api.getDatabaseReference(databaseName, referenceName); + } + + @Override + public DatabaseReference createReference( + String databaseName, + String referenceName, + DatabaseReferenceType type, + DatabaseReference source) { + return api.createDatabaseReference(databaseName, referenceName, type, source); + } + + @Override + public DatabaseReference fastForwardBranch( + String databaseName, String targetBranch, String sourceTag) { + return api.fastForwardDatabaseBranch(databaseName, targetBranch, sourceTag); + } + + @Override + public DatabaseReference deleteReference( + String databaseName, + String referenceName, + @Nullable DatabaseReferenceType expectedType) { + return api.deleteDatabaseReference(databaseName, referenceName, expectedType); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index f691fc06f5b7..58ffc5edb518 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -44,6 +44,7 @@ import org.apache.paimon.management.PermissionManagement; import org.apache.paimon.management.PolicyManagement; import org.apache.paimon.management.SemanticViewManagement; +import org.apache.paimon.management.TreeManagement; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; @@ -166,6 +167,11 @@ public SemanticViewManagement semanticViewManagement() { return new RESTSemanticViewManagement(api); } + @Experimental + public TreeManagement treeManagement() { + return new RESTTreeManagement(api); + } + @Override public List listDatabases() { return api.listDatabases(); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java new file mode 100644 index 000000000000..adb307836e76 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.management.TreeManagement; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.rest.exceptions.NotImplementedException; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.apache.paimon.options.CatalogOptions.WAREHOUSE; +import static org.apache.paimon.rest.DatabaseReferenceType.BRANCH; +import static org.apache.paimon.rest.DatabaseReferenceType.TAG; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Exercises database tree management through a configured REST catalog and its HTTP client. */ +class RESTCatalogTreeManagementTest { + + private static final String DATABASE = "training db"; + private static final String TREES_PATH = "/v1/catalog%2Fid/databases/training+db/trees"; + private static final String MAIN_JSON = "{\"type\":\"BRANCH\",\"name\":\"main\"}"; + private static final String BRANCH_JSON = "{\"type\":\"BRANCH\",\"name\":\"exp-1\"}"; + private static final String TAG_JSON = "{\"type\":\"TAG\",\"name\":\"train-v1\"}"; + + private MockWebServer server; + private RESTCatalog catalog; + private TreeManagement trees; + + @BeforeEach + void setUp() throws Exception { + server = new MockWebServer(); + server.start(); + enqueue( + 200, + "{\"defaults\":{},\"overrides\":{\"prefix\":\"catalog/id\"," + + "\"header.X-Catalog-Context\":\"configured\"}}"); + + Options options = new Options(); + options.set(URI, server.url("/").toString()); + options.set(WAREHOUSE, "warehouse-id"); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + catalog = new RESTCatalog(CatalogContext.create(options)); + trees = catalog.treeManagement(); + + RecordedRequest config = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(config).isNotNull(); + assertThat(config.getRequestUrl().encodedPath()).isEqualTo("/v1/config"); + assertThat(config.getRequestUrl().queryParameter("warehouse")).isEqualTo("warehouse-id"); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @AfterEach + void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + if (server != null) { + server.shutdown(); + } + } + + @Test + void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { + DatabaseReference main = new DatabaseReference(BRANCH, "main"); + DatabaseReference branch = new DatabaseReference(BRANCH, "exp-1"); + DatabaseReference tag = new DatabaseReference(TAG, "train-v1"); + + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + assertThat(trees.getReference(DATABASE, "main")).isEqualTo(main); + takeRequest("GET", TREES_PATH + "/main"); + + enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); + assertThat(trees.createReference(DATABASE, "exp-1", BRANCH, main)).isEqualTo(branch); + RecordedRequest createBranch = takeRequest("POST", TREES_PATH); + assertThat(createBranch.getRequestUrl().queryParameter("name")).isEqualTo("exp-1"); + assertThat(createBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertReferenceBody(createBranch, "BRANCH", "main"); + + enqueue(200, "{\"reference\":" + TAG_JSON + "}"); + assertThat(trees.createReference(DATABASE, "train-v1", TAG, branch)).isEqualTo(tag); + RecordedRequest createTag = takeRequest("POST", TREES_PATH); + assertThat(createTag.getRequestUrl().queryParameter("name")).isEqualTo("train-v1"); + assertThat(createTag.getRequestUrl().queryParameter("type")).isEqualTo("tag"); + assertReferenceBody(createTag, "BRANCH", "exp-1"); + + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); + RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); + assertThat(fastForward.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(fastForward.getRequestUrl().queryParameter("mode")).isEqualTo("FAST_FORWARD"); + assertReferenceBody(fastForward, "TAG", "train-v1"); + + enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); + assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); + RecordedRequest deleteBranch = takeRequest("DELETE", TREES_PATH + "/exp-1"); + assertThat(deleteBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(deleteBranch.getBodySize()).isZero(); + + enqueue(200, "{\"reference\":" + TAG_JSON + "}"); + assertThat(trees.deleteReference(DATABASE, "train-v1", null)).isEqualTo(tag); + assertThat(takeRequest("DELETE", TREES_PATH + "/train-v1").getRequestUrl().query()) + .isNull(); + assertThat(server.getRequestCount()).isEqualTo(7); + } + + @Test + void testListPageAndListAllPreserveFilterAndTokens() throws Exception { + enqueue(200, "{\"references\":[" + TAG_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); + PagedList page = + trees.listReferencesPaged(DATABASE, TAG, 10, "start +/%"); + assertThat(page.getElements()).containsExactly(new DatabaseReference(TAG, "train-v1")); + assertThat(page.getNextPageToken()).isEqualTo("next +/%?&"); + RecordedRequest paged = takeRequest("GET", TREES_PATH); + assertThat(paged.getRequestUrl().queryParameter("type")).isEqualTo("tag"); + assertThat(paged.getRequestUrl().queryParameter("maxResults")).isEqualTo("10"); + assertThat(paged.getRequestUrl().queryParameter("pageToken")).isEqualTo("start +/%"); + + enqueue(200, "{\"references\":[" + MAIN_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); + enqueue(200, "{\"references\":[" + BRANCH_JSON + "]}"); + assertThat(trees.listReferences(DATABASE, BRANCH)) + .containsExactly( + new DatabaseReference(BRANCH, "main"), + new DatabaseReference(BRANCH, "exp-1")); + RecordedRequest first = takeRequest("GET", TREES_PATH); + assertThat(first.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(first.getRequestUrl().queryParameter("pageToken")).isNull(); + RecordedRequest second = takeRequest("GET", TREES_PATH); + assertThat(second.getRequestUrl().queryParameter("type")).isEqualTo("branch"); + assertThat(second.getRequestUrl().queryParameter("pageToken")).isEqualTo("next +/%?&"); + assertThat(second.getRequestUrl().queryParameter("maxResults")).isNull(); + } + + @Test + void testListAllTypesAndEmptyReferences() throws Exception { + enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); + assertThat(trees.listReferences(DATABASE, null)) + .containsExactly( + new DatabaseReference(BRANCH, "main"), + new DatabaseReference(TAG, "train-v1")); + assertThat(takeRequest("GET", TREES_PATH).getRequestUrl().query()).isNull(); + + enqueue(200, "{\"references\":[]}"); + assertThat(trees.listReferences(DATABASE, null)).isEmpty(); + takeRequest("GET", TREES_PATH); + assertThat(server.getRequestCount()).isEqualTo(3); + } + + @Test + void testErrorsPropagate() { + enqueue(404, "{\"code\":404,\"message\":\"reference missing\"}"); + assertThatThrownBy(() -> trees.getReference(DATABASE, "missing")) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("reference missing"); + + enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); + assertThatThrownBy( + () -> + trees.createReference( + DATABASE, + "exp-1", + BRANCH, + new DatabaseReference(BRANCH, "main"))) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("reference already exists"); + + enqueue(501, "{\"code\":501,\"message\":\"trees unsupported\"}"); + assertThatThrownBy(() -> trees.listReferences(DATABASE, null)) + .isInstanceOf(NotImplementedException.class) + .hasMessageContaining("trees unsupported"); + assertThat(server.getRequestCount()).isEqualTo(4); + } + + private void enqueue(int status, String body) { + server.enqueue( + new MockResponse() + .setResponseCode(status) + .setHeader("Content-Type", "application/json") + .setBody(body)); + } + + private RecordedRequest takeRequest(String method, String path) throws Exception { + RecordedRequest request = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(request).isNotNull(); + assertThat(request.getMethod()).isEqualTo(method); + assertThat(request.getRequestUrl().encodedPath()).isEqualTo(path); + assertThat(request.getHeader("Authorization")).isEqualTo("Bearer test-token"); + assertThat(request.getHeader("X-Catalog-Context")).isEqualTo("configured"); + return request; + } + + private static void assertReferenceBody(RecordedRequest request, String type, String name) + throws Exception { + assertThat(RESTApi.fromJson(request.getBody().readUtf8(), Map.class)) + .containsEntry("type", type) + .containsEntry("name", name) + .hasSize(2); + } +} From 87de0554252c742d7fbf34046424f84a8658f05e Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:45:26 +0800 Subject: [PATCH 03/10] [rest] Simplify database reference requests and pagination --- docs/static/rest-catalog-open-api.yaml | 95 +++++++++---------- .../paimon/management/TreeManagement.java | 9 -- .../apache/paimon/rest/DatabaseReference.java | 2 +- .../org/apache/paimon/rest/HttpClient.java | 28 ++---- .../java/org/apache/paimon/rest/RESTApi.java | 60 +++--------- .../paimon/rest/RESTCatalogOptions.java | 8 -- .../CreateDatabaseReferenceRequest.java | 71 ++++++++++++++ .../DeleteDatabaseReferenceRequest.java | 57 +++++++++++ .../FastForwardDatabaseBranchRequest.java | 50 ++++++++++ ...se.java => DatabaseReferenceResponse.java} | 5 +- .../rest/RESTApiDatabaseReferenceTest.java | 73 +++++++------- .../RequestJacksonCompatibilityTest.java | 42 ++++++++ .../rest/RESTCatalogTreeManagementTest.java | 54 ++++++----- 13 files changed, 353 insertions(+), 201 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java rename paimon-api/src/main/java/org/apache/paimon/rest/responses/{SingleDatabaseReferenceResponse.java => DatabaseReferenceResponse.java} (90%) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 24964106bc00..98c1350d0b52 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -208,7 +208,7 @@ paths: tags: - database-reference summary: List database references - operationId: listDatabaseReferences + operationId: listDatabaseReferencesPaged parameters: - name: prefix in: path @@ -266,30 +266,19 @@ paths: required: true schema: type: string - - name: name - in: query - required: true - schema: - type: string - - name: type - in: query - required: true - schema: - type: string - enum: [ "branch", "tag" ] requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/DatabaseReference' + $ref: '#/components/schemas/CreateDatabaseReferenceRequest' responses: "200": description: Created branch or immutable tag. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -330,7 +319,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -362,31 +351,19 @@ paths: required: true schema: type: string - - name: mode - in: query - required: true - schema: - type: string - const: "FAST_FORWARD" - - name: type - in: query - required: false - schema: - type: string - const: "branch" requestBody: required: true content: application/json: schema: - $ref: '#/components/schemas/DatabaseReference' + $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' responses: "200": description: Updated target branch. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -424,19 +401,19 @@ paths: required: true schema: type: string - - name: type - in: query - required: false - schema: - type: string - enum: [ "branch", "tag" ] + requestBody: + required: false + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' responses: "200": description: Deleted branch or immutable tag. content: application/json: schema: - $ref: '#/components/schemas/SingleDatabaseReferenceResponse' + $ref: '#/components/schemas/DatabaseReferenceResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -519,7 +496,6 @@ paths: in: query schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -712,7 +688,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -847,7 +822,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' requestBody: content: application/json: @@ -1063,7 +1037,6 @@ paths: required: true schema: type: string - - $ref: '#/components/parameters/DatabaseReferenceHeader' responses: "200": description: OK @@ -2660,15 +2633,6 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: - parameters: - DatabaseReferenceHeader: - name: Paimon-Reference - in: header - required: false - description: Database-level branch or immutable tag. Required for a versioned database. - schema: - type: string - pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" ############################# # Reusable Response Objects # ############################# @@ -4028,7 +3992,38 @@ components: name: type: string pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" - SingleDatabaseReferenceResponse: + CreateDatabaseReferenceRequest: + type: object + required: + - name + - type + - source + properties: + name: + type: string + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + type: + type: string + enum: [ "BRANCH", "TAG" ] + source: + $ref: '#/components/schemas/DatabaseReference' + FastForwardDatabaseBranchRequest: + type: object + required: + - sourceTag + properties: + sourceTag: + type: string + description: Immutable tag in the same database to fast-forward the target branch to. + pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + DeleteDatabaseReferenceRequest: + type: object + properties: + type: + type: string + description: Expected reference type. Omit to delete without checking the type. + enum: [ "BRANCH", "TAG" ] + DatabaseReferenceResponse: type: object required: - reference diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 29d7c3fe776b..4ca5dd9a26e7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -25,19 +25,10 @@ import javax.annotation.Nullable; -import java.util.List; - /** Control-plane contract for database-level writable branches and immutable tags. */ @Experimental public interface TreeManagement { - /** Lists all references, following pagination. A null type includes branches and tags. */ - default List listReferences( - String databaseName, @Nullable DatabaseReferenceType type) { - return PagedList.listAllFromPagedApi( - pageToken -> listReferencesPaged(databaseName, type, null, pageToken)); - } - /** * Lists one page of references. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 416dff1749ce..845a68deb003 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -31,7 +31,7 @@ /** A named database-level branch or immutable tag. */ @Experimental -public class DatabaseReference implements RESTRequest { +public class DatabaseReference { private static final String FIELD_TYPE = "type"; private static final String FIELD_NAME = "name"; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index e30ecab35f3f..57f44a93fa03 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -98,21 +98,12 @@ public T post( RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - return post(path, Collections.emptyMap(), body, responseType, restAuthFunction); - } - - public T post( - String path, - Map queryParams, - RESTRequest body, - Class responseType, - RESTAuthFunction restAuthFunction) { - HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, queryParams)); + HttpPost httpPost = HttpClientUtils.newHttpPost(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPost.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, queryParams, "POST", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "POST", encodedBody, restAuthFunction); httpPost.setHeaders(authHeaders); // A POST the server cannot absorb twice is sent exactly once, whatever the status says. return exec( @@ -125,16 +116,15 @@ public T post( public T put( String path, - Map queryParams, RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, queryParams)); + HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = getHeaders(path, queryParams, "PUT", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "PUT", encodedBody, restAuthFunction); httpPut.setHeaders(authHeaders); return exec( httpPut, @@ -146,28 +136,26 @@ public T put( @Override public T delete(String path, RESTAuthFunction restAuthFunction) { - return delete(path, Collections.emptyMap(), null, null, restAuthFunction); + return delete(path, null, null, restAuthFunction); } @Override public T delete( String path, RESTRequest body, RESTAuthFunction restAuthFunction) { - return delete(path, Collections.emptyMap(), body, null, restAuthFunction); + return delete(path, body, null, restAuthFunction); } public T delete( String path, - Map queryParams, RESTRequest body, Class responseType, RESTAuthFunction restAuthFunction) { - HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, queryParams)); + HttpDelete httpDelete = HttpClientUtils.newHttpDelete(getRequestUrl(path, null)); String encodedBody = RESTUtil.encodedBody(body); if (encodedBody != null) { httpDelete.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); } - Header[] authHeaders = - getHeaders(path, queryParams, "DELETE", encodedBody, restAuthFunction); + Header[] authHeaders = getHeaders(path, "DELETE", encodedBody, restAuthFunction); httpDelete.setHeaders(authHeaders); return exec(httpDelete, responseType); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 64b5aeab8873..44018af6c895 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -47,14 +47,17 @@ import org.apache.paimon.rest.requests.AuthTableQueryRequest; import org.apache.paimon.rest.requests.CommitTableRequest; import org.apache.paimon.rest.requests.CreateBranchRequest; +import org.apache.paimon.rest.requests.CreateDatabaseReferenceRequest; import org.apache.paimon.rest.requests.CreateDatabaseRequest; import org.apache.paimon.rest.requests.CreateFunctionRequest; import org.apache.paimon.rest.requests.CreatePartitionsRequest; import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; +import org.apache.paimon.rest.requests.DeleteDatabaseReferenceRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.rest.requests.FastForwardDatabaseBranchRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; @@ -75,6 +78,7 @@ import org.apache.paimon.rest.responses.CommitTableResponse; import org.apache.paimon.rest.responses.ConfigResponse; import org.apache.paimon.rest.responses.CreatePartitionsResponse; +import org.apache.paimon.rest.responses.DatabaseReferenceResponse; import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; @@ -110,7 +114,6 @@ import org.apache.paimon.rest.responses.ListViewsGloballyResponse; import org.apache.paimon.rest.responses.ListViewsResponse; import org.apache.paimon.rest.responses.PagedResponse; -import org.apache.paimon.rest.responses.SingleDatabaseReferenceResponse; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; @@ -196,10 +199,7 @@ public class RESTApi { public static final String PARTITION_NAME_PATTERN = "partitionNamePattern"; public static final String TAG_NAME_PREFIX = "tagNamePrefix"; - private static final String REFERENCE_NAME = "name"; private static final String REFERENCE_TYPE = "type"; - private static final String REFERENCE_UPDATE_MODE = "mode"; - private static final String FAST_FORWARD = "FAST_FORWARD"; public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; @@ -367,23 +367,6 @@ public void alterDatabase(String name, List removals, Map listDatabaseReferences( - String databaseName, @Nullable DatabaseReferenceType type) { - return listDataFromPageApi( - queryParams -> { - if (type != null) { - queryParams.put(REFERENCE_TYPE, type.queryValue()); - } - return client.get( - resourcePaths.databaseTrees(databaseName), - queryParams, - ListDatabaseReferencesResponse.class, - restAuthFunction); - }); - } - /** List one page of database-level branches and immutable tags. */ @Experimental public PagedList listDatabaseReferencesPaged( @@ -409,10 +392,10 @@ public PagedList listDatabaseReferencesPaged( /** Get one database-level branch or immutable tag. */ @Experimental public DatabaseReference getDatabaseReference(String databaseName, String referenceName) { - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.get( resourcePaths.databaseTree(databaseName, referenceName), - SingleDatabaseReferenceResponse.class, + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -424,15 +407,11 @@ public DatabaseReference createDatabaseReference( String referenceName, DatabaseReferenceType type, DatabaseReference source) { - Map queryParams = Maps.newHashMap(); - queryParams.put(REFERENCE_NAME, referenceName); - queryParams.put(REFERENCE_TYPE, type.queryValue()); - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.post( resourcePaths.databaseTrees(databaseName), - queryParams, - source, - SingleDatabaseReferenceResponse.class, + new CreateDatabaseReferenceRequest(referenceName, type, source), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -441,15 +420,11 @@ public DatabaseReference createDatabaseReference( @Experimental public DatabaseReference fastForwardDatabaseBranch( String databaseName, String targetBranch, String sourceTag) { - Map queryParams = Maps.newHashMap(); - queryParams.put(REFERENCE_UPDATE_MODE, FAST_FORWARD); - queryParams.put(REFERENCE_TYPE, DatabaseReferenceType.BRANCH.queryValue()); - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.put( resourcePaths.databaseTree(databaseName, targetBranch), - queryParams, - new DatabaseReference(DatabaseReferenceType.TAG, sourceTag), - SingleDatabaseReferenceResponse.class, + new FastForwardDatabaseBranchRequest(sourceTag), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } @@ -460,16 +435,11 @@ public DatabaseReference deleteDatabaseReference( String databaseName, String referenceName, @Nullable DatabaseReferenceType expectedType) { - Map queryParams = Maps.newHashMap(); - if (expectedType != null) { - queryParams.put(REFERENCE_TYPE, expectedType.queryValue()); - } - SingleDatabaseReferenceResponse response = + DatabaseReferenceResponse response = client.delete( resourcePaths.databaseTree(databaseName, referenceName), - queryParams, - null, - SingleDatabaseReferenceResponse.class, + new DeleteDatabaseReferenceRequest(expectedType), + DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java index 3629907dc65f..e7da1f6827dc 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTCatalogOptions.java @@ -98,14 +98,6 @@ public class RESTCatalogOptions { .withDescription( "The user agent of http client connecting to REST Catalog server."); - /** Database-level branch or immutable tag carried by table API requests. */ - public static final ConfigOption DATABASE_REFERENCE = - ConfigOptions.key("header.Paimon-Reference") - .stringType() - .noDefaultValue() - .withDescription( - "Database-level branch or immutable tag used by REST table requests."); - public static final ConfigOption DLF_OSS_ENDPOINT = ConfigOptions.key("dlf.oss-endpoint") .stringType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java new file mode 100644 index 000000000000..cb70e45dc471 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/CreateDatabaseReferenceRequest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for creating a database branch or immutable tag from an existing reference. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class CreateDatabaseReferenceRequest implements RESTRequest { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_TYPE = "type"; + private static final String FIELD_SOURCE = "source"; + + private final String name; + private final DatabaseReferenceType type; + private final DatabaseReference source; + + @JsonCreator + @ConstructorProperties({FIELD_NAME, FIELD_TYPE, FIELD_SOURCE}) + public CreateDatabaseReferenceRequest( + @JsonProperty(FIELD_NAME) String name, + @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, + @JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.name = name; + this.type = type; + this.source = source; + } + + @JsonGetter(FIELD_NAME) + public String getName() { + return name; + } + + @JsonGetter(FIELD_TYPE) + public DatabaseReferenceType getType() { + return type; + } + + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java new file mode 100644 index 000000000000..a7193b9b608e --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DeleteDatabaseReferenceRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +/** Request for deleting a database reference, optionally checking its type. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class DeleteDatabaseReferenceRequest implements RESTRequest { + + private static final String FIELD_TYPE = "type"; + + @Nullable private final DatabaseReferenceType type; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE}) + public DeleteDatabaseReferenceRequest( + @Nullable @JsonProperty(FIELD_TYPE) DatabaseReferenceType type) { + this.type = type; + } + + @Nullable + @JsonGetter(FIELD_TYPE) + @JsonInclude(JsonInclude.Include.NON_NULL) + public DatabaseReferenceType getType() { + return type; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java new file mode 100644 index 000000000000..f05b56d35458 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for fast-forwarding a database branch to an immutable tag. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class FastForwardDatabaseBranchRequest implements RESTRequest { + + private static final String FIELD_SOURCE_TAG = "sourceTag"; + + private final String sourceTag; + + @JsonCreator + @ConstructorProperties({FIELD_SOURCE_TAG}) + public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE_TAG) String sourceTag) { + this.sourceTag = sourceTag; + } + + @JsonGetter(FIELD_SOURCE_TAG) + public String getSourceTag() { + return sourceTag; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java similarity index 90% rename from paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java rename to paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java index 1bbda053890b..d8570559fdd7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/SingleDatabaseReferenceResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/DatabaseReferenceResponse.java @@ -30,7 +30,7 @@ /** Response containing one database-level reference. */ @Experimental -public class SingleDatabaseReferenceResponse implements RESTResponse { +public class DatabaseReferenceResponse implements RESTResponse { private static final String FIELD_REFERENCE = "reference"; @@ -39,8 +39,7 @@ public class SingleDatabaseReferenceResponse implements RESTResponse { @JsonCreator @ConstructorProperties({FIELD_REFERENCE}) - public SingleDatabaseReferenceResponse( - @JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { + public DatabaseReferenceResponse(@JsonProperty(FIELD_REFERENCE) DatabaseReference reference) { this.reference = reference; } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 2ccae1011bce..260d1140f4e0 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -42,7 +42,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; -import static org.apache.paimon.rest.RESTCatalogOptions.DATABASE_REFERENCE; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; import static org.apache.paimon.rest.RESTCatalogOptions.URI; @@ -131,10 +130,10 @@ void testBranchAndImmutableTagHappyPath() throws Exception { new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertThat(branch).isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertRequest(2, "POST", TREES_PATH); - assertThat(queryParameters(requests.get(2).query)) - .containsEntry("name", "exp-1") - .containsEntry("type", "branch"); - assertReferenceBody(requests.get(2), "BRANCH", "main"); + assertBody( + requests.get(2), + "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," + + "\"source\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); DatabaseReference tag = @@ -145,19 +144,16 @@ void testBranchAndImmutableTagHappyPath() throws Exception { new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertThat(tag).isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); assertRequest(3, "POST", TREES_PATH); - assertThat(queryParameters(requests.get(3).query)) - .containsEntry("name", "train-v1") - .containsEntry("type", "tag"); - assertReferenceBody(requests.get(3), "BRANCH", "exp-1"); + assertBody( + requests.get(3), + "{\"name\":\"train-v1\",\"type\":\"TAG\"," + + "\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertRequest(4, "PUT", TREES_PATH + "/main"); - assertThat(queryParameters(requests.get(4).query)) - .containsEntry("mode", "FAST_FORWARD") - .containsEntry("type", "branch"); - assertReferenceBody(requests.get(4), "TAG", "train-v1"); + assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); assertThat( @@ -165,21 +161,31 @@ void testBranchAndImmutableTagHappyPath() throws Exception { "training db", "exp-1", DatabaseReferenceType.BRANCH)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); assertRequest(5, "DELETE", TREES_PATH + "/exp-1"); - assertThat(queryParameters(requests.get(5).query)).containsEntry("type", "branch"); - assertThat(requests.get(5).body).isEmpty(); + assertBody(requests.get(5), "{\"type\":\"BRANCH\"}"); } @Test - void testListAllReferencesFollowsPages() { + void testListReferencesPaged() { enqueue( 200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}]," + "\"nextPageToken\":\"p2\"}"); enqueue(200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"exp-1\"}]}"); - assertThat(api.listDatabaseReferences("training db", DatabaseReferenceType.BRANCH)) - .extracting(DatabaseReference::getName) - .containsExactly("main", "exp-1"); + PagedList first = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.BRANCH, 1, null); + assertThat(first.getElements()) + .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + assertThat(first.getNextPageToken()).isEqualTo("p2"); + assertThat(requests).hasSize(1); + + PagedList second = + api.listDatabaseReferencesPaged( + "training db", DatabaseReferenceType.BRANCH, 1, first.getNextPageToken()); + assertThat(second.getElements()) + .containsExactly(new DatabaseReference(DatabaseReferenceType.BRANCH, "exp-1")); + assertThat(second.getNextPageToken()).isNull(); assertThat(requests).hasSize(2); assertThat(queryParameters(requests.get(0).query)).containsEntry("type", "branch"); assertThat(queryParameters(requests.get(1).query)) @@ -188,19 +194,12 @@ void testListAllReferencesFollowsPages() { } @Test - void testReferenceOptionIsSentToTableApis() { - enqueue(200, "{\"tables\":[]}"); - Options options = new Options(); - options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); - options.set(PREFIX, "catalog/id"); - options.set(TOKEN_PROVIDER, "bear"); - options.set(TOKEN, "test-token"); - options.set(DATABASE_REFERENCE, "train-v1"); - - new RESTApi(options, false).listTables("training db"); - - assertRequest(0, "GET", "/v1/catalog%2Fid/databases/training+db/tables"); - assertThat(requests.get(0).reference).isEqualTo("train-v1"); + void testDeleteReferenceWithoutType() throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); + assertThat(api.deleteDatabaseReference("training db", "train-v1", null)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + assertRequest(0, "DELETE", TREES_PATH + "/train-v1"); + assertBody(requests.get(0), "{}"); } private void enqueue(int code, String body) { @@ -214,12 +213,10 @@ private void assertRequest(int index, String method, String path) { assertThat(request.authorization).isEqualTo("Bearer test-token"); } - private static void assertReferenceBody(Request request, String type, String name) - throws Exception { + private static void assertBody(Request request, String expectedJson) throws Exception { + assertThat(request.query).isNull(); assertThat(RESTApi.fromJson(request.body, Map.class)) - .containsEntry("type", type) - .containsEntry("name", name) - .hasSize(2); + .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); } private static Map queryParameters(String query) { @@ -258,7 +255,6 @@ private static class Request { private final String query; private final String body; private final String authorization; - private final String reference; private Request(HttpExchange exchange) throws IOException { method = exchange.getRequestMethod(); @@ -266,7 +262,6 @@ private Request(HttpExchange exchange) throws IOException { query = exchange.getRequestURI().getRawQuery(); body = read(exchange.getRequestBody()); authorization = exchange.getRequestHeaders().getFirst("Authorization"); - reference = exchange.getRequestHeaders().getFirst("Paimon-Reference"); } private static String read(InputStream input) throws IOException { diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index e7e8a5517e94..7dec699b4367 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -18,6 +18,8 @@ package org.apache.paimon.rest.requests; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTRequest; @@ -113,6 +115,11 @@ public class RequestJacksonCompatibilityTest { }, "partitionSpecs", "ignoreIfNotExists"), + requestCase( + FastForwardDatabaseBranchRequest.class, + "{\"sourceTag\":\"train-v1\"}", + request -> assertThat(request.getSourceTag()).isEqualTo("train-v1"), + "sourceTag"), requestCase( ListPartitionsByFilterRequest.class, "{\"filter\":\"dt = '2026-08-24'\"," @@ -171,10 +178,12 @@ public class RequestJacksonCompatibilityTest { AlterTableRequest.class, AlterViewRequest.class, CommitTableRequest.class, + CreateDatabaseReferenceRequest.class, CreateFunctionRequest.class, CreatePartitionsRequest.class, CreateTableRequest.class, CreateViewRequest.class, + DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, GrantPermissionRequest.class, PolicyRequest.class, @@ -211,6 +220,39 @@ void testConstructorPropertyNamesAndOrder(RequestCase requestCase) { .isEqualTo(requestCase.propertyNames); } + @Test + void testCreateDatabaseReferenceRequestRoundTrips() throws Exception { + String json = + "{\"name\":\"exp-1\",\"type\":\"BRANCH\"," + + "\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"; + CreateDatabaseReferenceRequest request = + EXTERNAL_MAPPER.readValue(json, CreateDatabaseReferenceRequest.class); + CreateDatabaseReferenceRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), CreateDatabaseReferenceRequest.class); + assertThat(roundTrip.getName()).isEqualTo("exp-1"); + assertThat(roundTrip.getType()).isEqualTo(DatabaseReferenceType.BRANCH); + assertThat(roundTrip.getSource()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train-v1")); + } + + @Test + void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { + DeleteDatabaseReferenceRequest request = + EXTERNAL_MAPPER.readValue( + "{\"type\":\"TAG\"}", DeleteDatabaseReferenceRequest.class); + assertThat( + RESTApi.fromJson( + RESTApi.toJson(request), + DeleteDatabaseReferenceRequest.class) + .getType()) + .isEqualTo(DatabaseReferenceType.TAG); + + DeleteDatabaseReferenceRequest withoutType = + EXTERNAL_MAPPER.readValue("{}", DeleteDatabaseReferenceRequest.class); + assertThat(RESTApi.toJson(withoutType)).isEqualTo("{}"); + assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index adb307836e76..b3464f9961e4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -105,39 +105,35 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.createReference(DATABASE, "exp-1", BRANCH, main)).isEqualTo(branch); RecordedRequest createBranch = takeRequest("POST", TREES_PATH); - assertThat(createBranch.getRequestUrl().queryParameter("name")).isEqualTo("exp-1"); - assertThat(createBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertReferenceBody(createBranch, "BRANCH", "main"); + assertBody( + createBranch, + "{\"name\":\"exp-1\",\"type\":\"BRANCH\",\"source\":" + MAIN_JSON + "}"); enqueue(200, "{\"reference\":" + TAG_JSON + "}"); assertThat(trees.createReference(DATABASE, "train-v1", TAG, branch)).isEqualTo(tag); RecordedRequest createTag = takeRequest("POST", TREES_PATH); - assertThat(createTag.getRequestUrl().queryParameter("name")).isEqualTo("train-v1"); - assertThat(createTag.getRequestUrl().queryParameter("type")).isEqualTo("tag"); - assertReferenceBody(createTag, "BRANCH", "exp-1"); + assertBody( + createTag, + "{\"name\":\"train-v1\",\"type\":\"TAG\",\"source\":" + BRANCH_JSON + "}"); enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); - assertThat(fastForward.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertThat(fastForward.getRequestUrl().queryParameter("mode")).isEqualTo("FAST_FORWARD"); - assertReferenceBody(fastForward, "TAG", "train-v1"); + assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); RecordedRequest deleteBranch = takeRequest("DELETE", TREES_PATH + "/exp-1"); - assertThat(deleteBranch.getRequestUrl().queryParameter("type")).isEqualTo("branch"); - assertThat(deleteBranch.getBodySize()).isZero(); + assertBody(deleteBranch, "{\"type\":\"BRANCH\"}"); enqueue(200, "{\"reference\":" + TAG_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "train-v1", null)).isEqualTo(tag); - assertThat(takeRequest("DELETE", TREES_PATH + "/train-v1").getRequestUrl().query()) - .isNull(); + assertBody(takeRequest("DELETE", TREES_PATH + "/train-v1"), "{}"); assertThat(server.getRequestCount()).isEqualTo(7); } @Test - void testListPageAndListAllPreserveFilterAndTokens() throws Exception { + void testListPagesPreserveFilterAndTokens() throws Exception { enqueue(200, "{\"references\":[" + TAG_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); PagedList page = trees.listReferencesPaged(DATABASE, TAG, 10, "start +/%"); @@ -150,13 +146,18 @@ void testListPageAndListAllPreserveFilterAndTokens() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "],\"nextPageToken\":\"next +/%?&\"}"); enqueue(200, "{\"references\":[" + BRANCH_JSON + "]}"); - assertThat(trees.listReferences(DATABASE, BRANCH)) - .containsExactly( - new DatabaseReference(BRANCH, "main"), - new DatabaseReference(BRANCH, "exp-1")); + PagedList firstPage = + trees.listReferencesPaged(DATABASE, BRANCH, null, null); + assertThat(firstPage.getElements()).containsExactly(new DatabaseReference(BRANCH, "main")); + assertThat(firstPage.getNextPageToken()).isEqualTo("next +/%?&"); RecordedRequest first = takeRequest("GET", TREES_PATH); assertThat(first.getRequestUrl().queryParameter("type")).isEqualTo("branch"); assertThat(first.getRequestUrl().queryParameter("pageToken")).isNull(); + PagedList secondPage = + trees.listReferencesPaged(DATABASE, BRANCH, null, firstPage.getNextPageToken()); + assertThat(secondPage.getElements()) + .containsExactly(new DatabaseReference(BRANCH, "exp-1")); + assertThat(secondPage.getNextPageToken()).isNull(); RecordedRequest second = takeRequest("GET", TREES_PATH); assertThat(second.getRequestUrl().queryParameter("type")).isEqualTo("branch"); assertThat(second.getRequestUrl().queryParameter("pageToken")).isEqualTo("next +/%?&"); @@ -166,14 +167,17 @@ void testListPageAndListAllPreserveFilterAndTokens() throws Exception { @Test void testListAllTypesAndEmptyReferences() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); - assertThat(trees.listReferences(DATABASE, null)) + assertThat(trees.listReferencesPaged(DATABASE, null, null, null).getElements()) .containsExactly( new DatabaseReference(BRANCH, "main"), new DatabaseReference(TAG, "train-v1")); assertThat(takeRequest("GET", TREES_PATH).getRequestUrl().query()).isNull(); enqueue(200, "{\"references\":[]}"); - assertThat(trees.listReferences(DATABASE, null)).isEmpty(); + PagedList emptyPage = + trees.listReferencesPaged(DATABASE, null, null, null); + assertThat(emptyPage.getElements()).isEmpty(); + assertThat(emptyPage.getNextPageToken()).isNull(); takeRequest("GET", TREES_PATH); assertThat(server.getRequestCount()).isEqualTo(3); } @@ -197,7 +201,7 @@ void testErrorsPropagate() { .hasMessageContaining("reference already exists"); enqueue(501, "{\"code\":501,\"message\":\"trees unsupported\"}"); - assertThatThrownBy(() -> trees.listReferences(DATABASE, null)) + assertThatThrownBy(() -> trees.listReferencesPaged(DATABASE, null, null, null)) .isInstanceOf(NotImplementedException.class) .hasMessageContaining("trees unsupported"); assertThat(server.getRequestCount()).isEqualTo(4); @@ -221,11 +225,9 @@ private RecordedRequest takeRequest(String method, String path) throws Exception return request; } - private static void assertReferenceBody(RecordedRequest request, String type, String name) - throws Exception { + private static void assertBody(RecordedRequest request, String expectedJson) throws Exception { + assertThat(request.getRequestUrl().query()).isNull(); assertThat(RESTApi.fromJson(request.getBody().readUtf8(), Map.class)) - .containsEntry("type", type) - .containsEntry("name", name) - .hasSize(2); + .isEqualTo(RESTApi.fromJson(expectedJson, Map.class)); } } From de4bf3b049aeb36f743c528f1d0c890bff656c2d Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 16:59:35 +0800 Subject: [PATCH 04/10] [rest] Use a dedicated forward path for database branches --- docs/static/rest-catalog-open-api.yaml | 33 ++++++++++--------- .../org/apache/paimon/rest/HttpClient.java | 21 ------------ .../apache/paimon/rest/HttpClientUtils.java | 5 --- .../java/org/apache/paimon/rest/RESTApi.java | 4 +-- .../org/apache/paimon/rest/ResourcePaths.java | 6 ++++ .../rest/RESTApiDatabaseReferenceTest.java | 2 +- .../rest/RESTCatalogTreeManagementTest.java | 2 +- 7 files changed, 27 insertions(+), 46 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 98c1350d0b52..ff452c9147f1 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -330,11 +330,11 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - put: + delete: tags: - database-reference - summary: Fast-forward database branch from immutable tag - operationId: fastForwardDatabaseBranch + summary: Delete database reference + operationId: deleteDatabaseReference parameters: - name: prefix in: path @@ -352,14 +352,14 @@ paths: schema: type: string requestBody: - required: true + required: false content: application/json: schema: - $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' + $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' responses: "200": - description: Updated target branch. + description: Deleted branch or immutable tag. content: application/json: schema: @@ -367,24 +367,25 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database, target branch, or source tag does not exist. + description: Database or reference does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not an ancestor of the source tag. + description: Reference type does not match or the default branch is protected. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - delete: + /v1/{prefix}/databases/{database}/trees/{name}/forward: + post: tags: - database-reference - summary: Delete database reference - operationId: deleteDatabaseReference + summary: Fast-forward database branch from immutable tag + operationId: fastForwardDatabaseBranch parameters: - name: prefix in: path @@ -402,14 +403,14 @@ paths: schema: type: string requestBody: - required: false + required: true content: application/json: schema: - $ref: '#/components/schemas/DeleteDatabaseReferenceRequest' + $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' responses: "200": - description: Deleted branch or immutable tag. + description: Updated target branch. content: application/json: schema: @@ -417,13 +418,13 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database or reference does not exist. + description: Database, target branch, or source tag does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Reference type does not match or the default branch is protected. + description: Target is not an ancestor of the source tag. content: application/json: schema: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java index 57f44a93fa03..925809504a56 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClient.java @@ -32,7 +32,6 @@ import org.apache.hc.client5.http.classic.methods.HttpDelete; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.ContentType; @@ -114,26 +113,6 @@ public T post( : null); } - public T put( - String path, - RESTRequest body, - Class responseType, - RESTAuthFunction restAuthFunction) { - HttpPut httpPut = HttpClientUtils.newHttpPut(getRequestUrl(path, null)); - String encodedBody = RESTUtil.encodedBody(body); - if (encodedBody != null) { - httpPut.setEntity(new StringEntity(encodedBody, ContentType.APPLICATION_JSON)); - } - Header[] authHeaders = getHeaders(path, "PUT", encodedBody, restAuthFunction); - httpPut.setHeaders(authHeaders); - return exec( - httpPut, - responseType, - body != null && !body.isRetrySafe() - ? ExponentialHttpRequestRetryStrategy.retryUnsafeContext() - : null); - } - @Override public T delete(String path, RESTAuthFunction restAuthFunction) { return delete(path, null, null, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java index 94d0ba80f723..1444c8118d14 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java @@ -26,7 +26,6 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.classic.methods.HttpHead; import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.classic.methods.HttpPut; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.entity.DecompressingEntity; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -239,10 +238,6 @@ public static HttpPost newHttpPost(String uri) { return newRequest(uri, HttpPost::new); } - public static HttpPut newHttpPut(String uri) { - return newRequest(uri, HttpPut::new); - } - public static HttpDelete newHttpDelete(String uri) { return newRequest(uri, HttpDelete::new); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 44018af6c895..435e70adcb9a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -421,8 +421,8 @@ public DatabaseReference createDatabaseReference( public DatabaseReference fastForwardDatabaseBranch( String databaseName, String targetBranch, String sourceTag) { DatabaseReferenceResponse response = - client.put( - resourcePaths.databaseTree(databaseName, targetBranch), + client.post( + resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), new FastForwardDatabaseBranchRequest(sourceTag), DatabaseReferenceResponse.class, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 862f78860200..636ccea026d8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -157,6 +157,12 @@ public String databaseTree(String databaseName, String referenceName) { return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); } + /** Action endpoint for fast-forwarding a database-level branch. */ + @Experimental + public String forwardDatabaseBranch(String databaseName, String branch) { + return SLASH.join(databaseTree(databaseName, branch), "forward"); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 260d1140f4e0..a5ffa5102ca1 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -152,7 +152,7 @@ void testBranchAndImmutableTagHappyPath() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(4, "PUT", TREES_PATH + "/main"); + assertRequest(4, "POST", TREES_PATH + "/main/forward"); assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index b3464f9961e4..0d23747be2df 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -118,7 +118,7 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); - RecordedRequest fastForward = takeRequest("PUT", TREES_PATH + "/main"); + RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); From cad64b8a31877d440a6a0e84151565c69f445f76 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:10:58 +0800 Subject: [PATCH 05/10] [rest] Allow database forward from branches and tags --- docs/static/rest-catalog-open-api.yaml | 18 ++++++++++-------- .../paimon/management/TreeManagement.java | 5 +++-- .../java/org/apache/paimon/rest/RESTApi.java | 6 +++--- .../paimon/rest/RESTTreeManagement.java | 4 ++-- .../FastForwardDatabaseBranchRequest.java | 19 ++++++++++--------- .../rest/RESTApiDatabaseReferenceTest.java | 16 ++++++++++++---- .../RequestJacksonCompatibilityTest.java | 19 ++++++++++++++----- .../rest/RESTCatalogTreeManagementTest.java | 15 +++++++++++---- 8 files changed, 65 insertions(+), 37 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index ff452c9147f1..4f443f7aeae4 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -384,7 +384,11 @@ paths: post: tags: - database-reference - summary: Fast-forward database branch from immutable tag + summary: Fast-forward database branch from a branch or tag + description: >- + The target must be a branch. The source branch or tag is resolved in the same database + when the request is processed. Fast-forward succeeds only if the target is an ancestor + of the source; divergent histories return a conflict. operationId: fastForwardDatabaseBranch parameters: - name: prefix @@ -418,13 +422,13 @@ paths: "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - description: Database, target branch, or source tag does not exist. + description: Database, target branch, or source reference does not exist. content: application/json: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not an ancestor of the source tag. + description: Target is not a branch or is not an ancestor of the source reference. content: application/json: schema: @@ -4011,12 +4015,10 @@ components: FastForwardDatabaseBranchRequest: type: object required: - - sourceTag + - source properties: - sourceTag: - type: string - description: Immutable tag in the same database to fast-forward the target branch to. - pattern: "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" + source: + $ref: '#/components/schemas/DatabaseReference' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 4ca5dd9a26e7..cbecace2a1cd 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -52,8 +52,9 @@ DatabaseReference createReference( DatabaseReferenceType type, DatabaseReference source); - /** Fast-forwards a branch to an immutable tag in the same database. */ - DatabaseReference fastForwardBranch(String databaseName, String targetBranch, String sourceTag); + /** Fast-forwards a branch to another branch or immutable tag in the same database. */ + DatabaseReference fastForwardBranch( + String databaseName, String targetBranch, DatabaseReference source); /** * Deletes and returns a named reference. A missing reference is an error. diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 435e70adcb9a..f96e777ce985 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -416,14 +416,14 @@ public DatabaseReference createDatabaseReference( return checkNotNull(response.getReference(), "Reference response must contain reference"); } - /** Fast-forward a database-level branch to an immutable tag. */ + /** Fast-forward a database-level branch to another branch or immutable tag. */ @Experimental public DatabaseReference fastForwardDatabaseBranch( - String databaseName, String targetBranch, String sourceTag) { + String databaseName, String targetBranch, DatabaseReference source) { DatabaseReferenceResponse response = client.post( resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), - new FastForwardDatabaseBranchRequest(sourceTag), + new FastForwardDatabaseBranchRequest(source), DatabaseReferenceResponse.class, restAuthFunction); return checkNotNull(response.getReference(), "Reference response must contain reference"); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index 2be049fb8e02..c1ed8c34dc2e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -59,8 +59,8 @@ public DatabaseReference createReference( @Override public DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, String sourceTag) { - return api.fastForwardDatabaseBranch(databaseName, targetBranch, sourceTag); + String databaseName, String targetBranch, DatabaseReference source) { + return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); } @Override diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java index f05b56d35458..d35f98afc16c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.requests; import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -28,23 +29,23 @@ import java.beans.ConstructorProperties; -/** Request for fast-forwarding a database branch to an immutable tag. */ +/** Request for fast-forwarding a database branch to another branch or immutable tag. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) public class FastForwardDatabaseBranchRequest implements RESTRequest { - private static final String FIELD_SOURCE_TAG = "sourceTag"; + private static final String FIELD_SOURCE = "source"; - private final String sourceTag; + private final DatabaseReference source; @JsonCreator - @ConstructorProperties({FIELD_SOURCE_TAG}) - public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE_TAG) String sourceTag) { - this.sourceTag = sourceTag; + @ConstructorProperties({FIELD_SOURCE}) + public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.source = source; } - @JsonGetter(FIELD_SOURCE_TAG) - public String getSourceTag() { - return sourceTag; + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index a5ffa5102ca1..0535965eac0d 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -26,6 +26,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -95,8 +97,9 @@ void tearDown() { } } - @Test - void testBranchAndImmutableTagHappyPath() throws Exception { + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testBranchAndImmutableTagHappyPath(DatabaseReferenceType sourceType) throws Exception { enqueue( 200, "{\"references\":[{\"type\":\"BRANCH\",\"name\":\"main\"}," @@ -150,10 +153,15 @@ void testBranchAndImmutableTagHappyPath() throws Exception { + "\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); - assertThat(api.fastForwardDatabaseBranch("training db", "main", "train-v1")) + DatabaseReference source = sourceType == DatabaseReferenceType.BRANCH ? branch : tag; + assertThat(api.fastForwardDatabaseBranch("training db", "main", source)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); assertRequest(4, "POST", TREES_PATH + "/main/forward"); - assertBody(requests.get(4), "{\"sourceTag\":\"train-v1\"}"); + assertBody( + requests.get(4), + sourceType == DatabaseReferenceType.BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"exp-1\"}}"); assertThat( diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 7dec699b4367..576cce333bd8 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import java.beans.ConstructorProperties; @@ -115,11 +116,6 @@ public class RequestJacksonCompatibilityTest { }, "partitionSpecs", "ignoreIfNotExists"), - requestCase( - FastForwardDatabaseBranchRequest.class, - "{\"sourceTag\":\"train-v1\"}", - request -> assertThat(request.getSourceTag()).isEqualTo("train-v1"), - "sourceTag"), requestCase( ListPartitionsByFilterRequest.class, "{\"filter\":\"dt = '2026-08-24'\"," @@ -185,6 +181,7 @@ public class RequestJacksonCompatibilityTest { CreateViewRequest.class, DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, + FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, PolicyRequest.class, RegisterTableRequest.class, @@ -253,6 +250,18 @@ void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) + throws Exception { + String json = "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"training\"}}"; + FastForwardDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, FastForwardDatabaseBranchRequest.class); + FastForwardDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), FastForwardDatabaseBranchRequest.class); + assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 0d23747be2df..4425e1352713 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -32,6 +32,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -92,8 +94,10 @@ void tearDown() throws Exception { } } - @Test - void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testBranchAndTagOperationsUseCatalogConfiguration(DatabaseReferenceType sourceType) + throws Exception { DatabaseReference main = new DatabaseReference(BRANCH, "main"); DatabaseReference branch = new DatabaseReference(BRANCH, "exp-1"); DatabaseReference tag = new DatabaseReference(TAG, "train-v1"); @@ -117,9 +121,12 @@ void testBranchAndTagOperationsUseCatalogConfiguration() throws Exception { "{\"name\":\"train-v1\",\"type\":\"TAG\",\"source\":" + BRANCH_JSON + "}"); enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); - assertThat(trees.fastForwardBranch(DATABASE, "main", "train-v1")).isEqualTo(main); + DatabaseReference source = sourceType == BRANCH ? branch : tag; + assertThat(trees.fastForwardBranch(DATABASE, "main", source)).isEqualTo(main); RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); - assertBody(fastForward, "{\"sourceTag\":\"train-v1\"}"); + assertBody( + fastForward, + "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); From cb0b0c244708a9886c28e7a7e771a5f0c7543ee6 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:21:11 +0800 Subject: [PATCH 06/10] [rest] Add database branch merge API --- docs/static/rest-catalog-open-api.yaml | 67 +++++++++++++++++++ .../paimon/management/TreeManagement.java | 10 +++ .../java/org/apache/paimon/rest/RESTApi.java | 14 ++++ .../paimon/rest/RESTTreeManagement.java | 6 ++ .../org/apache/paimon/rest/ResourcePaths.java | 6 ++ .../requests/MergeDatabaseBranchRequest.java | 51 ++++++++++++++ .../rest/RESTApiDatabaseReferenceTest.java | 18 +++++ .../RequestJacksonCompatibilityTest.java | 15 +++++ .../rest/RESTCatalogTreeManagementTest.java | 49 ++++++++++++++ 9 files changed, 236 insertions(+) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 4f443f7aeae4..d2086de62596 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -435,6 +435,66 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{name}/merge: + post: + tags: + - database-reference + summary: Merge a branch or tag into a database branch + description: >- + The target must be a branch. The source branch or tag is resolved in the same database + when the request is processed. Merge compares table entries with their common ancestor, + including table creation and deletion. Changes made only on one side are preserved; + identical changes on both sides are accepted. Different changes to the same table entry + conflict; table row data is not merged. The server checks for conflicts before publishing + the result. A conflict leaves the target unchanged, and the source reference is never + modified. Fast-forward and no-op merges succeed. The server must retain ancestry and + merge relationships to resolve subsequent merges. + operationId: mergeDatabaseBranch + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + - name: name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/MergeDatabaseBranchRequest' + responses: + "200": + description: Target branch after the merge, including when no changes were needed. + content: + application/json: + schema: + $ref: '#/components/schemas/DatabaseReferenceResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + description: Database, target branch, or source reference does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "409": + description: Target is not a branch, no common ancestor is available, or table entries conflict. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/register: post: tags: @@ -4019,6 +4079,13 @@ components: properties: source: $ref: '#/components/schemas/DatabaseReference' + MergeDatabaseBranchRequest: + type: object + required: + - source + properties: + source: + $ref: '#/components/schemas/DatabaseReference' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index cbecace2a1cd..920dbec16245 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -56,6 +56,16 @@ DatabaseReference createReference( DatabaseReference fastForwardBranch( String databaseName, String targetBranch, DatabaseReference source); + /** + * Merges a branch or immutable tag into a target branch in the same database. + * + *

Table entries are merged relative to a common ancestor. Conflicting changes fail the merge + * without modifying the target; the source reference is never modified. A merge with no changes + * succeeds. + */ + DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source); + /** * Deletes and returns a named reference. A missing reference is an error. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index f96e777ce985..ea4ffb2f65f0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -63,6 +63,7 @@ import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; +import org.apache.paimon.rest.requests.MergeDatabaseBranchRequest; import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RegisterTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; @@ -429,6 +430,19 @@ public DatabaseReference fastForwardDatabaseBranch( return checkNotNull(response.getReference(), "Reference response must contain reference"); } + /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ + @Experimental + public DatabaseReference mergeDatabaseBranch( + String databaseName, String targetBranch, DatabaseReference source) { + DatabaseReferenceResponse response = + client.post( + resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), + new MergeDatabaseBranchRequest(source), + DatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull(response.getReference(), "Reference response must contain reference"); + } + /** Delete one database-level branch or immutable tag. */ @Experimental public DatabaseReference deleteDatabaseReference( diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index c1ed8c34dc2e..87e84b223775 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -63,6 +63,12 @@ public DatabaseReference fastForwardBranch( return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); } + @Override + public DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source) { + return api.mergeDatabaseBranch(databaseName, targetBranch, source); + } + @Override public DatabaseReference deleteReference( String databaseName, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 636ccea026d8..01422fdb7e1f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -163,6 +163,12 @@ public String forwardDatabaseBranch(String databaseName, String branch) { return SLASH.join(databaseTree(databaseName, branch), "forward"); } + /** Action endpoint for merging a branch or tag into a database-level branch. */ + @Experimental + public String mergeDatabaseBranch(String databaseName, String branch) { + return SLASH.join(databaseTree(databaseName, branch), "merge"); + } + public String tables(String databaseName) { return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java new file mode 100644 index 000000000000..34c11f261b9f --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Request for merging a branch or immutable tag into a database branch. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class MergeDatabaseBranchRequest implements RESTRequest { + + private static final String FIELD_SOURCE = "source"; + + private final DatabaseReference source; + + @JsonCreator + @ConstructorProperties({FIELD_SOURCE}) + public MergeDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + this.source = source; + } + + @JsonGetter(FIELD_SOURCE) + public DatabaseReference getSource() { + return source; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 0535965eac0d..91b838320857 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -201,6 +201,24 @@ void testListReferencesPaged() { .containsEntry("pageToken", "p2"); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeBranchOrTag(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat(api.mergeDatabaseBranch("training db", "main", source)) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + + assertRequest(0, "POST", TREES_PATH + "/main/merge"); + assertBody( + requests.get(0), + sourceType == DatabaseReferenceType.BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); + assertThat(requests).hasSize(1); + } + @Test void testDeleteReferenceWithoutType() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 576cce333bd8..952ef5ac388b 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -183,6 +183,7 @@ public class RequestJacksonCompatibilityTest { DropPolicyRequest.class, FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, + MergeDatabaseBranchRequest.class, PolicyRequest.class, RegisterTableRequest.class, RenameTableRequest.class, @@ -262,6 +263,20 @@ void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType source assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) + throws Exception { + String json = + "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"experiment\"}}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getSource()) + .isEqualTo(new DatabaseReference(sourceType, "experiment")); + } + @Test void testRequestCreatorAllowlistsAreComplete() throws Exception { Set> simpleRequests = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 4425e1352713..20868aadb3c0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -171,6 +171,55 @@ void testListPagesPreserveFilterAndTokens() throws Exception { assertThat(second.getRequestUrl().queryParameter("maxResults")).isNull(); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeUsesCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat(trees.mergeBranch(DATABASE, "main", source)) + .isEqualTo(new DatabaseReference(BRANCH, "main")); + + RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); + assertBody( + merge, + sourceType == BRANCH + ? "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}}" + : "{\"source\":{\"type\":\"TAG\",\"name\":\"experiment\"}}"); + assertThat(server.getRequestCount()).isEqualTo(2); + } + + @Test + void testMergeErrorsPreserveDetails() throws Exception { + DatabaseReference source = new DatabaseReference(BRANCH, "experiment"); + enqueue( + 409, + "{\"code\":409,\"message\":\"Conflicting changes to table features\"," + + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOfSatisfying( + AlreadyExistsException.class, + conflict -> { + assertThat(conflict.resourceType()).isEqualTo("TABLE"); + assertThat(conflict.resourceName()).isEqualTo("training db.features"); + }) + .hasMessageContaining("Conflicting changes to table features"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(404, "{\"code\":404,\"message\":\"source reference missing\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("source reference missing"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(501, "{\"code\":501,\"message\":\"merge unsupported\"}"); + assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) + .isInstanceOf(NotImplementedException.class) + .hasMessageContaining("merge unsupported"); + takeRequest("POST", TREES_PATH + "/main/merge"); + assertThat(server.getRequestCount()).isEqualTo(4); + } + @Test void testListAllTypesAndEmptyReferences() throws Exception { enqueue(200, "{\"references\":[" + MAIN_JSON + "," + TAG_JSON + "]}"); From 20366895e3cb2b771b2170760149f1f73641a17b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 17:43:39 +0800 Subject: [PATCH 07/10] [rest] Unify database merges with per-table merge modes --- docs/static/rest-catalog-open-api.yaml | 115 ++++++++---------- .../paimon/management/TreeManagement.java | 33 ++++- .../org/apache/paimon/rest/MergeMode.java | 34 ++++++ .../java/org/apache/paimon/rest/RESTApi.java | 43 ++++--- .../paimon/rest/RESTTreeManagement.java | 17 +-- .../org/apache/paimon/rest/ResourcePaths.java | 6 - ...BranchRequest.java => TableMergeMode.java} | 34 ++++-- .../exceptions/MergeConflictException.java | 48 ++++++++ .../requests/MergeDatabaseBranchRequest.java | 44 ++++++- .../rest/RESTApiDatabaseReferenceTest.java | 37 +++++- .../RequestJacksonCompatibilityTest.java | 74 +++++++++-- .../rest/RESTCatalogTreeManagementTest.java | 80 ++++++++++-- 12 files changed, 417 insertions(+), 148 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java rename paimon-api/src/main/java/org/apache/paimon/rest/{requests/FastForwardDatabaseBranchRequest.java => TableMergeMode.java} (62%) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index d2086de62596..0d9c1b85bcfb 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -380,61 +380,6 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{name}/forward: - post: - tags: - - database-reference - summary: Fast-forward database branch from a branch or tag - description: >- - The target must be a branch. The source branch or tag is resolved in the same database - when the request is processed. Fast-forward succeeds only if the target is an ancestor - of the source; divergent histories return a conflict. - operationId: fastForwardDatabaseBranch - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: name - in: path - required: true - schema: - type: string - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/FastForwardDatabaseBranchRequest' - responses: - "200": - description: Updated target branch. - content: - application/json: - schema: - $ref: '#/components/schemas/DatabaseReferenceResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - description: Database, target branch, or source reference does not exist. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - "409": - description: Target is not a branch or is not an ancestor of the source reference. - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' /v1/{prefix}/databases/{database}/trees/{name}/merge: post: tags: @@ -442,13 +387,23 @@ paths: summary: Merge a branch or tag into a database branch description: >- The target must be a branch. The source branch or tag is resolved in the same database - when the request is processed. Merge compares table entries with their common ancestor, - including table creation and deletion. Changes made only on one side are preserved; - identical changes on both sides are accepted. Different changes to the same table entry - conflict; table row data is not merged. The server checks for conflicts before publishing - the result. A conflict leaves the target unchanged, and the source reference is never - modified. Fast-forward and no-op merges succeed. The server must retain ancestry and - merge relationships to resolve subsequent merges. + when the request is processed. Merge compares complete table versions with their merge + base, including schemas, properties, snapshots, and table creation or deletion; table row + data is not merged. Changes made only on the target are preserved. Source-side changes + use defaultMergeMode (NORMAL when omitted), overridden by tableMergeModes for individual + table names. NORMAL accepts one-sided or identical changes and rejects different changes + to the same table. FORCE accepts the source-side change even on conflict, including a + deletion. DROP skips all source-side changes to that table, even without a conflict, and + preserves its target state. Modes do not replace target-only changes with unchanged source + versions. The server checks for unresolved conflicts before publishing the result; a + conflict leaves the target unchanged, and the source reference is never modified. + Identical references or an already-merged source succeed without modifying the target. + When the target is an ancestor of the source, the server may fast-forward only if the + selected modes produce exactly the source state. Divergent histories use three-way merge; + no available merge base is a conflict. A successful merge must record the source as merged, + including changes skipped by DROP, even when the table contents remain unchanged. Repeating + a merge of the same source state does not reapply skipped changes. The server retains + ancestry and merge relationships to resolve subsequent merges. operationId: mergeDatabaseBranch parameters: - name: prefix @@ -479,6 +434,8 @@ paths: application/json: schema: $ref: '#/components/schemas/DatabaseReferenceResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' "401": $ref: '#/components/responses/UnauthorizedErrorResponse' "404": @@ -488,7 +445,7 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' "409": - description: Target is not a branch, no common ancestor is available, or table entries conflict. + description: Target is not a branch, no merge base is available, or table conflicts remain unresolved. content: application/json: schema: @@ -4072,13 +4029,24 @@ components: enum: [ "BRANCH", "TAG" ] source: $ref: '#/components/schemas/DatabaseReference' - FastForwardDatabaseBranchRequest: + MergeMode: + type: string + enum: [ "NORMAL", "FORCE", "DROP" ] + description: >- + NORMAL performs three-way conflict detection. FORCE accepts source-side table changes + even on conflict. DROP skips all source-side changes to the table and preserves its + target state. Each mode operates on complete table versions, not individual rows. + TableMergeMode: type: object required: - - source + - table + - mergeMode properties: - source: - $ref: '#/components/schemas/DatabaseReference' + table: + type: string + description: Exact table name within the database being merged. + mergeMode: + $ref: '#/components/schemas/MergeMode' MergeDatabaseBranchRequest: type: object required: @@ -4086,6 +4054,19 @@ components: properties: source: $ref: '#/components/schemas/DatabaseReference' + defaultMergeMode: + description: Mode for tables without a per-table override. Defaults to NORMAL. + default: NORMAL + allOf: + - $ref: '#/components/schemas/MergeMode' + tableMergeModes: + type: array + description: >- + Per-table modes override defaultMergeMode. Omit or use an empty array to apply the + default to every table. Each table name may appear at most once; duplicates are a + bad request. Names without source-side changes have no effect. + items: + $ref: '#/components/schemas/TableMergeMode' DeleteDatabaseReferenceRequest: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java index 920dbec16245..8f864017676a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/management/TreeManagement.java @@ -22,9 +22,13 @@ import org.apache.paimon.annotation.Experimental; import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; +import org.apache.paimon.rest.TableMergeMode; import javax.annotation.Nullable; +import java.util.List; + /** Control-plane contract for database-level writable branches and immutable tags. */ @Experimental public interface TreeManagement { @@ -52,19 +56,36 @@ DatabaseReference createReference( DatabaseReferenceType type, DatabaseReference source); - /** Fast-forwards a branch to another branch or immutable tag in the same database. */ - DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, DatabaseReference source); - /** * Merges a branch or immutable tag into a target branch in the same database. * *

Table entries are merged relative to a common ancestor. Conflicting changes fail the merge * without modifying the target; the source reference is never modified. A merge with no changes - * succeeds. + * succeeds. The server automatically fast-forwards when possible. + */ + default DatabaseReference mergeBranch( + String databaseName, String targetBranch, DatabaseReference source) { + return mergeBranch(databaseName, targetBranch, source, null, null); + } + + /** + * Merges a branch or immutable tag using default and per-table merge modes. + * + *

Modes apply to source-side changes to complete table versions, including creation and + * deletion; table row data is not merged. Per-table modes override the default. Unresolved + * conflicts leave the target unchanged, and the source is never modified. A successful merge + * records the source as merged, including changes skipped by {@link MergeMode#DROP}. + * + * @param defaultMergeMode mode for tables without an override; null means {@link + * MergeMode#NORMAL} + * @param tableMergeModes per-table overrides; null or empty uses the default for every table */ DatabaseReference mergeBranch( - String databaseName, String targetBranch, DatabaseReference source); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes); /** * Deletes and returns a named reference. A missing reference is an error. diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java b/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java new file mode 100644 index 000000000000..8b18529e9a30 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/MergeMode.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; + +/** How source-side table changes are handled when merging database references. */ +@Experimental +public enum MergeMode { + /** Merge changes relative to the common ancestor, failing on conflicting table versions. */ + NORMAL, + + /** Accept source-side changes even when they conflict with the target table version. */ + FORCE, + + /** Skip all source-side changes to the table, preserving its target state. */ + DROP +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index ea4ffb2f65f0..decefe3e3712 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -39,6 +39,7 @@ import org.apache.paimon.rest.auth.RESTAuthFunction; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.exceptions.ForbiddenException; +import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.requests.AlterDatabaseRequest; import org.apache.paimon.rest.requests.AlterFunctionRequest; @@ -57,7 +58,6 @@ import org.apache.paimon.rest.requests.DeleteDatabaseReferenceRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; import org.apache.paimon.rest.requests.DropPolicyRequest; -import org.apache.paimon.rest.requests.FastForwardDatabaseBranchRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; @@ -417,30 +417,35 @@ public DatabaseReference createDatabaseReference( return checkNotNull(response.getReference(), "Reference response must contain reference"); } - /** Fast-forward a database-level branch to another branch or immutable tag. */ + /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ @Experimental - public DatabaseReference fastForwardDatabaseBranch( + public DatabaseReference mergeDatabaseBranch( String databaseName, String targetBranch, DatabaseReference source) { - DatabaseReferenceResponse response = - client.post( - resourcePaths.forwardDatabaseBranch(databaseName, targetBranch), - new FastForwardDatabaseBranchRequest(source), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + return mergeDatabaseBranch(databaseName, targetBranch, source, null, null); } - /** Merge a branch or immutable tag into a database-level branch, failing on conflicts. */ + /** Merge a branch or immutable tag using default and per-table merge modes. */ @Experimental public DatabaseReference mergeDatabaseBranch( - String databaseName, String targetBranch, DatabaseReference source) { - DatabaseReferenceResponse response = - client.post( - resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), - new MergeDatabaseBranchRequest(source), - DatabaseReferenceResponse.class, - restAuthFunction); - return checkNotNull(response.getReference(), "Reference response must contain reference"); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes) { + try { + DatabaseReferenceResponse response = + client.post( + resourcePaths.mergeDatabaseBranch(databaseName, targetBranch), + new MergeDatabaseBranchRequest( + source, defaultMergeMode, tableMergeModes), + DatabaseReferenceResponse.class, + restAuthFunction); + return checkNotNull( + response.getReference(), "Reference response must contain reference"); + } catch (AlreadyExistsException e) { + throw new MergeConflictException( + e, e.resourceType(), e.resourceName(), "%s", e.getMessage()); + } } /** Delete one database-level branch or immutable tag. */ diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java index 87e84b223775..f5cf357b877b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTTreeManagement.java @@ -24,6 +24,8 @@ import javax.annotation.Nullable; +import java.util.List; + /** REST implementation of tree management, bound to the configured REST catalog prefix. */ @Experimental public class RESTTreeManagement implements TreeManagement { @@ -57,16 +59,15 @@ public DatabaseReference createReference( return api.createDatabaseReference(databaseName, referenceName, type, source); } - @Override - public DatabaseReference fastForwardBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return api.fastForwardDatabaseBranch(databaseName, targetBranch, source); - } - @Override public DatabaseReference mergeBranch( - String databaseName, String targetBranch, DatabaseReference source) { - return api.mergeDatabaseBranch(databaseName, targetBranch, source); + String databaseName, + String targetBranch, + DatabaseReference source, + @Nullable MergeMode defaultMergeMode, + @Nullable List tableMergeModes) { + return api.mergeDatabaseBranch( + databaseName, targetBranch, source, defaultMergeMode, tableMergeModes); } @Override diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 01422fdb7e1f..0b25fa3de9d2 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -157,12 +157,6 @@ public String databaseTree(String databaseName, String referenceName) { return SLASH.join(databaseTrees(databaseName), encodeString(referenceName)); } - /** Action endpoint for fast-forwarding a database-level branch. */ - @Experimental - public String forwardDatabaseBranch(String databaseName, String branch) { - return SLASH.join(databaseTree(databaseName, branch), "forward"); - } - /** Action endpoint for merging a branch or tag into a database-level branch. */ @Experimental public String mergeDatabaseBranch(String databaseName, String branch) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java similarity index 62% rename from paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java rename to paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java index d35f98afc16c..b7eab7f34ed0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/FastForwardDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/TableMergeMode.java @@ -16,11 +16,9 @@ * limitations under the License. */ -package org.apache.paimon.rest.requests; +package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.rest.DatabaseReference; -import org.apache.paimon.rest.RESTRequest; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; @@ -29,23 +27,33 @@ import java.beans.ConstructorProperties; -/** Request for fast-forwarding a database branch to another branch or immutable tag. */ +/** Overrides the default merge mode for one table name within the database being merged. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) -public class FastForwardDatabaseBranchRequest implements RESTRequest { +public class TableMergeMode { - private static final String FIELD_SOURCE = "source"; + private static final String FIELD_TABLE = "table"; + private static final String FIELD_MERGE_MODE = "mergeMode"; - private final DatabaseReference source; + private final String table; + private final MergeMode mergeMode; @JsonCreator - @ConstructorProperties({FIELD_SOURCE}) - public FastForwardDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { - this.source = source; + @ConstructorProperties({FIELD_TABLE, FIELD_MERGE_MODE}) + public TableMergeMode( + @JsonProperty(FIELD_TABLE) String table, + @JsonProperty(FIELD_MERGE_MODE) MergeMode mergeMode) { + this.table = table; + this.mergeMode = mergeMode; } - @JsonGetter(FIELD_SOURCE) - public DatabaseReference getSource() { - return source; + @JsonGetter(FIELD_TABLE) + public String getTable() { + return table; + } + + @JsonGetter(FIELD_MERGE_MODE) + public MergeMode getMergeMode() { + return mergeMode; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java b/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java new file mode 100644 index 000000000000..ba8433bbe754 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/exceptions/MergeConflictException.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.exceptions; + +import org.apache.paimon.annotation.Experimental; + +/** Exception thrown when an HTTP 409 prevents a database branch merge. */ +@Experimental +public class MergeConflictException extends RESTException { + + private final String resourceType; + private final String resourceName; + + public MergeConflictException( + Throwable cause, + String resourceType, + String resourceName, + String message, + Object... args) { + super(cause, message, args); + this.resourceType = resourceType; + this.resourceName = resourceName; + } + + public String resourceType() { + return resourceType; + } + + public String resourceName() { + return resourceName; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java index 34c11f261b9f..5beb2e7973ec 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/MergeDatabaseBranchRequest.java @@ -20,14 +20,22 @@ import org.apache.paimon.annotation.Experimental; import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.MergeMode; import org.apache.paimon.rest.RESTRequest; +import org.apache.paimon.rest.TableMergeMode; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + import java.beans.ConstructorProperties; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** Request for merging a branch or immutable tag into a database branch. */ @Experimental @@ -35,17 +43,49 @@ public class MergeDatabaseBranchRequest implements RESTRequest { private static final String FIELD_SOURCE = "source"; + private static final String FIELD_DEFAULT_MERGE_MODE = "defaultMergeMode"; + private static final String FIELD_TABLE_MERGE_MODES = "tableMergeModes"; private final DatabaseReference source; + @Nullable private final MergeMode defaultMergeMode; + @Nullable private final List tableMergeModes; + + public MergeDatabaseBranchRequest(DatabaseReference source) { + this(source, null, null); + } @JsonCreator - @ConstructorProperties({FIELD_SOURCE}) - public MergeDatabaseBranchRequest(@JsonProperty(FIELD_SOURCE) DatabaseReference source) { + @ConstructorProperties({FIELD_SOURCE, FIELD_DEFAULT_MERGE_MODE, FIELD_TABLE_MERGE_MODES}) + public MergeDatabaseBranchRequest( + @JsonProperty(FIELD_SOURCE) DatabaseReference source, + @Nullable @JsonProperty(FIELD_DEFAULT_MERGE_MODE) MergeMode defaultMergeMode, + @Nullable @JsonProperty(FIELD_TABLE_MERGE_MODES) List tableMergeModes) { this.source = source; + this.defaultMergeMode = defaultMergeMode; + this.tableMergeModes = + tableMergeModes == null + ? null + : Collections.unmodifiableList(new ArrayList<>(tableMergeModes)); } @JsonGetter(FIELD_SOURCE) public DatabaseReference getSource() { return source; } + + /** Null uses the server default, {@link MergeMode#NORMAL}. */ + @Nullable + @JsonGetter(FIELD_DEFAULT_MERGE_MODE) + @JsonInclude(JsonInclude.Include.NON_NULL) + public MergeMode getDefaultMergeMode() { + return defaultMergeMode; + } + + /** Per-table modes override the default; null or empty supplies no overrides. */ + @Nullable + @JsonGetter(FIELD_TABLE_MERGE_MODES) + @JsonInclude(JsonInclude.Include.NON_NULL) + public List getTableMergeModes() { + return tableMergeModes; + } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java index 91b838320857..d8991249eac4 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiDatabaseReferenceTest.java @@ -36,6 +36,7 @@ import java.net.InetSocketAddress; import java.net.URLDecoder; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -154,9 +155,9 @@ void testBranchAndImmutableTagHappyPath(DatabaseReferenceType sourceType) throws enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); DatabaseReference source = sourceType == DatabaseReferenceType.BRANCH ? branch : tag; - assertThat(api.fastForwardDatabaseBranch("training db", "main", source)) + assertThat(api.mergeDatabaseBranch("training db", "main", source)) .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); - assertRequest(4, "POST", TREES_PATH + "/main/forward"); + assertRequest(4, "POST", TREES_PATH + "/main/merge"); assertBody( requests.get(4), sourceType == DatabaseReferenceType.BRANCH @@ -219,6 +220,38 @@ void testMergeBranchOrTag(DatabaseReferenceType sourceType) throws Exception { assertThat(requests).hasSize(1); } + @ParameterizedTest + @EnumSource(MergeMode.class) + void testMergeModesAreSentInBody(MergeMode defaultMergeMode) throws Exception { + enqueue(200, "{\"reference\":{\"type\":\"BRANCH\",\"name\":\"main\"}}"); + DatabaseReference source = + new DatabaseReference(DatabaseReferenceType.BRANCH, "experiment"); + + assertThat( + api.mergeDatabaseBranch( + "training db", + "main", + source, + defaultMergeMode, + Arrays.asList( + new TableMergeMode("features.v2", MergeMode.FORCE), + new TableMergeMode("scratch", MergeMode.DROP), + new TableMergeMode("labels", MergeMode.NORMAL)))) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "main")); + + assertRequest(0, "POST", TREES_PATH + "/main/merge"); + assertBody( + requests.get(0), + "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"" + + defaultMergeMode.name() + + "\"," + + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," + + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"); + assertThat(requests).hasSize(1); + } + @Test void testDeleteReferenceWithoutType() throws Exception { enqueue(200, "{\"reference\":{\"type\":\"TAG\",\"name\":\"train-v1\"}}"); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 952ef5ac388b..9e67468e6324 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -20,8 +20,10 @@ import org.apache.paimon.rest.DatabaseReference; import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; import org.apache.paimon.rest.RESTApi; import org.apache.paimon.rest.RESTRequest; +import org.apache.paimon.rest.TableMergeMode; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; @@ -29,6 +31,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; import java.beans.ConstructorProperties; import java.lang.reflect.Constructor; @@ -181,7 +184,6 @@ public class RequestJacksonCompatibilityTest { CreateViewRequest.class, DeleteDatabaseReferenceRequest.class, DropPolicyRequest.class, - FastForwardDatabaseBranchRequest.class, GrantPermissionRequest.class, MergeDatabaseBranchRequest.class, PolicyRequest.class, @@ -251,18 +253,6 @@ void testDeleteDatabaseReferenceRequestRoundTrips() throws Exception { assertThat(RESTApi.fromJson("{}", DeleteDatabaseReferenceRequest.class).getType()).isNull(); } - @ParameterizedTest - @EnumSource(DatabaseReferenceType.class) - void testFastForwardDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) - throws Exception { - String json = "{\"source\":{\"type\":\"" + sourceType.name() + "\",\"name\":\"training\"}}"; - FastForwardDatabaseBranchRequest request = - EXTERNAL_MAPPER.readValue(json, FastForwardDatabaseBranchRequest.class); - FastForwardDatabaseBranchRequest roundTrip = - RESTApi.fromJson(RESTApi.toJson(request), FastForwardDatabaseBranchRequest.class); - assertThat(roundTrip.getSource()).isEqualTo(new DatabaseReference(sourceType, "training")); - } - @ParameterizedTest @EnumSource(DatabaseReferenceType.class) void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) @@ -275,6 +265,64 @@ void testMergeDatabaseBranchRequestRoundTrips(DatabaseReferenceType sourceType) RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); assertThat(roundTrip.getSource()) .isEqualTo(new DatabaseReference(sourceType, "experiment")); + assertThat(roundTrip.getDefaultMergeMode()).isNull(); + assertThat(roundTrip.getTableMergeModes()).isNull(); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @ParameterizedTest + @EnumSource(MergeMode.class) + void testMergeModesRoundTrip(MergeMode mode) throws Exception { + String json = + "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"" + + mode.name() + + "\"," + + "\"tableMergeModes\":[{\"table\":\"features.v2\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}," + + "{\"table\":\"labels\",\"mergeMode\":\"NORMAL\"}]}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getDefaultMergeMode()).isEqualTo(mode); + assertThat(roundTrip.getTableMergeModes()) + .extracting(TableMergeMode::getTable) + .containsExactly("features.v2", "scratch", "labels"); + assertThat(roundTrip.getTableMergeModes()) + .extracting(TableMergeMode::getMergeMode) + .containsExactly(MergeMode.FORCE, MergeMode.DROP, MergeMode.NORMAL); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @Test + void testMergeWithEmptyOverrides() throws Exception { + String json = + "{\"source\":{\"type\":\"TAG\",\"name\":\"train-v1\"},\"tableMergeModes\":[]}"; + MergeDatabaseBranchRequest request = + EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class); + MergeDatabaseBranchRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), MergeDatabaseBranchRequest.class); + assertThat(roundTrip.getDefaultMergeMode()).isNull(); + assertThat(roundTrip.getTableMergeModes()).isEmpty(); + assertThat(RESTApi.fromJson(RESTApi.toJson(roundTrip), Map.class)) + .isEqualTo(RESTApi.fromJson(json, Map.class)); + } + + @ParameterizedTest + @ValueSource( + strings = { + "\"defaultMergeMode\":\"UNKNOWN\"", + "\"tableMergeModes\":[{\"table\":\"features\",\"mergeMode\":\"UNKNOWN\"}]" + }) + void testUnknownMergeModesAreRejected(String modes) { + String json = "{\"source\":{\"type\":\"BRANCH\",\"name\":\"experiment\"}," + modes + "}"; + assertThatThrownBy(() -> EXTERNAL_MAPPER.readValue(json, MergeDatabaseBranchRequest.class)) + .hasMessageContaining("UNKNOWN"); + assertThatThrownBy(() -> RESTApi.fromJson(json, MergeDatabaseBranchRequest.class)) + .hasMessageContaining("UNKNOWN"); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java index 20868aadb3c0..e4cb58ebc7a9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTreeManagementTest.java @@ -23,6 +23,8 @@ import org.apache.paimon.management.TreeManagement; import org.apache.paimon.options.Options; import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.BadRequestException; +import org.apache.paimon.rest.exceptions.MergeConflictException; import org.apache.paimon.rest.exceptions.NoSuchResourceException; import org.apache.paimon.rest.exceptions.NotImplementedException; @@ -35,6 +37,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -122,11 +125,9 @@ void testBranchAndTagOperationsUseCatalogConfiguration(DatabaseReferenceType sou enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); DatabaseReference source = sourceType == BRANCH ? branch : tag; - assertThat(trees.fastForwardBranch(DATABASE, "main", source)).isEqualTo(main); - RecordedRequest fastForward = takeRequest("POST", TREES_PATH + "/main/forward"); - assertBody( - fastForward, - "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); + assertThat(trees.mergeBranch(DATABASE, "main", source)).isEqualTo(main); + RecordedRequest merge = takeRequest("POST", TREES_PATH + "/main/merge"); + assertBody(merge, "{\"source\":" + (sourceType == BRANCH ? BRANCH_JSON : TAG_JSON) + "}"); enqueue(200, "{\"reference\":" + BRANCH_JSON + "}"); assertThat(trees.deleteReference(DATABASE, "exp-1", BRANCH)).isEqualTo(branch); @@ -189,21 +190,76 @@ void testMergeUsesCatalogConfiguration(DatabaseReferenceType sourceType) throws assertThat(server.getRequestCount()).isEqualTo(2); } + @ParameterizedTest + @EnumSource(DatabaseReferenceType.class) + void testMergeModesUseCatalogConfiguration(DatabaseReferenceType sourceType) throws Exception { + enqueue(200, "{\"reference\":" + MAIN_JSON + "}"); + DatabaseReference source = new DatabaseReference(sourceType, "experiment"); + + assertThat( + trees.mergeBranch( + DATABASE, + "main", + source, + MergeMode.NORMAL, + Arrays.asList( + new TableMergeMode("features", MergeMode.FORCE), + new TableMergeMode("scratch", MergeMode.DROP)))) + .isEqualTo(new DatabaseReference(BRANCH, "main")); + + assertBody( + takeRequest("POST", TREES_PATH + "/main/merge"), + "{\"source\":{\"type\":\"" + + sourceType.name() + + "\",\"name\":\"experiment\"}," + + "\"defaultMergeMode\":\"NORMAL\",\"tableMergeModes\":[" + + "{\"table\":\"features\",\"mergeMode\":\"FORCE\"}," + + "{\"table\":\"scratch\",\"mergeMode\":\"DROP\"}]}"); + assertThat(server.getRequestCount()).isEqualTo(2); + } + @Test void testMergeErrorsPreserveDetails() throws Exception { DatabaseReference source = new DatabaseReference(BRANCH, "experiment"); - enqueue( - 409, - "{\"code\":409,\"message\":\"Conflicting changes to table features\"," - + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}"); + server.enqueue( + new MockResponse() + .setResponseCode(409) + .setHeader("Content-Type", "application/json") + .setHeader("x-request-id", "merge-request") + .setBody( + "{\"message\":\"Conflicting changes to table features (100%)\"," + + "\"resourceType\":\"TABLE\",\"resourceName\":\"training db.features\"}")); assertThatThrownBy(() -> trees.mergeBranch(DATABASE, "main", source)) .isInstanceOfSatisfying( - AlreadyExistsException.class, + MergeConflictException.class, conflict -> { assertThat(conflict.resourceType()).isEqualTo("TABLE"); assertThat(conflict.resourceName()).isEqualTo("training db.features"); + assertThat(conflict.getCause()) + .isInstanceOf(AlreadyExistsException.class) + .hasMessage(conflict.getMessage()); }) - .hasMessageContaining("Conflicting changes to table features"); + .hasMessage("Conflicting changes to table features (100%) requestId:merge-request"); + takeRequest("POST", TREES_PATH + "/main/merge"); + + enqueue(409, "{\"code\":409,\"message\":\"reference already exists\"}"); + assertThatThrownBy(() -> trees.createReference(DATABASE, "existing", BRANCH, source)) + .isExactlyInstanceOf(AlreadyExistsException.class); + takeRequest("POST", TREES_PATH); + + enqueue(400, "{\"code\":400,\"message\":\"duplicate table merge mode\"}"); + assertThatThrownBy( + () -> + trees.mergeBranch( + DATABASE, + "main", + source, + MergeMode.NORMAL, + Arrays.asList( + new TableMergeMode("features", MergeMode.FORCE), + new TableMergeMode("features", MergeMode.DROP)))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("duplicate table merge mode"); takeRequest("POST", TREES_PATH + "/main/merge"); enqueue(404, "{\"code\":404,\"message\":\"source reference missing\"}"); @@ -217,7 +273,7 @@ void testMergeErrorsPreserveDetails() throws Exception { .isInstanceOf(NotImplementedException.class) .hasMessageContaining("merge unsupported"); takeRequest("POST", TREES_PATH + "/main/merge"); - assertThat(server.getRequestCount()).isEqualTo(4); + assertThat(server.getRequestCount()).isEqualTo(6); } @Test From da8e1af2eb3d7ce88b934efcee747789c93044f5 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 18:07:39 +0800 Subject: [PATCH 08/10] [docs] Describe database versioning APIs and server MVP --- .../docs/concepts/rest/database-versioning.md | 488 ++++++++++++++++++ docs/docs/concepts/rest/index.md | 2 + docs/docs/concepts/rest/rest-api.md | 7 +- docs/docs/program-api/rest-api.mdx | 1 + docs/sidebars.js | 1 + 5 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 docs/docs/concepts/rest/database-versioning.md diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md new file mode 100644 index 000000000000..ea956a615452 --- /dev/null +++ b/docs/docs/concepts/rest/database-versioning.md @@ -0,0 +1,488 @@ +--- +title: "Database Branches and Tags" +--- + + + +# Database Branches and Tags + +Database references group the versions of several tables under one branch or tag. A typical +training workflow starts an experiment from `main`, writes derived data on the experiment branch, +freezes the inputs under a tag, and merges accepted changes back into `main`. + +This page describes the experimental REST management contract and a proposed server MVP that +reuses Paimon's existing [table branches](../../maintenance/manage-branches) and +[table tags](../../maintenance/manage-tags). + +:::info Implementation status + +The Java reference-management client and the `/trees` wire contract are implemented. Reference +storage, table-level orchestration, and database merge execution must be implemented by the catalog +server. The server implementation below is a design, not a claim that an existing service supports it. + +The MVP uses explicit table-level branch and tag addressing. This page introduces no new reference +header or catalog option. Selecting a complete database view for table listing and DDL remains +additional work, described under [Beyond the fixed-table MVP](#beyond-the-fixed-table-mvp). + +::: + +## Scope and terminology + +| Term | Meaning | +| --- | --- | +| Database branch | A writable reference to a database's table membership and table versions. | +| Database tag | An immutable reference to a captured membership and versions. Deleting a tag is allowed; moving it is not. | +| Table branch/tag | The existing Paimon storage and read/write mechanism used behind a database reference. | +| Table membership | The names and identities of the tables visible in a database reference. | +| Merge base | The historical state used to distinguish source changes from target changes. It includes previous merge relationships. | + +References belong to one database, not the whole catalog. Branches and tags share a name namespace +within that database. A reference contains `type` (`BRANCH` or `TAG`) and `name`. Names match +`[A-Za-z0-9][A-Za-z0-9._-]{0,127}`. Public references have no hash or reference ID. + +### Initial server MVP + +Start with managed native Paimon tables and a fixed set of logical table names. Create and populate +those tables on `main` before starting the experiment. Use batch writers and pause writes during +branch creation, tag creation, and merge. Resume with freshly loaded tables after publication. + +This scope can demonstrate isolated table writes, multi-table training inputs frozen under a tag, +and table-version merge. It does not require a public multi-table transaction API, public hashes, +row-level conflict resolution, or concurrent streaming publication. + +Branch-local table creation, deletion, and rename need reference-aware namespace handling. The +merge contract covers table creation and deletion, but the first fixed-table server can defer those +operations until the table APIs can address the corresponding database view. Format Tables, +Object Tables, external tables, views, functions, and catalog permissions are outside this initial +versioned-table scope. + +## Reference management API + +All paths use the configured catalog `prefix`. For brevity, the following table uses +`B = /v1/{prefix}/databases/{database}`. Encode each path segment; names in JSON remain unencoded. + +| Method and path | Request | Result | +| --- | --- | --- | +| `GET B/trees` | Optional `type`, `maxResults`, and `pageToken` query parameters. | One page of references. | +| `GET B/trees/{name}` | No body. | One reference. | +| `POST B/trees` | New name, type, and an existing source reference. | The created reference. | +| `POST B/trees/{name}/merge` | Source reference and optional merge modes. The path names the target branch. | The target reference after success. | +| `DELETE B/trees/{name}` | Optional expected `type` in the body. | The deleted reference. | + +Database merge includes fast-forward when applicable. There is no database-level `/forward` +endpoint. The existing table-level forward API is separate. + +### Create a branch or tag + +Create an experiment branch from `main`: + +```http +POST /v1/catalog/databases/training/trees +Content-Type: application/json + +{ + "name": "experiment", + "type": "BRANCH", + "source": {"type": "BRANCH", "name": "main"} +} +``` + +Freeze the experiment under a database tag: + +```json +{ + "name": "train_v1", + "type": "TAG", + "source": {"type": "BRANCH", "name": "experiment"} +} +``` + +Both requests use the same path. The source must exist in the same database. A source can be a +branch or an immutable tag; the new reference can also be either type. Successful singular +operations return `DatabaseReferenceResponse`: + +```json +{"reference": {"type": "TAG", "name": "train_v1"}} +``` + +### Inspect and list + +```http +GET /v1/catalog/databases/training/trees/train_v1 +GET /v1/catalog/databases/training/trees?type=tag&maxResults=100 +``` + +The list filter uses lowercase `branch` or `tag`; JSON reference types use uppercase enum names. +Omitting `type` includes both. A missing or zero `maxResults` uses the server default. Pass the +returned `nextPageToken` unchanged to request the next page; a missing token ends iteration. + +```json +{ + "references": [{"type": "TAG", "name": "train_v1"}], + "nextPageToken": "next-page" +} +``` + +Getting a reference returns its name and type, not the table membership, source branch, or table +version map. Pagination discovers references; it does not create a frozen view across pages. + +### Merge + +```http +POST /v1/catalog/databases/training/trees/main/merge +Content-Type: application/json + +{ + "source": {"type": "BRANCH", "name": "experiment"}, + "defaultMergeMode": "NORMAL", + "tableMergeModes": [ + {"table": "features", "mergeMode": "FORCE"}, + {"table": "scratch", "mergeMode": "DROP"} + ] +} +``` + +Only `source` is required. Omitting modes gives `NORMAL` for every table. Per-table modes override +the default, and an omitted or empty override list applies the default everywhere. Table names +are exact names within this database. Duplicate table overrides are a bad request; an override +for a table without source-side changes has no effect. + +The target is always a branch. A source tag is allowed and remains immutable. The response remains +`DatabaseReferenceResponse`; it does not include a commit hash or a detailed merge report. + +### Delete + +```http +DELETE /v1/catalog/databases/training/trees/train_v1 +Content-Type: application/json + +{"type": "TAG"} +``` + +The optional type checks the reference before deletion. Omitting the body or sending `{}` omits +that check. An absent reference is an error. The MVP server should protect the default `main` +branch. Logical deletion does not authorize deleting table versions still needed by another +reference. + +### Errors + +| Situation | HTTP behavior | +| --- | --- | +| Missing database or reference | `404`; merge distinguishes the missing source or target in its error details. | +| Creating an existing reference | `409`. | +| Merge target is a tag, no merge base is available, or unresolved table conflicts remain | `409`; the target stays unchanged. | +| Invalid merge request, such as duplicate per-table modes | `400`. | +| Deleting a protected default branch or supplying the wrong expected type | `409`. | +| Server does not implement an operation | No client fallback; the server error is propagated. | + +Errors use `ErrorResponse`. The Java merge client converts `409` to `MergeConflictException` and +preserves the resource type/name, message, request ID, and cause. Resource creation still uses +`AlreadyExistsException`. See the [OpenAPI specification](/rest-catalog-open-api.yaml) for the +individual operations and their documented responses. + +## Java management usage + +Obtain tree management from an already configured `RESTCatalog`. It shares that catalog's +prefix, authentication, and HTTP configuration: + +```java +import org.apache.paimon.PagedList; +import org.apache.paimon.management.TreeManagement; +import org.apache.paimon.rest.DatabaseReference; +import org.apache.paimon.rest.DatabaseReferenceType; +import org.apache.paimon.rest.MergeMode; +import org.apache.paimon.rest.TableMergeMode; + +import java.util.Collections; + +TreeManagement trees = restCatalog.treeManagement(); +DatabaseReference main = new DatabaseReference(DatabaseReferenceType.BRANCH, "main"); +DatabaseReference experiment = trees.createReference( + "training", "experiment", DatabaseReferenceType.BRANCH, main); + +// Run batch writes on the corresponding table branches before freezing this tag. +DatabaseReference trainingTag = trees.createReference( + "training", "train_v1", DatabaseReferenceType.TAG, experiment); + +PagedList page = trees.listReferencesPaged( + "training", DatabaseReferenceType.TAG, 100, null); + +// Default three-way merge, failing on conflicting table versions. +trees.mergeBranch("training", "main", trainingTag); +``` + +When resolving a conflict, use the following call instead of the default merge to accept the +source version of `features`. Changing modes after a successful merge does not reapply that source: + +```java +trees.mergeBranch( + "training", "main", trainingTag, MergeMode.NORMAL, + Collections.singletonList(new TableMergeMode("features", MergeMode.FORCE))); +``` + +`RESTApi` exposes equivalent methods: `listDatabaseReferencesPaged`, `getDatabaseReference`, +`createDatabaseReference`, `mergeDatabaseBranch`, and `deleteDatabaseReference`. Listing is paged; +there is no non-paged database-reference helper. + +These are management calls. Creating a database branch does not switch the catalog's ordinary +table operations to that branch. + +## Reusing table branches and tags on the server + +The server coordinates existing table-level operations and keeps database metadata around them. +An illustrative mapping is: + +```text +training / main [BRANCH] + features -> table identity A, table branch main + labels -> table identity B, table branch main + +training / experiment [BRANCH] + features -> table identity A, table branch experiment + labels -> table identity B, table branch experiment + +training / train_v1 [TAG] + features -> table identity A, experiment branch, pinned table tag train_v1 + labels -> table identity B, experiment branch, pinned table tag train_v1 +``` + +For the explicit-addressing MVP, names such as `experiment` can also name the corresponding table +branches, and `train_v1` can name each table tag in its source table branch. These names are owned +by the service. Reject collisions with unrelated existing table references; do not adopt them just +because the names match. Additional internal baseline tags can use private, service-generated names. + +The public database-reference name rules and native table-branch rules are not identical. For +example, the database protocol permits a purely numeric name, while native table branch creation +rejects it. A server supporting the full name contract needs an alias mapping to valid physical +branch names and must resolve the logical table-branch address through that mapping. It must not +silently narrow the database API's name rules. The examples use names valid in both layers. + +### Minimal metadata + +The server needs: + +- A database reference record: name, type, current membership, and internal ancestry/merge history. +- A mapping from logical table name to stable table identity and backing table branch or tag. +- Captured table versions at branch points, tag creation, and merges, including the schema and + snapshot state needed for comparisons and reads. + +A captured version can reuse a snapshot UUID, a pinned schema, and relevant table properties. +Empty tables need an explicit no-snapshot state. Numeric snapshot/schema IDs alone are not enough +to compare independently written branches. Table identity distinguishes a dropped-and-recreated +table from its predecessor. Copying metadata to a new physical branch does not itself constitute a +logical table change. + +These records can live in the catalog backend. Their internal identities are not public hashes and +need not introduce a new versioned storage engine. The server must update its recorded table state +when a managed branch accepts a table commit or schema change; names alone cannot support merge. +Use the server's table commit and schema operations for those writes. Uncoordinated filesystem +writes or direct edits of service-owned table references would bypass this bookkeeping. + +### Bootstrap main + +A version-enabled new database starts with an empty `main` branch. Otherwise every create-reference +request would require a source that does not yet exist. For an existing database, the server can +initialize `main` from its current tables while writers are stopped. Automatic online conversion of +an actively written database is outside the first MVP. + +This is a server lifecycle rule, not an additional REST endpoint. Reference creation always keeps +its existing `source` field. + +### Create a database branch + +1. Capture the source membership and each selected table version while writes are paused. +2. For a populated table, pin the selected source snapshot with a service-owned table tag and create + the destination table branch from that tag. +3. For an empty table, create a schema-only table branch. Preserve the selected schema and properties. +4. Record the common baseline and publish the database branch after all table branches are ready. + +The existing `FileSystemBranchManager.createBranch(name)` creates an empty branch by copying +schemas. It does not clone the source data. `createBranch(name, tagName)` copies the selected +snapshot and its schemas. If the captured current schema is newer than the snapshot's schema, +the server must also preserve that schema-only change; snapshot cloning alone is insufficient. + +No data files need to be copied merely to create a branch. In the single-process MVP, table-level +setup can run sequentially; do not expose an incomplete database reference as successfully created. +Failures can leave private work to clean up or resume. + +### Create and retain a database tag + +Capture the table membership and pin a table tag for each populated table. Persist the source table +branch with each pin: native Paimon tags belong to a table branch, not a database-wide directory. +An empty table has no snapshot to tag, so its frozen entry must retain the schema and empty state; +the server cannot blindly call `createTag` on every table. + +A database tag never follows subsequent writes to its source. For an empty tagged table, reads must +remain empty even if the source later receives its first snapshot. The source's later schema +changes must also leave the tagged schema unchanged. A demonstration server that has not implemented +empty-table reads must restrict tagging to populated tables explicitly. + +Service-owned pins must not expire through ordinary automatic tag-retention settings or be replaced +through user table-tag operations. Table branch deletion removes its metadata directory, including +the tags in that directory. Keep a backing branch while a database tag or merge baseline still needs +it, or relocate the retained metadata before deleting it. + +The first MVP can defer physical deletion and cleanup. Removing a logical database reference need +not immediately drop its underlying table branches or files. Enable physical cleanup only when it +accounts for all retained database references and merge baselines. + +## Merge semantics and execution + +Merge operates on complete table versions, including schema, properties, and snapshot state. It +also defines how table presence or absence is combined once branch-aware DDL is available. + +Let `B`, `S`, and `T` be a table's base, source, and target version, with absence represented as a +state. First determine whether the source changed relative to `B`: + +| Condition or mode | Result | +| --- | --- | +| `S = B` | Keep `T`, including target-only changes. | +| Source changed, mode `DROP` | Keep `T`, even when there would be no conflict. | +| Source changed, mode `FORCE` | Use `S`, including source-side deletion. | +| Source changed, mode `NORMAL`, and `T = B` | Use `S`. | +| Source changed, mode `NORMAL`, and `S = T` | Accept the identical result. | +| Source changed, mode `NORMAL`, and both sides changed differently | Fail the merge with `409`. | + +Different tables can therefore change independently and merge successfully. Different versions of +the same table conflict under `NORMAL`, even when an application might know how to combine their +rows. `FORCE` selects a complete source version; `DROP` skips all source changes to the selected +table, not just conflicting changes. + +### Publication + +1. Resolve the source and target and find their merge base, including earlier merges. +2. Compare table versions and compute the complete result using the selected modes. +3. If any unresolved conflict exists, return `409` before changing target tables. +4. Prepare the selected target table versions with table-level snapshot/schema mechanisms and publish + the database result. Record the source as merged; never modify the source reference. + +When the target is an ancestor of the source, fast-forward is possible only if the chosen modes +produce exactly the source state. Already-merged sources and identical reference states succeed +without changing the target. Divergent histories use three-way merge. + +The existing table `mergeBranch` implementation merges append-only data-file changes. It is not an +implementation of this whole-table-version algorithm. Existing table `fastForward` also has its own +replacement semantics and can remove target metadata and tags. A server needs an adapter that +checks the database result first and preserves retained references; looping over either operation +without that adapter is insufficient. + +Preparing fresh backing branches and publishing a new mapping is one possible server implementation. +A server retaining the explicit table branch names must instead provide a safe way to install the +prepared versions behind those names. The client-visible table address must continue to resolve +correctly. In either case, source and target must remain independently writable: pointing both at +the same mutable table branch would make future source writes modify the target as well. + +### Repeated merge + +After a successful merge, the server records the integrated source version even if `DROP` preserved +all target table contents. Repeating a merge of that same source state must not bring skipped +changes back. A later source write can participate in a subsequent merge using the updated history. + +Do not identify prior merges only by the source branch name; that branch can continue to advance. +Internal ancestry is required even though the public API has no hash. The first MVP can serialize +these operations and pause writers instead of introducing public concurrency tokens or multi-table +transactions. Reads during a multi-table publication need not provide an atomic database view in +this restricted MVP. Partial backend execution still needs a recoverable server operation record; +an HTTP success must mean that the planned result is installed. + +## Exercise the fixed-table MVP + +The following workflow requires a server that implements the orchestration above. The current Java +client tests alone do not provide that server. + +1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. + Stop writes and create database branch `experiment` from `main` using `/trees`. +2. Read and write the corresponding table branches through existing table addressing. For example, + Spark uses these table names: + + ```sql + SELECT * FROM training.`features$branch_experiment`; + SELECT * FROM training.`labels$branch_experiment`; + ``` + + Batch writes use the same branch-qualified names. An unqualified table name still addresses its + ordinary main table; creating the database branch does not switch the current catalog. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. The server creates + and protects the corresponding table tags. In this explicit-addressing workflow, the training + job records both the source branch and tag name: + + ```sql + SELECT * FROM training.`features$branch_experiment` VERSION AS OF 'train_v1'; + SELECT * FROM training.`labels$branch_experiment` VERSION AS OF 'train_v1'; + ``` + + These examples compose existing table branch and time-travel syntax. The server integration + must verify that both snapshot and schema lookups resolve the protected tag. The database tag + response itself does not return the source branch or a table mapping. +4. Advance the experiment tables, then repeat the tagged reads. They must return the earlier data + and schemas. Keep the tag's backing table branches while these reads are needed. +5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the + evaluated source version. Merging the live `experiment` branch would instead include its newer + state. If both sides changed a table, inspect the conflict and deliberately choose a per-table + mode when appropriate. +6. Reload main tables and verify the published state. Resume writes separately on `main` and + `experiment` and verify that neither changes the other. Merge the same source state again to + check no-op behavior. Delete unused database references through tree management. + +## Beyond the fixed-table MVP + +A complete database view needs a defined way for ordinary table APIs to identify the selected +database branch or tag. Existing table branch suffixes identify one table branch; they do not scope +`listTables`, create-table, drop-table, or rename operations to a database reference. + +The following work remains separate from the management API: + +- Address and return the selected reference's table membership, including tables absent on `main`. +- Apply branch-local DDL to that membership, with stable table identities for rename and new + identities for drop-and-recreate. +- Resolve a database tag to its table mappings without requiring the caller to remember its source + branch; expose the frozen schema and empty-table state as well as the snapshot. +- Preserve the selected context through schema/snapshot access, catalog loaders, task serialization, + and caches. Table identifiers by ID and catalog-wide listings need equally explicit semantics. + +This document does not select a new header, query parameter, catalog option, or endpoint for those +operations. Such an extension needs its own agreed wire contract and client integration. Database +reference management can be implemented first, and a fixed-table training workflow can use explicit +table addressing while that namespace design is settled. + +## Validation and implementation sequence + +The existing reference tests validate HTTP paths, request bodies, authentication/configuration, +pagination, JSON compatibility, and exception propagation. Mocked success responses do not test +server branch isolation, snapshot retention, or the merge algorithm. + +Implement and verify in this order: + +1. **Reference records and bootstrap:** create `main`, list/get/create/delete references, and protect + managed table-reference names. +2. **Table orchestration:** clone populated and empty tables correctly; record baselines; route + explicit table branches through existing Paimon readers and writers. +3. **Frozen training inputs:** pin table tags and schemas, validate repeated reads after source + writes, and retain dependencies after logical reference deletion. Add empty-table coverage when + that case is enabled. +4. **Merge:** verify automatic fast-forward, independent changes to different tables, same-table + conflicts leaving the target unchanged, all three modes, repeated merge including `DROP`, and + continued independent writes after merge. +5. **Complete database views:** settle reference-aware listing and DDL, then add membership changes + and tag-only discovery to the integration tests. + +A useful acceptance test uses real Paimon snapshots for two tables and exercises the workflow above +against a stateful server. Passing that test establishes the fixed-table MVP; a full database-view +MVP additionally requires the final namespace step. diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index f2974ef44190..d62f8807a26e 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,6 +74,8 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. +- [Database Branches and Tags](./database-versioning): experimental reference-management APIs and + the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index 18e54faea4db..c310bb09c1b7 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -52,7 +52,8 @@ payloads, and error responses are defined in the OpenAPI specification. | Commits and snapshots | Commit, roll back, and inspect table versions. | Table-scoped `commit`, `rollback`, `rollback-schema`, `snapshot`, and `snapshots`. | | Data access | Request storage credentials and authorize a query. | Table-scoped `token` and `auth`. | | Partitions | List, create, drop, and mark partitions done. | Table-scoped `partitions`. | -| Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | +| Table branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | +| Database branches and tags | List, get, create, delete, and merge references. | Database-scoped `trees` and `trees/{name}/merge`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | @@ -60,6 +61,10 @@ In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing endpoints are described in the specification alongside their database-scoped counterparts. +See [Database Branches and Tags](./database-versioning) for reference-management examples, merge +modes, and the server MVP design. These management APIs do not switch ordinary table requests to +a database reference automatically. + ## Partition Compatibility Partition options use the existing `POST .../partitions` request. `partitionOptions` follows the diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index ed19d4e380c9..8e7739bb6c7b 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -36,6 +36,7 @@ metadata requests without bringing in the full table read/write bundle. | Load a `Table` and read or write rows | [Java API](java-api) with a REST catalog | | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | +| Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | ## Dependency diff --git a/docs/sidebars.js b/docs/sidebars.js index 9c9f11a2991c..cdc959aa0bb5 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -44,6 +44,7 @@ const sidebars = { "concepts/rest/tables", "concepts/rest/pvfs", "concepts/rest/rest-api", + "concepts/rest/database-versioning", "concepts/rest/management-api" ] }, From e376e0f5b80cbe8ebf82e5ecf09d356c5b9523ac Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 21:55:09 +0800 Subject: [PATCH 09/10] [rest] Scope table operations to database references --- .../docs/concepts/rest/database-versioning.md | 213 ++++-- docs/docs/concepts/rest/index.md | 2 +- docs/docs/concepts/rest/rest-api.md | 5 +- docs/docs/program-api/rest-api.mdx | 1 + docs/scripts/validate-rest-openapi.js | 37 + docs/static/rest-catalog-open-api.yaml | 720 ++++++++++++++++++ .../apache/paimon/rest/DatabaseReference.java | 8 +- .../java/org/apache/paimon/rest/RESTApi.java | 20 + .../org/apache/paimon/rest/ResourcePaths.java | 143 ++-- .../org/apache/paimon/rest/RESTCatalog.java | 52 +- .../apache/paimon/rest/RESTCatalogLoader.java | 15 +- .../paimon/rest/RESTCatalogReferenceTest.java | 469 ++++++++++++ 12 files changed, 1545 insertions(+), 140 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index ea956a615452..eef723837b83 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -27,19 +27,20 @@ Database references group the versions of several tables under one branch or tag training workflow starts an experiment from `main`, writes derived data on the experiment branch, freezes the inputs under a tag, and merges accepted changes back into `main`. -This page describes the experimental REST management contract and a proposed server MVP that +This page describes the experimental REST reference and table contracts and a proposed server MVP that reuses Paimon's existing [table branches](../../maintenance/manage-branches) and [table tags](../../maintenance/manage-tags). :::info Implementation status -The Java reference-management client and the `/trees` wire contract are implemented. Reference -storage, table-level orchestration, and database merge execution must be implemented by the catalog -server. The server implementation below is a design, not a claim that an existing service supports it. +The Java reference-management client, reference-scoped table client, and their wire contracts are +implemented. Reference storage, table-level orchestration, and database merge execution must be +implemented by the catalog server. The server implementation below is a design, not a claim that +an existing service supports it. -The MVP uses explicit table-level branch and tag addressing. This page introduces no new reference -header or catalog option. Selecting a complete database view for table listing and DDL remains -additional work, described under [Beyond the fixed-table MVP](#beyond-the-fixed-table-mvp). +Table operations select a database reference through `/trees/{reference}` in the resource path. +Callers use logical table names without constructing table branch suffixes or remembering a tag's +source branch. This requires no new reference header or catalog option. ::: @@ -69,7 +70,8 @@ row-level conflict resolution, or concurrent streaming publication. Branch-local table creation, deletion, and rename need reference-aware namespace handling. The merge contract covers table creation and deletion, but the first fixed-table server can defer those -operations until the table APIs can address the corresponding database view. Format Tables, +operations until reference-aware namespace storage is implemented. Their scoped REST routes already +reuse the ordinary table request and response schemas; rename is deferred. Format Tables, Object Tables, external tables, views, functions, and catalog permissions are outside this initial versioned-table scope. @@ -197,6 +199,108 @@ preserves the resource type/name, message, request ID, and cause. Resource creat `AlreadyExistsException`. See the [OpenAPI specification](/rest-catalog-open-api.yaml) for the individual operations and their documented responses. +## Reference-scoped table API + +Let `S = /v1/{prefix}/databases/{database}/trees/{reference}`. The reference name selects either +an existing branch or an immutable tag. It is resolved by the server; the client need not first +fetch its type. These endpoints reuse the ordinary table request and response structures: + +| Method and path | Existing request / response | Scope | +| --- | --- | --- | +| `GET S/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | +| `GET S/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | +| `GET S/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | +| `POST S/tables` | `CreateTableRequest`. | Create a table in a branch. | +| `POST S/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | +| `DELETE S/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | +| `GET S/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | +| `GET S/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | +| `GET S/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | +| `GET S/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | +| `GET S/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | +| `POST S/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | +| `GET S/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | +| `POST S/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | + +For example, read the same logical table through a live experiment and a frozen training tag: + +```http +GET /v1/catalog/databases/training/trees/experiment/tables/features +GET /v1/catalog/databases/training/trees/train_v1/tables/features +``` + +The response name remains `features`. `GetTableResponse` carries the resolved schema, path and +storage options; the server may supply an internal physical branch in the existing schema options. +A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and `statistics` fields. +The reference path determines the target; identifiers and table IDs in the request must agree with +the table resolved from that path. Caller-supplied table branch suffixes are not part of this contract. + +### Branch and tag behavior + +A branch resolves to its current membership and table versions. A tag resolves to the membership, +schemas, options and snapshots captured when it was created, even after its source branch advances. +Tag snapshot listing exposes only the pinned snapshot. `LATEST` and `EARLIEST` select that snapshot; +other version selectors must resolve to it or return `404`. Schema reads may access the captured +schema and older schemas retained for reading the captured data, but never later source schemas. +An empty captured table is still returned by `GET table`; snapshot lookup returns `404` with +`resourceType: SNAPSHOT`. + +The server rejects content changes through a tag with `409`. Read authorization remains allowed +through `POST .../auth`; HTTP method alone does not determine whether an operation is a write. +Tag credentials must permit reading without allowing mutation of retained metadata or data. + +Missing references and tables return `404`. Unsupported scoped operations return `501`, without +falling back to the ordinary main-table path. Existing unscoped URLs retain their behavior. + +### Java table usage + +Bind a separate client instance to the desired database reference: + +```java +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.rest.RESTCatalog; +import org.apache.paimon.table.Table; + +RESTCatalog experimentCatalog = restCatalog.withReference("training", "experiment"); +RESTCatalog trainingCatalog = restCatalog.withReference("training", "train_v1"); + +Table experimentFeatures = experimentCatalog.getTable(Identifier.create("training", "features")); +Table trainingFeatures = trainingCatalog.getTable(Identifier.create("training", "features")); + +// Use experimentFeatures with the ordinary Paimon batch write API. +// Use trainingFeatures with the ordinary Paimon read API. +``` + +`withReference` leaves the original catalog unchanged and does not fetch catalog configuration +again. Each returned catalog keeps its own binding and local caches. Its serialized +`RESTCatalogLoader`, and loaders inside serialized table objects, retain the binding for later +snapshot reads, commits, schema changes and token requests. Storage commits can supply a physical +table branch internally; the bound catalog sends the logical table name and keeps the reference +path authoritative. + +The lightweight client supports the same binding through +`RESTApi.withReference("training", "experiment")`. Existing table methods and DTOs remain usable. +Table operations must use the bound database. Binding is currently a Java API, not a SQL catalog +option; engine configuration for selecting a reference is additional integration work. + +The scoped client does not yet support global table listing, lookup by table ID, rename, register, +replace, rollback, partition/consumer endpoints, or nested table branch/tag management. Such calls +fail locally instead of reaching an unscoped table route. Database and reference management, +functions, views and catalog-level management retain their existing meaning; this binding versions +only the supported table endpoints. Table policy endpoints are also outside the scoped MVP. + +### Server routing and reuse + +Resolve `(database, reference, logical table)` once into an internal request context containing +reference type, table identity and backing table version. Pass that context into the existing table +handlers. Validate authentication against the actual scoped request, and authorize access to the +resolved table. A path rewrite alone is insufficient: listing must use the selected membership, +tags need frozen metadata, and commits must update the selected branch's recorded table state. + +The additional routing and DTO work is small. Runtime work is a reference/table mapping lookup, +which can be cached; adding the scope does not require proxying or copying table data. Reference +creation, retention, namespace changes and merge still require the server orchestration below. + ## Java management usage Obtain tree management from an already configured `RESTCatalog`. It shares that catalog's @@ -263,7 +367,7 @@ training / train_v1 [TAG] labels -> table identity B, experiment branch, pinned table tag train_v1 ``` -For the explicit-addressing MVP, names such as `experiment` can also name the corresponding table +Names such as `experiment` can also name the corresponding backing table branches, and `train_v1` can name each table tag in its source table branch. These names are owned by the service. Reject collisions with unrelated existing table references; do not adopt them just because the names match. Additional internal baseline tags can use private, service-generated names. @@ -271,7 +375,7 @@ because the names match. Additional internal baseline tags can use private, serv The public database-reference name rules and native table-branch rules are not identical. For example, the database protocol permits a purely numeric name, while native table branch creation rejects it. A server supporting the full name contract needs an alias mapping to valid physical -branch names and must resolve the logical table-branch address through that mapping. It must not +branch names and must resolve the scoped logical table address through that mapping. It must not silently narrow the database API's name rules. The examples use names valid in both layers. ### Minimal metadata @@ -384,9 +488,9 @@ checks the database result first and preserves retained references; looping over without that adapter is insufficient. Preparing fresh backing branches and publishing a new mapping is one possible server implementation. -A server retaining the explicit table branch names must instead provide a safe way to install the -prepared versions behind those names. The client-visible table address must continue to resolve -correctly. In either case, source and target must remain independently writable: pointing both at +The server can update the reference mapping to those prepared versions while retaining any +physical branches needed by tags or merge baselines. The client-visible scoped table address must +continue to resolve correctly. In either case, source and target must remain independently writable: pointing both at the same mutable table branch would make future source writes modify the target as well. ### Repeated merge @@ -404,84 +508,63 @@ an HTTP success must mean that the planned result is installed. ## Exercise the fixed-table MVP -The following workflow requires a server that implements the orchestration above. The current Java -client tests alone do not provide that server. +The following workflow requires a server that implements the orchestration above. The client tests +exercise scoped HTTP routing and table loaders; they do not implement database reference storage +or the database merge algorithm. 1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. - Stop writes and create database branch `experiment` from `main` using `/trees`. -2. Read and write the corresponding table branches through existing table addressing. For example, - Spark uses these table names: - - ```sql - SELECT * FROM training.`features$branch_experiment`; - SELECT * FROM training.`labels$branch_experiment`; - ``` - - Batch writes use the same branch-qualified names. An unqualified table name still addresses its - ordinary main table; creating the database branch does not switch the current catalog. -3. Stop experiment writes and create database tag `train_v1` from `experiment`. The server creates - and protects the corresponding table tags. In this explicit-addressing workflow, the training - job records both the source branch and tag name: - - ```sql - SELECT * FROM training.`features$branch_experiment` VERSION AS OF 'train_v1'; - SELECT * FROM training.`labels$branch_experiment` VERSION AS OF 'train_v1'; - ``` - - These examples compose existing table branch and time-travel syntax. The server integration - must verify that both snapshot and schema lookups resolve the protected tag. The database tag - response itself does not return the source branch or a table mapping. -4. Advance the experiment tables, then repeat the tagged reads. They must return the earlier data - and schemas. Keep the tag's backing table branches while these reads are needed. + Stop writes and create database branch `experiment` from `main` using tree management. +2. Bind `restCatalog.withReference("training", "experiment")`. List and load `features` and `labels` + by their ordinary names, then write experiment data with the usual batch write API. Their + metadata reads and commits use `/trees/experiment/tables/...`. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. Bind a second + catalog with `restCatalog.withReference("training", "train_v1")`. Load the same logical table + names for training; the service resolves the pinned table versions without a source-branch hint. +4. Advance the experiment tables, then reload and read them through the tag-bound catalog. The + tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. 5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the evaluated source version. Merging the live `experiment` branch would instead include its newer - state. If both sides changed a table, inspect the conflict and deliberately choose a per-table - mode when appropriate. + state. If both sides changed a table, choose a per-table merge mode when appropriate. 6. Reload main tables and verify the published state. Resume writes separately on `main` and `experiment` and verify that neither changes the other. Merge the same source state again to check no-op behavior. Delete unused database references through tree management. ## Beyond the fixed-table MVP -A complete database view needs a defined way for ordinary table APIs to identify the selected -database branch or tag. Existing table branch suffixes identify one table branch; they do not scope -`listTables`, create-table, drop-table, or rename operations to a database reference. - -The following work remains separate from the management API: - -- Address and return the selected reference's table membership, including tables absent on `main`. -- Apply branch-local DDL to that membership, with stable table identities for rename and new - identities for drop-and-recreate. -- Resolve a database tag to its table mappings without requiring the caller to remember its source - branch; expose the frozen schema and empty-table state as well as the snapshot. -- Preserve the selected context through schema/snapshot access, catalog loaders, task serialization, - and caches. Table identifiers by ID and catalog-wide listings need equally explicit semantics. +The REST scope and client binding now identify the selected database view for table listing, +reads, commits and the ordinary create/alter/drop endpoints. A complete server namespace still +needs branch-local membership changes, stable identities across rename, and new identities for +drop-and-recreate. The fixed-table server may return `501` for unsupported scoped DDL. -This document does not select a new header, query parameter, catalog option, or endpoint for those -operations. Such an extension needs its own agreed wire contract and client integration. Database -reference management can be implemented first, and a fixed-table training workflow can use explicit -table addressing while that namespace design is settled. +Global table IDs, global listings, rename and the other deferred endpoints need explicit scope +semantics before they can be enabled on a bound client. SQL engine configuration also needs to +preserve the same binding when constructing catalogs. These additions do not require callers to +construct per-table branch names. ## Validation and implementation sequence -The existing reference tests validate HTTP paths, request bodies, authentication/configuration, -pagination, JSON compatibility, and exception propagation. Mocked success responses do not test -server branch isolation, snapshot retention, or the merge algorithm. +The reference tests validate HTTP paths, request bodies, authentication/configuration, pagination, +JSON compatibility, exception propagation and reference preservation through serialized catalogs +and tables. The OpenAPI validator checks that scoped endpoints reuse the corresponding ordinary +request and success-response structures. A stateful test fixture also uses real Paimon data files +to exercise batch writes on separate branches, frozen tag reads after source writes, and tag write +rejection. This validates client integration with a resolving server; production reference +lifecycle, snapshot retention and database merge still require server integration tests. Implement and verify in this order: 1. **Reference records and bootstrap:** create `main`, list/get/create/delete references, and protect managed table-reference names. 2. **Table orchestration:** clone populated and empty tables correctly; record baselines; route - explicit table branches through existing Paimon readers and writers. + scoped logical table names through existing Paimon readers and writers. 3. **Frozen training inputs:** pin table tags and schemas, validate repeated reads after source writes, and retain dependencies after logical reference deletion. Add empty-table coverage when that case is enabled. 4. **Merge:** verify automatic fast-forward, independent changes to different tables, same-table conflicts leaving the target unchanged, all three modes, repeated merge including `DROP`, and continued independent writes after merge. -5. **Complete database views:** settle reference-aware listing and DDL, then add membership changes - and tag-only discovery to the integration tests. +5. **Complete database views:** implement branch-local DDL storage and verify membership changes, + then extend the scoped protocol to the deferred operations as needed. A useful acceptance test uses real Paimon snapshots for two tables and exercises the workflow above against a stateful server. Passing that test establishes the fixed-table MVP; a full database-view diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index d62f8807a26e..6f2c9d18cde0 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,7 +74,7 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. -- [Database Branches and Tags](./database-versioning): experimental reference-management APIs and +- [Database Branches and Tags](./database-versioning): experimental reference management, scoped table APIs, and the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index c310bb09c1b7..cae9e3b6e18c 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -62,8 +62,9 @@ In this table, **table-scoped** means endpoints are described in the specification alongside their database-scoped counterparts. See [Database Branches and Tags](./database-versioning) for reference-management examples, merge -modes, and the server MVP design. These management APIs do not switch ordinary table requests to -a database reference automatically. +modes, and the server MVP design. Supported table operations can select a reference through +`/v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}`, using the existing request and +response structures. Java clients bind the scope with `withReference(database, reference)`. ## Partition Compatibility diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index 8e7739bb6c7b..45864e944493 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -37,6 +37,7 @@ metadata requests without bringing in the full table read/write bundle. | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | | Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | +| Tables within a database branch/tag | [Reference-scoped table usage](../concepts/rest/database-versioning#java-table-usage) | ## Dependency diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index e51eb2295da0..6eaaaa6b6794 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -225,6 +225,43 @@ function requireExactEnum(contract, schemaName, expectedValues) { function validateCatalogOpenApi() { const contract = validateCommon('rest-catalog-open-api.yaml'); + const databasePath = '/v1/{prefix}/databases/{database}'; + [ + '/tables', + '/table-details', + '/tables/{table}', + '/tables/{table}/commit', + '/tables/{table}/token', + '/tables/{table}/auth', + '/tables/{table}/snapshot', + '/tables/{table}/snapshots', + '/tables/{table}/snapshots/{version}', + '/tables/{table}/schemas', + '/tables/{table}/schemas/{version}', + ].forEach((suffix) => { + const original = contract.spec.paths[databasePath + suffix]; + const scopedPath = databasePath + '/trees/{reference}' + suffix; + const scoped = contract.spec.paths[scopedPath]; + contract.checkSpec(scoped, `Missing reference-scoped table path: ${scopedPath}`); + Object.entries(original).forEach(([method, operation]) => { + if (!HTTP_METHODS.has(method)) { + return; + } + const counterpart = scoped[method]; + contract.checkSpec(counterpart, `Missing ${method} on ${scopedPath}`); + ['requestBody', 'responses'].forEach((field) => { + const value = (op) => field === 'responses' ? op.responses['200'] : op[field]; + contract.checkSpec( + JSON.stringify(value(operation)) === JSON.stringify(value(counterpart)), + `${scopedPath} must reuse the unscoped ${method} ${field} contract`, + ); + }); + contract.requireResponses(counterpart.operationId, ['404', '501']); + if (method !== 'get' && !suffix.endsWith('/auth')) { + contract.requireResponses(counterpart.operationId, ['409']); + } + }); + }); [ 'getConfig', 'createDatabase', diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 0d9c1b85bcfb..133944ea412e 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -452,6 +452,569 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/trees/{reference}/tables: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + get: + tags: + - database-reference + summary: List tables in a database reference + operationId: listTablesInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListTablesResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + post: + tags: + - database-reference + summary: Create table in a database reference + operationId: createTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CreateTableRequest" + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/table-details: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + get: + tags: + - database-reference + summary: List table details in a database reference + operationId: listTableDetailsInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: tableType + description: Filter tables by table type. All table types will be returned if not set or empty. + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListTableDetailsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table in a database reference + operationId: getTableInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. Return the logical table name and the + resolved schema, path and storage options. Internal branch aliases may be carried in schema + options. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + post: + tags: + - database-reference + summary: Alter table in a database reference + operationId: alterTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AlterTableRequest" + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + delete: + tags: + - database-reference + summary: Drop table in a database reference + operationId: dropTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + responses: + "200": + description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/commit: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + post: + tags: + - database-reference + summary: Commit table in a database reference + operationId: commitTableInReference + description: >- + Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table + IDs in the body must agree with the logical table resolved from the path. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/CommitTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/token: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table token in a database reference + operationId: getTableTokenInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. Credentials for a tag must allow reading + without allowing mutation of retained metadata or data. + responses: + "200": + description: DataToken for visit data. + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableDataTokenResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/auth: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + post: + tags: + - database-reference + summary: Auth table query in a database reference + operationId: authTableQueryInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/AuthTableQueryRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/AuthTableQueryResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshot: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: Get table snapshot in a database reference + operationId: getTableSnapshotInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. An empty captured table returns 404 with + resourceType SNAPSHOT. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetTableSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots/{version}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + - name: version + in: path + required: true + schema: + type: string + get: + tags: + - database-reference + summary: Get version snapshot in a database reference + operationId: getVersionSnapshotInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. A tag exposes only its pinned snapshot: + LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetVersionSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: List snapshots in a database reference + operationId: listSnapshotsInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. A tag exposes only its pinned snapshot: + LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. + parameters: + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSnapshotsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + get: + tags: + - database-reference + summary: List table schemas in a database reference + operationId: listSchemasInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. LATEST selects the captured schema. Schema + history is limited to schemas retained for the captured table version; later source schemas + are not visible. + parameters: + - name: maxResults + in: query + schema: + type: integer + minimum: 0 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSchemasResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas/{version}: + description: >- + Table operations within an existing database branch or immutable tag. Uses the same request and + response schemas as the corresponding unscoped table endpoint. + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Reference" + - $ref: "#/components/parameters/Table" + - $ref: "#/components/parameters/Version" + get: + tags: + - database-reference + summary: Get table schema in a database reference + operationId: getSchemaInReference + description: >- + Resolve membership and the table version through the reference. A tag returns frozen metadata + and never the latest state of its source branch. LATEST selects the captured schema. Schema + history is limited to schemas retained for the captured table version; later source schemas + are not visible. + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetSchemaResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + /v1/{prefix}/databases/{database}/register: post: tags: @@ -1180,6 +1743,72 @@ paths: $ref: '#/components/responses/TableNotExistErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}/schemas: + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Table" + get: + tags: + - table + summary: List table schemas + operationId: listSchemas + parameters: + - name: maxResults + in: query + schema: + type: integer + minimum: 0 + - name: pageToken + in: query + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ListSchemasResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "404": + description: Table or schema does not exist. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + /v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}: + parameters: + - $ref: "#/components/parameters/Prefix" + - $ref: "#/components/parameters/Database" + - $ref: "#/components/parameters/Table" + - $ref: "#/components/parameters/Version" + get: + tags: + - table + summary: Get table schema + operationId: getSchema + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/GetSchemaResponse" + "401": + $ref: "#/components/responses/UnauthorizedErrorResponse" + "404": + description: Table or schema does not exist. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + "500": + $ref: "#/components/responses/ServerErrorResponse" + /v1/{prefix}/databases/{database}/tables/{table}/partitions: get: tags: @@ -2655,10 +3284,70 @@ paths: $ref: '#/components/responses/SemanticViewNotImplementedErrorResponse' components: + parameters: + Prefix: + name: prefix + in: path + required: true + schema: + type: string + Database: + name: database + in: path + required: true + schema: + type: string + Reference: + name: reference + in: path + required: true + schema: + type: string + pattern: ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ + description: An existing database branch or immutable tag. Never falls back to main. + Table: + name: table + in: path + required: true + schema: + type: string + description: Logical table name. A database reference does not require a table branch suffix. + Version: + name: version + in: path + required: true + schema: + type: string + description: A schema ID or LATEST; resolved within the selected table version. + ############################# # Reusable Response Objects # ############################# responses: + ReferenceTableNotExistErrorResponse: + description: >- + Database, reference, table, snapshot or schema does not exist within the selected reference. + Return resourceType SNAPSHOT for an existing table with no snapshot. Never fall back to main. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + ReferenceTableConflictErrorResponse: + description: >- + The target is an immutable tag, the table already exists, or the operation conflicts with the + selected table state. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + ReferenceTableNotImplementedErrorResponse: + description: >- + The server does not implement this reference-scoped table operation. No fallback to an unscoped + route is allowed. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" SemanticViewNotExistErrorResponse: description: Database or semantic view does not exist. content: @@ -2914,6 +3603,37 @@ components: message: Internal Server Error code: 500 schemas: + TableSchema: + allOf: + - $ref: "#/components/schemas/Schema" + - type: object + properties: + version: + type: integer + id: + type: integer + format: int64 + highestFieldId: + type: integer + timeMillis: + type: integer + format: int64 + GetSchemaResponse: + type: object + properties: + schema: + $ref: "#/components/schemas/TableSchema" + ListSchemasResponse: + type: object + properties: + schemas: + type: array + items: + $ref: "#/components/schemas/TableSchema" + nextPageToken: + type: + - string + - "null" SemanticViewDefinition: type: object required: [ format, content ] diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 845a68deb003..353ac2d605cb 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -49,12 +49,16 @@ public DatabaseReference( @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, @JsonProperty(FIELD_NAME) String name) { checkArgument(type != null, "Reference type must not be null"); - checkArgument( - name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + validateName(name); this.type = type; this.name = name; } + static void validateName(String name) { + checkArgument( + name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + } + @JsonGetter(FIELD_TYPE) public DatabaseReferenceType getType() { return type; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index decefe3e3712..723bc1e8f7db 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -257,6 +257,26 @@ public RESTApi(Options options, boolean configRequired) { this.resourcePaths = ResourcePaths.forCatalogProperties(options); } + private RESTApi(RESTApi api, ResourcePaths resourcePaths) { + this.client = api.client; + this.restAuthFunction = api.restAuthFunction; + this.options = api.options; + this.resourcePaths = resourcePaths; + } + + /** + * Returns a client whose table operations address one database branch or immutable tag. + * + *

The original client is unchanged. Table names remain logical names, without a table branch + * suffix. The server resolves the reference and enforces tag immutability. Operations without a + * reference-scoped table route are unsupported on this client; database and reference + * management retain their catalog-wide meaning. + */ + @Experimental + public RESTApi withReference(String database, String reference) { + return new RESTApi(this, resourcePaths.withReference(database, reference)); + } + /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 0b25fa3de9d2..e1b9d8615c2e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -19,11 +19,14 @@ package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.options.Options; import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; +import javax.annotation.Nullable; + import static org.apache.paimon.rest.RESTUtil.encodeString; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -64,9 +67,47 @@ public static ResourcePaths forCatalogProperties(Options options) { } private final String prefix; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; public ResourcePaths(String prefix) { - this.prefix = encodeString(prefix); + this(encodeString(prefix), null, null); + } + + private ResourcePaths( + String encodedPrefix, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this.prefix = encodedPrefix; + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; + } + + /** Returns paths for table operations within one database branch or immutable tag. */ + @Experimental + public ResourcePaths withReference(String database, String reference) { + checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); + DatabaseReference.validateName(reference); + return new ResourcePaths(prefix, database, reference); + } + + private String tableScope(String database) { + if (referenceName == null) { + return database(database); + } + checkArgument( + referenceDatabase.equals(database), + "Table operation must use reference database %s, not %s", + referenceDatabase, + database); + return databaseTree(database, referenceName); + } + + private void checkUnscoped(String operation) { + if (referenceName != null) { + throw new UnsupportedOperationException( + operation + " is not supported in a database reference scope"); + } } /** Labels attached to one entity, whose canonical name is encoded as a single segment. */ @@ -127,6 +168,7 @@ public String revokePermission() { /** Policy collection nested below its attachment resource. */ @Experimental public String policies(PermissionResource resource) { + checkUnscoped("policies"); resource.validatePolicyAttachment(); return SLASH.join(table(resource.getDatabase(), resource.getTable()), POLICIES); } @@ -164,36 +206,38 @@ public String mergeDatabaseBranch(String databaseName, String branch) { } public String tables(String databaseName) { - return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLES); + return SLASH.join(tableScope(databaseName), TABLES); } public String tableDetails(String databaseName) { - return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), TABLE_DETAILS); + return SLASH.join(tableScope(databaseName), TABLE_DETAILS); } public String tables() { + checkUnscoped("tables"); return SLASH.join(V1, prefix, TABLES); } public String table(String tableId) { + checkUnscoped("table"); return SLASH.join(V1, prefix, TABLES, ID, encodeString(tableId)); } public String table(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName)); + checkArgument( + referenceName == null + || Identifier.create(databaseName, objectName).getBranchName() == null, + "Table branch suffixes cannot be combined with a database reference"); + return SLASH.join(tables(databaseName), encodeString(objectName)); } public String renameTable() { + checkUnscoped("renameTable"); return SLASH.join(V1, prefix, TABLES, "rename"); } public String replaceTable(String databaseName, String objectName) { + checkUnscoped("replaceTable"); return SLASH.join( V1, prefix, @@ -205,17 +249,11 @@ public String replaceTable(String databaseName, String objectName) { } public String commitTable(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "commit"); + return SLASH.join(table(databaseName, objectName), "commit"); } public String rollbackTable(String databaseName, String objectName) { + checkUnscoped("rollbackTable"); return SLASH.join( V1, prefix, @@ -227,6 +265,7 @@ public String rollbackTable(String databaseName, String objectName) { } public String rollbackSchemaTable(String databaseName, String objectName) { + checkUnscoped("rollbackSchemaTable"); return SLASH.join( V1, prefix, @@ -238,63 +277,28 @@ public String rollbackSchemaTable(String databaseName, String objectName) { } public String registerTable(String databaseName) { + checkUnscoped("registerTable"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), REGISTER); } public String tableToken(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "token"); + return SLASH.join(table(databaseName, objectName), "token"); } public String tableSnapshot(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "snapshot"); + return SLASH.join(table(databaseName, objectName), "snapshot"); } public String tableSnapshot(String databaseName, String objectName, String version) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SNAPSHOTS, - version); + return SLASH.join(snapshots(databaseName, objectName), encodeString(version)); } public String snapshots(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SNAPSHOTS); + return SLASH.join(table(databaseName, objectName), SNAPSHOTS); } public String schemas(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - SCHEMAS); + return SLASH.join(table(databaseName, objectName), SCHEMAS); } public String schemas(String databaseName, String objectName, String version) { @@ -302,17 +306,11 @@ public String schemas(String databaseName, String objectName, String version) { } public String authTable(String databaseName, String objectName) { - return SLASH.join( - V1, - prefix, - DATABASES, - encodeString(databaseName), - TABLES, - encodeString(objectName), - "auth"); + return SLASH.join(table(databaseName, objectName), "auth"); } public String partitions(String databaseName, String objectName) { + checkUnscoped("partitions"); return SLASH.join( V1, prefix, @@ -324,6 +322,7 @@ public String partitions(String databaseName, String objectName) { } public String dropPartitions(String databaseName, String objectName) { + checkUnscoped("dropPartitions"); return SLASH.join( V1, prefix, @@ -336,6 +335,7 @@ public String dropPartitions(String databaseName, String objectName) { } public String markDonePartitions(String databaseName, String objectName) { + checkUnscoped("markDonePartitions"); return SLASH.join( V1, prefix, @@ -348,6 +348,7 @@ public String markDonePartitions(String databaseName, String objectName) { } public String listPartitionsByNames(String databaseName, String objectName) { + checkUnscoped("listPartitionsByNames"); return SLASH.join( V1, prefix, @@ -360,6 +361,7 @@ public String listPartitionsByNames(String databaseName, String objectName) { } public String listPartitionsByFilter(String databaseName, String objectName) { + checkUnscoped("listPartitionsByFilter"); return SLASH.join( V1, prefix, @@ -372,6 +374,7 @@ public String listPartitionsByFilter(String databaseName, String objectName) { } public String branches(String databaseName, String objectName) { + checkUnscoped("branches"); return SLASH.join( V1, prefix, @@ -383,6 +386,7 @@ public String branches(String databaseName, String objectName) { } public String branch(String databaseName, String objectName, String branchName) { + checkUnscoped("branch"); return SLASH.join( V1, prefix, @@ -395,6 +399,7 @@ public String branch(String databaseName, String objectName, String branchName) } public String forwardBranch(String databaseName, String tableName, String branch) { + checkUnscoped("forwardBranch"); return SLASH.join( V1, prefix, @@ -408,6 +413,7 @@ public String forwardBranch(String databaseName, String tableName, String branch } public String tags(String databaseName, String objectName) { + checkUnscoped("tags"); return SLASH.join( V1, prefix, @@ -419,6 +425,7 @@ public String tags(String databaseName, String objectName) { } public String consumers(String databaseName, String objectName) { + checkUnscoped("consumers"); return SLASH.join( V1, prefix, @@ -430,6 +437,7 @@ public String consumers(String databaseName, String objectName) { } public String resetConsumer(String databaseName, String objectName) { + checkUnscoped("resetConsumer"); return SLASH.join( V1, prefix, @@ -442,6 +450,7 @@ public String resetConsumer(String databaseName, String objectName) { } public String tag(String databaseName, String objectName, String tagName) { + checkUnscoped("tag"); return SLASH.join( V1, prefix, diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 58ffc5edb518..e830e40a1011 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -115,6 +115,8 @@ public class RESTCatalog implements Catalog { private final RESTApi api; private final CatalogContext context; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; private final boolean dataTokenEnabled; protected final Map tableDefaultOptions; private final @Nullable LocalCacheManager cacheManager; @@ -124,7 +126,30 @@ public RESTCatalog(CatalogContext context) { } public RESTCatalog(CatalogContext context, boolean configRequired) { - this.api = new RESTApi(context.options(), configRequired); + this(context, configRequired, null, null); + } + + RESTCatalog( + CatalogContext context, + boolean configRequired, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this( + context, + new RESTApi(context.options(), configRequired), + referenceDatabase, + referenceName); + } + + private RESTCatalog( + CatalogContext context, + RESTApi api, + @Nullable String referenceDatabase, + @Nullable String referenceName) { + this.api = + referenceName == null ? api : api.withReference(referenceDatabase, referenceName); + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; this.context = CatalogContext.create( api.options(), @@ -143,7 +168,20 @@ public Map options() { @Override public RESTCatalogLoader catalogLoader() { - return new RESTCatalogLoader(context); + return new RESTCatalogLoader(context, referenceDatabase, referenceName); + } + + /** + * Returns a separate catalog whose table operations use one database branch or immutable tag. + * + *

The binding is preserved by {@link #catalogLoader()}. Use ordinary logical table names; + * the server resolves their backing versions. No additional configuration request is made. + * Database and reference management are not versioned by this binding. + */ + @Experimental + public RESTCatalog withReference(String database, String reference) { + DatabaseReference.validateName(reference); + return new RESTCatalog(context, api, database, reference); } @Experimental @@ -533,6 +571,16 @@ public boolean commitSnapshot( Snapshot snapshot, List statistics) throws TableNotExistException { + // CatalogSnapshotCommit supplies the physical storage branch. A database reference + // already selects the write target, so keep its logical table name on the wire. + if (referenceName != null && identifier.getBranchName() != null) { + identifier = + new Identifier( + identifier.getDatabaseName(), + identifier.getTableName(), + null, + identifier.getSystemTableName()); + } try { return api.commitSnapshot( identifier, tableUuid, baseSnapshotUuid, snapshot, statistics); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java index efc5a0b46ca4..05aa369eaa4d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java @@ -21,15 +21,28 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogLoader; +import javax.annotation.Nullable; + /** Loader to create {@link RESTCatalog}. */ public class RESTCatalogLoader implements CatalogLoader { private static final long serialVersionUID = 1L; private final CatalogContext context; + @Nullable private final String referenceDatabase; + @Nullable private final String referenceName; public RESTCatalogLoader(CatalogContext context) { + this(context, null, null); + } + + RESTCatalogLoader( + CatalogContext context, + @Nullable String referenceDatabase, + @Nullable String referenceName) { this.context = context; + this.referenceDatabase = referenceDatabase; + this.referenceName = referenceName; } public CatalogContext context() { @@ -38,6 +51,6 @@ public CatalogContext context() { @Override public RESTCatalog load() { - return new RESTCatalog(context, false); + return new RESTCatalog(context, false, referenceDatabase, referenceName); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java new file mode 100644 index 000000000000..9ab1269b467f --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.SnapshotCommit; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.options.Options; +import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.requests.CommitTableRequest; +import org.apache.paimon.rest.responses.GetSchemaResponse; +import org.apache.paimon.rest.responses.GetTableResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.SnapshotManager; + +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.apache.paimon.CoreOptions.BRANCH; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies reference scope through ordinary table APIs, serialization and storage commits. */ +class RESTCatalogReferenceTest { + + private static final String DATABASE = "training db"; + private static final Identifier TABLE = Identifier.create(DATABASE, "features"); + private static final String DATABASE_PATH = "/v1/catalog%2Fid/databases/training+db"; + private static final String SNAPSHOT_JSON = + "{\"version\":3,\"id\":7,\"schemaId\":2,\"uuid\":\"snapshot-7\"," + + "\"commitKind\":\"APPEND\",\"commitUser\":\"writer\",\"commitIdentifier\":1," + + "\"timeMillis\":1000,\"totalRecordCount\":3,\"deltaRecordCount\":3}"; + + @TempDir Path tempDir; + + private MockWebServer server; + private RESTCatalog catalog; + + @BeforeEach + void setUp() throws Exception { + server = new MockWebServer(); + server.start(); + enqueue( + 200, + "{\"defaults\":{},\"overrides\":{\"prefix\":\"catalog/id\"," + + "\"header.X-Catalog-Context\":\"configured\"}}"); + Options options = new Options(); + options.set(URI, server.url("/").toString()); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "test-token"); + catalog = new RESTCatalog(CatalogContext.create(options)); + assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()).isEqualTo("/v1/config"); + } + + @AfterEach + void tearDown() throws Exception { + catalog.close(); + server.shutdown(); + } + + @ParameterizedTest + @ValueSource(strings = {"experiment", "train_v1"}) + void testTableAndSerializedLoaderKeepReference(String reference) throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, reference); + String scope = DATABASE_PATH + "/trees/" + reference; + enqueue(200, "{\"tables\":[\"features\",\"labels\"]}"); + assertThat(scoped.listTables(DATABASE)).containsExactly("features", "labels"); + takeRequest("GET", scope + "/tables"); + + enqueue(200, tableResponse("physical-experiment")); + FileStoreTable table = (FileStoreTable) scoped.getTable(TABLE); + assertThat(table.catalogEnvironment().identifier()).isEqualTo(TABLE); + assertThat(table.snapshotManager().branch()).isEqualTo("physical-experiment"); + assertThat(table.schema().id()).isEqualTo(2); + takeRequest("GET", scope + "/tables/features"); + + // A task receives a serialized table. Its snapshot loader must still address the tree. + FileStoreTable restored = InstantiationUtil.clone(table); + enqueue(200, "{\"snapshot\":{\"snapshot\":" + SNAPSHOT_JSON + "}}"); + assertThat(restored.snapshotManager().latestSnapshot().id()).isEqualTo(7); + takeRequest("GET", scope + "/tables/features/snapshot"); + + RESTCatalog loaded = InstantiationUtil.clone(scoped.catalogLoader()).load(); + enqueue( + 200, + RESTApi.toJson( + new GetSchemaResponse( + TableSchema.create(2, schema("physical-experiment"))))); + assertThat(loaded.loadSchema(TABLE, "LATEST").get().id()).isEqualTo(2); + takeRequest("GET", scope + "/tables/features/schemas/LATEST"); + + // Binding another catalog must not change the original catalog's route or metadata. + enqueue(200, tableResponse("main")); + FileStoreTable main = (FileStoreTable) catalog.getTable(TABLE); + assertThat(main.snapshotManager().branch()).isEqualTo("main"); + takeRequest("GET", DATABASE_PATH + "/tables/features"); + assertThat(server.getRequestCount()).isEqualTo(6); + } + + @Test + void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, "experiment"); + enqueue(200, tableResponse("physical-experiment")); + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + takeRequest("GET", DATABASE_PATH + "/trees/experiment/tables/features"); + + Snapshot snapshot = Snapshot.fromJson(SNAPSHOT_JSON); + enqueue(200, "{\"success\":true}"); + try (SnapshotCommit commit = + table.catalogEnvironment().snapshotCommit(table.snapshotManager())) { + assertThat( + commit.commit( + "snapshot-6", + snapshot, + table.snapshotManager().branch(), + emptyList())) + .isTrue(); + } + RecordedRequest request = + takeRequest("POST", DATABASE_PATH + "/trees/experiment/tables/features/commit"); + CommitTableRequest body = + RESTApi.fromJson(request.getBody().readUtf8(), CommitTableRequest.class); + assertThat(body.getTableId()).isEqualTo("table-id"); + assertThat(body.getBaseSnapshotUuid()).isEqualTo("snapshot-6"); + assertThat(body.getSnapshot()).isEqualTo(snapshot); + assertThat(body.getStatistics()).isEmpty(); + } + + @Test + void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { + RESTApi scoped = catalog.api().withReference(DATABASE, "train_v1"); + String scope = DATABASE_PATH + "/trees/train_v1"; + String tablePath = scope + "/tables/features"; + enqueue(200, "{\"tables\":[\"features\"],\"nextPageToken\":\"next\"}"); + assertThat(scoped.listTablesPaged(DATABASE, 1, null, "feat%", null).getNextPageToken()) + .isEqualTo("next"); + RecordedRequest first = takeRequest("GET", scope + "/tables"); + assertThat(first.getRequestUrl().queryParameter("tableNamePattern")).isEqualTo("feat%"); + enqueue(200, "{\"tables\":[\"labels\"]}"); + assertThat(scoped.listTablesPaged(DATABASE, 1, "next", null, null).getElements()) + .containsExactly("labels"); + assertThat( + takeRequest("GET", scope + "/tables") + .getRequestUrl() + .queryParameter("pageToken")) + .isEqualTo("next"); + + enqueue(200, "{\"tableDetails\":[" + tableResponse("physical-experiment") + "]}"); + assertThat(scoped.listTableDetails(DATABASE).get(0).getName()).isEqualTo("features"); + takeRequest("GET", scope + "/table-details"); + + enqueue(200, "{\"snapshot\":" + SNAPSHOT_JSON + "}"); + assertThat(scoped.loadSnapshot(TABLE, "LATEST").id()).isEqualTo(7); + takeRequest("GET", tablePath + "/snapshots/LATEST"); + enqueue(200, "{\"snapshots\":[" + SNAPSHOT_JSON + "]}"); + assertThat(scoped.listSnapshotsPaged(TABLE, 10, null).getElements().get(0).id()) + .isEqualTo(7); + takeRequest("GET", tablePath + "/snapshots"); + + TableSchema schema = TableSchema.create(2, schema("physical-experiment")); + enqueue(200, "{\"schemas\":[" + RESTApi.toJson(schema) + "]}"); + assertThat(scoped.listSchemasPaged(TABLE, 10, null).getElements()).containsExactly(schema); + takeRequest("GET", tablePath + "/schemas"); + + enqueue(200, "{\"token\":{\"key\":\"value\"},\"expiresAtMillis\":1234}"); + assertThat(scoped.loadTableToken(TABLE).getToken()).containsEntry("key", "value"); + takeRequest("GET", tablePath + "/token"); + enqueue(200, "{\"filter\":[],\"columnMasking\":{}}"); + scoped.authTableQuery(TABLE, singletonList("id")); + assertThat(takeRequest("POST", tablePath + "/auth").getBody().readUtf8()) + .isEqualTo("{\"select\":[\"id\"]}"); + } + + @Test + void testTableMutationsReuseRequestBodies() throws Exception { + RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); + for (RESTApi api : new RESTApi[] {catalog.api(), scoped}) { + enqueue(200, "{}"); + api.createTable(TABLE, schema("main")); + enqueue(200, "{}"); + api.alterTable(TABLE, singletonList(SchemaChange.setOption("key", "value"))); + enqueue(200, "{}"); + api.dropTable(TABLE); + } + RecordedRequest[] original = { + takeRequest("POST", DATABASE_PATH + "/tables"), + takeRequest("POST", DATABASE_PATH + "/tables/features"), + takeRequest("DELETE", DATABASE_PATH + "/tables/features") + }; + String scope = DATABASE_PATH + "/trees/experiment"; + RecordedRequest[] referenced = { + takeRequest("POST", scope + "/tables"), + takeRequest("POST", scope + "/tables/features"), + takeRequest("DELETE", scope + "/tables/features") + }; + for (int i = 0; i < original.length; i++) { + assertThat(referenced[i].getBody().readUtf8()) + .isEqualTo(original[i].getBody().readUtf8()); + } + } + + @Test + void testErrorsDoNotFallBackToDefaultBranch() throws Exception { + RESTCatalog scoped = catalog.withReference(DATABASE, "train_v1"); + enqueue(404, "{\"message\":\"reference missing\",\"code\":404}"); + assertThatThrownBy(() -> scoped.getTable(TABLE)) + .isInstanceOf(Catalog.TableNotExistException.class); + takeRequest("GET", DATABASE_PATH + "/trees/train_v1/tables/features"); + enqueue(409, "{\"message\":\"tag is immutable\",\"code\":409}"); + assertThatThrownBy( + () -> + scoped.commitSnapshot( + TABLE, + "table-id", + null, + Snapshot.fromJson(SNAPSHOT_JSON), + emptyList())) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("tag is immutable"); + takeRequest("POST", DATABASE_PATH + "/trees/train_v1/tables/features/commit"); + assertThat(server.getRequestCount()).isEqualTo(3); + } + + @Test + void testUnsupportedSelectorsNeverSendAnUnscopedRequest() { + RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); + assertThatThrownBy(() -> catalog.withReference(DATABASE, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> scoped.withReference(DATABASE, "../main")) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> scoped.listTables("other")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(DATABASE); + assertThatThrownBy(() -> scoped.getTable(new Identifier(DATABASE, "features", "other"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("branch suffixes"); + assertThatThrownBy(() -> scoped.getTableById("table-id")) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.listTablesPagedGlobally(null, null, null, null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.renameTable(TABLE, Identifier.create(DATABASE, "renamed"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> scoped.createBranch(TABLE, "nested", null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThat(server.getRequestCount()).isEqualTo(1); + } + + @Test + void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { + // A small stateful fixture resolves references; production reference lifecycle is separate. + org.apache.paimon.fs.Path location = + new org.apache.paimon.fs.Path(tempDir.resolve("features").toUri()); + LocalFileIO fileIO = LocalFileIO.create(); + for (String branch : new String[] {"main", "physical-experiment"}) { + new FileSystemSchemaManager(fileIO, location, branch).createTable(schema(branch)); + } + Map snapshots = new ConcurrentHashMap<>(); + ConcurrentLinkedQueue unexpected = new ConcurrentLinkedQueue<>(); + server.setDispatcher( + new Dispatcher() { + @Override + public MockResponse dispatch(RecordedRequest request) { + try { + String route = request.getRequestUrl().encodedPath(); + String prefix = DATABASE_PATH + "/trees/"; + if (!route.startsWith(prefix)) { + unexpected.add(route); + return response(500, "{}"); + } + String[] parts = route.substring(prefix.length()).split("/"); + String reference = parts[0]; + if (parts.length < 3 + || !parts[1].equals("tables") + || !parts[2].equals("features")) { + unexpected.add(route); + return response(500, "{}"); + } + String branch = + reference.equals("main") ? "main" : "physical-experiment"; + if (request.getMethod().equals("GET") && parts.length == 3) { + return response(200, tableResponse(branch, 0)); + } + if (request.getMethod().equals("GET") + && parts.length == 4 + && parts[3].equals("snapshot")) { + Snapshot snapshot = snapshots.get(reference); + return snapshot == null + ? response( + 404, + "{\"code\":404,\"resourceType\":\"SNAPSHOT\",\"message\":\"empty table\"}") + : response( + 200, + "{\"snapshot\":{\"snapshot\":" + + snapshot.toJson() + + "}}"); + } + if (request.getMethod().equals("POST") + && parts.length == 4 + && parts[3].equals("commit")) { + if (reference.equals("train_v1")) { + return response( + 409, "{\"code\":409,\"message\":\"tag is immutable\"}"); + } + CommitTableRequest commit = + RESTApi.fromJson( + request.getBody().readUtf8(), + CommitTableRequest.class); + Snapshot snapshot = commit.getSnapshot(); + fileIO.overwriteFileUtf8( + new SnapshotManager(fileIO, location, branch, null, null) + .snapshotPath(snapshot.id()), + snapshot.toJson()); + snapshots.put(reference, snapshot); + return response(200, "{\"success\":true}"); + } + unexpected.add(route); + return response(500, "{}"); + } catch (Exception e) { + unexpected.add(e.toString()); + return response(500, "{}"); + } + } + }); + + RESTCatalog main = catalog.withReference(DATABASE, "main"); + RESTCatalog experiment = catalog.withReference(DATABASE, "experiment"); + writeRows(main, 10); + writeRows(experiment, 20); + snapshots.put("train_v1", snapshots.get("experiment")); + RESTCatalog tag = catalog.withReference(DATABASE, "train_v1"); + assertThat(readRows(tag)).containsExactly(20); + + writeRows(experiment, 30); + assertThat(readRows(main)).containsExactly(10); + assertThat(readRows(experiment)).containsExactlyInAnyOrder(20, 30); + // A newly loaded tag table must not follow the source branch's latest snapshot. + assertThat(readRows(tag)).containsExactly(20); + assertThatThrownBy(() -> writeRows(tag, 99)).hasStackTraceContaining("tag is immutable"); + assertThat(readRows(tag)).containsExactly(20); + assertThat(snapshots.get("train_v1").id()).isEqualTo(1); + assertThat(snapshots.get("experiment").id()).isEqualTo(2); + assertThat(unexpected).isEmpty(); + } + + private void writeRows(RESTCatalog scoped, int value) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + write.write(GenericRow.of(value)); + commit.commit(write.prepareCommit()); + } + } + + private List readRows(RESTCatalog scoped) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + ReadBuilder builder = table.newReadBuilder(); + List rows = new ArrayList<>(); + try (RecordReader reader = + builder.newRead().createReader(builder.newScan().plan().splits())) { + reader.forEachRemaining(row -> rows.add(row.getInt(0))); + } + return rows; + } + + private Schema schema(String branch) { + return Schema.newBuilder() + .column("id", DataTypes.INT()) + .option("bucket", "-1") + .option(BRANCH.key(), branch) + .build(); + } + + private String tableResponse(String branch) throws Exception { + return tableResponse(branch, 2); + } + + private String tableResponse(String branch, long schemaId) throws Exception { + return RESTApi.toJson( + new GetTableResponse( + "table-id", + DATABASE, + "features", + tempDir.resolve("features").toUri().toString(), + false, + schemaId, + schema(branch), + null, + 0, + null, + 0, + null)); + } + + private void enqueue(int status, String body) { + server.enqueue(response(status, body)); + } + + private MockResponse response(int status, String body) { + return new MockResponse() + .setResponseCode(status) + .setHeader("Content-Type", "application/json") + .setBody(body); + } + + private RecordedRequest takeRequest(String method, String path) throws Exception { + RecordedRequest request = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(request).isNotNull(); + assertThat(request.getMethod()).isEqualTo(method); + assertThat(request.getRequestUrl().encodedPath()).isEqualTo(path); + assertThat(request.getHeader("Authorization")).isEqualTo("Bearer test-token"); + assertThat(request.getHeader("X-Catalog-Context")).isEqualTo("configured"); + assertThat(request.getHeader("Paimon-Reference")).isNull(); + return request; + } +} From 217b3de0c6fe16b0ba3ed8e821df956caa79d201 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 16 Sep 2026 22:15:45 +0800 Subject: [PATCH 10/10] [rest] Select database references through name suffixes --- .../docs/concepts/rest/database-versioning.md | 219 +-- docs/docs/concepts/rest/index.md | 2 +- docs/docs/concepts/rest/rest-api.md | 6 +- docs/docs/program-api/rest-api.mdx | 2 +- docs/scripts/validate-rest-openapi.js | 50 +- docs/static/rest-catalog-open-api.yaml | 1295 ++++++----------- .../paimon/rest/DatabaseIdentifier.java | 94 ++ .../apache/paimon/rest/DatabaseReference.java | 8 +- .../java/org/apache/paimon/rest/RESTApi.java | 28 +- .../org/apache/paimon/rest/ResourcePaths.java | 98 +- .../paimon/rest/DatabaseIdentifierTest.java | 81 ++ .../org/apache/paimon/rest/RESTCatalog.java | 50 +- .../apache/paimon/rest/RESTCatalogLoader.java | 15 +- .../paimon/rest/RESTCatalogReferenceTest.java | 189 ++- 14 files changed, 925 insertions(+), 1212 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java diff --git a/docs/docs/concepts/rest/database-versioning.md b/docs/docs/concepts/rest/database-versioning.md index eef723837b83..af694d295358 100644 --- a/docs/docs/concepts/rest/database-versioning.md +++ b/docs/docs/concepts/rest/database-versioning.md @@ -33,14 +33,15 @@ reuses Paimon's existing [table branches](../../maintenance/manage-branches) and :::info Implementation status -The Java reference-management client, reference-scoped table client, and their wire contracts are -implemented. Reference storage, table-level orchestration, and database merge execution must be +The Java reference-management client, database-name selector parser, and their wire contracts are +implemented. Ordinary table APIs carry the selector in the database name. Reference storage, table-level orchestration, and database merge execution must be implemented by the catalog server. The server implementation below is a design, not a claim that an existing service supports it. -Table operations select a database reference through `/trees/{reference}` in the resource path. -Callers use logical table names without constructing table branch suffixes or remembering a tag's -source branch. This requires no new reference header or catalog option. +Table operations select a database reference through a `$branch_` or `$tag_` suffix on +the database name. The existing table paths and request/response structures are reused. Callers use +ordinary table names without remembering a tag's source branch. No reference header, catalog +option, or separately bound client is needed. ::: @@ -201,39 +202,76 @@ individual operations and their documented responses. ## Reference-scoped table API -Let `S = /v1/{prefix}/databases/{database}/trees/{reference}`. The reference name selects either -an existing branch or an immutable tag. It is resolved by the server; the client need not first -fetch its type. These endpoints reuse the ordinary table request and response structures: +A database name can include exactly one reference selector: -| Method and path | Existing request / response | Scope | -| --- | --- | --- | -| `GET S/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | -| `GET S/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | -| `GET S/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | -| `POST S/tables` | `CreateTableRequest`. | Create a table in a branch. | -| `POST S/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | -| `DELETE S/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | -| `GET S/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | -| `GET S/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | -| `GET S/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | -| `GET S/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | -| `GET S/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | -| `POST S/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | -| `GET S/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | -| `POST S/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | - -For example, read the same logical table through a live experiment and a frozen training tag: +| Database name | Meaning | +| --- | --- | +| `training` | The ordinary physical database, with its existing main-table behavior. | +| `training$branch_experiment` | The writable database branch `experiment`. | +| `training$branch_main` | Explicit selection of the database branch `main`. | +| `training$tag_train_v1` | The immutable database tag `train_v1`. | + +The selector is carried in the existing database field, including inside `Identifier`. Encode the +complete database name once as one REST path segment. JSON names remain decoded. For example: ```http -GET /v1/catalog/databases/training/trees/experiment/tables/features -GET /v1/catalog/databases/training/trees/train_v1/tables/features +GET /v1/catalog/databases/training%24branch_experiment/tables/features +GET /v1/catalog/databases/training%24tag_train_v1/tables/features +POST /v1/catalog/databases/training%24branch_experiment/tables/features/commit ``` -The response name remains `features`. `GetTableResponse` carries the resolved schema, path and -storage options; the server may supply an internal physical branch in the existing schema options. -A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and `statistics` fields. -The reference path determines the target; identifiers and table IDs in the request must agree with -the table resolved from that path. Caller-supplied table branch suffixes are not part of this contract. +There are no additional table routes below `/trees/{reference}`. `/trees` remains the reference +management resource and always takes the physical database name, such as `training`. + +Let `D = /v1/{prefix}/databases/{database}` below, where `database` may carry a reference suffix. +These are the existing operations and request/response structures: + +| Method and path | Existing request / response | Scope | +| --- | --- | --- | +| `GET D` | `GetDatabaseResponse`. | Validate the database and selected reference; return virtual database metadata. | +| `GET D/tables` | `ListTablesResponse`; existing paging/filter query parameters. | Table membership of the reference. | +| `GET D/table-details` | `ListTableDetailsResponse`; existing paging/filter query parameters. | Table definitions within the reference. | +| `GET D/tables/{table}` | `GetTableResponse`. | Selected schema, storage options and path. | +| `POST D/tables` | `CreateTableRequest`. | Create a table in a branch. | +| `POST D/tables/{table}` | `AlterTableRequest`. | Alter a table in a branch. | +| `DELETE D/tables/{table}` | Existing drop-table response. | Remove a table from a branch. | +| `GET D/tables/{table}/snapshot` | `GetTableSnapshotResponse`. | Current branch snapshot or pinned tag snapshot. | +| `GET D/tables/{table}/snapshots/{version}` | `GetVersionSnapshotResponse`. | Resolve a version within this reference. | +| `GET D/tables/{table}/snapshots` | `ListSnapshotsResponse`; existing pagination. | Snapshot history visible through this reference. | +| `GET D/tables/{table}/schemas/{version}` | `GetSchemaResponse`. | Resolve a schema ID or `LATEST` within this reference. | +| `GET D/tables/{table}/schemas` | `ListSchemasResponse`; existing pagination. | Schema history retained for this reference. | +| `POST D/tables/{table}/commit` | `CommitTableRequest` / `CommitTableResponse`. | Commit a snapshot to the selected branch. | +| `GET D/tables/{table}/token` | `GetTableTokenResponse`. | Credentials for the resolved table version. | +| `POST D/tables/{table}/auth` | `AuthTableQueryRequest` / `AuthTableQueryResponse`. | Authorize a read of the resolved table. | + +`GetTableResponse` retains the requested database name including its suffix and the logical table +name, such as `features`. It carries the resolved schema, path and storage options; the server may +supply a physical branch alias through existing schema options. Request identifiers retain the +same full database name. A commit keeps the existing `tableId`, `baseSnapshotUuid`, `snapshot`, and +`statistics` fields. The path selects the reference; request identifiers and table IDs must agree +with the resolved table. + +### Database lookup and naming rules + +`GET database` must resolve a suffixed name, because SQL engines can check namespace existence +before accessing a table. The response represents the virtual database and retains its full name. +Database listing returns physical database names only; use `/trees` to discover branches and tags. + +CREATE, DROP and ALTER DATABASE do not accept reference suffixes. In particular, dropping a +virtual database must never drop its physical database. Create, delete and merge references through +`/databases/training/trees` instead. This does not prevent ordinary create/alter/drop **table** +operations from modifying membership or metadata in a writable branch. + +The markers `$branch_` and `$tag_` are case-sensitive reserved syntax. The base database must be +nonblank, and the reference follows the name rules above. Missing names, multiple selectors, or +invalid reference names are rejected rather than interpreted as literal database names. Other +uses of `$`, such as `training$archive`, remain literal. Catalogs adopting this contract must resolve +any pre-existing physical database names containing the reserved markers before enabling it; +lookup must not switch between literal and reference meanings based on which object exists. + +Caller-supplied table branch suffixes cannot be combined with a database selector. For example, +`training$branch_a.features$branch_b` is rejected. Storage commits can supply a physical table branch +internally; RESTCatalog removes that internal table suffix while preserving the database selector. ### Branch and tag behavior @@ -249,57 +287,68 @@ The server rejects content changes through a tag with `409`. Read authorization through `POST .../auth`; HTTP method alone does not determine whether an operation is a write. Tag credentials must permit reading without allowing mutation of retained metadata or data. -Missing references and tables return `404`. Unsupported scoped operations return `501`, without -falling back to the ordinary main-table path. Existing unscoped URLs retain their behavior. +Missing databases, references and tables return `404`. A selector whose type does not match the +reference, such as `$branch_train_v1` for a tag, returns `409`. Malformed selectors return `400`. +Unsupported operations on references return `501`. None of these errors permits retrying the +request against the physical database without its suffix. ### Java table usage -Bind a separate client instance to the desired database reference: +Use the same catalog for ordinary databases and any number of database references: ```java import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.rest.RESTCatalog; import org.apache.paimon.table.Table; -RESTCatalog experimentCatalog = restCatalog.withReference("training", "experiment"); -RESTCatalog trainingCatalog = restCatalog.withReference("training", "train_v1"); +Identifier experiment = Identifier.create("training$branch_experiment", "features"); +Identifier trainingTag = Identifier.create("training$tag_train_v1", "features"); -Table experimentFeatures = experimentCatalog.getTable(Identifier.create("training", "features")); -Table trainingFeatures = trainingCatalog.getTable(Identifier.create("training", "features")); +Table experimentFeatures = restCatalog.getTable(experiment); +Table trainingFeatures = restCatalog.getTable(trainingTag); +restCatalog.listTables("training$branch_experiment"); +restCatalog.getDatabase("training$tag_train_v1"); // Use experimentFeatures with the ordinary Paimon batch write API. // Use trainingFeatures with the ordinary Paimon read API. ``` -`withReference` leaves the original catalog unchanged and does not fetch catalog configuration -again. Each returned catalog keeps its own binding and local caches. Its serialized -`RESTCatalogLoader`, and loaders inside serialized table objects, retain the binding for later -snapshot reads, commits, schema changes and token requests. Storage commits can supply a physical -table branch internally; the bound catalog sends the logical table name and keeps the reference -path authoritative. +`RESTApi` uses these same identifiers with its existing table methods. `Identifier` already retains +the full database name through serialization and in table loaders; no extra reference fields are +stored in RESTCatalog or RESTCatalogLoader. Subsequent snapshot reads, schema changes, commits, +auth and token requests carry the same database name. Caches keyed by full table identifiers +naturally distinguish the physical database, branches and tags. + +SQL clients can pass the selector as a quoted database name, using their ordinary identifier +quoting rules. For example: + +```sql +SELECT * FROM `training$branch_experiment`.features; +SELECT * FROM `training$tag_train_v1`.features; +``` -The lightweight client supports the same binding through -`RESTApi.withReference("training", "experiment")`. Existing table methods and DTOs remain usable. -Table operations must use the bound database. Binding is currently a Java API, not a SQL catalog -option; engine configuration for selecting a reference is additional integration work. +A REST server implementing virtual database lookup and table resolution is required. There is no +new engine catalog option or reference-switch operation. -The scoped client does not yet support global table listing, lookup by table ID, rename, register, -replace, rollback, partition/consumer endpoints, or nested table branch/tag management. Such calls -fail locally instead of reaching an unscoped table route. Database and reference management, -functions, views and catalog-level management retain their existing meaning; this binding versions -only the supported table endpoints. Table policy endpoints are also outside the scoped MVP. +Rename, register, replace, rollback, partition/consumer endpoints, nested table branch/tag +management, views, functions and table policies do not yet accept database reference suffixes in +the Java client. Global table listing and lookup by table ID retain their physical-catalog meaning; +they have no database selector. Extending those operations to discover or address references is +additional work. Catalog-level permissions and reference management continue to use physical names. ### Server routing and reuse -Resolve `(database, reference, logical table)` once into an internal request context containing -reference type, table identity and backing table version. Pass that context into the existing table -handlers. Validate authentication against the actual scoped request, and authorize access to the -resolved table. A path rewrite alone is insufficient: listing must use the selected membership, -tags need frozen metadata, and commits must update the selected branch's recorded table state. +Decode the database path segment and parse it with `DatabaseIdentifier.parse(name)`. The result +contains the physical database name and an optional typed `DatabaseReference`. Resolve that +reference and the logical table once into a request context with table identity and backing version, +then reuse the existing table handlers. Validate authentication against the actual request path +and authorize access to the resolved table. Preserve the full requested database name in returned +identifiers so follow-up calls stay on the same reference. -The additional routing and DTO work is small. Runtime work is a reference/table mapping lookup, -which can be cached; adding the scope does not require proxying or copying table data. Reference -creation, retention, namespace changes and merge still require the server orchestration below. +Parsing the suffix does not replace reference management: listing still needs the selected +membership, tags need frozen metadata, and commits must update the selected branch's recorded +table state. The additional request cost is a reference/table mapping lookup, which can be cached; +this addressing scheme does not require proxying or copying table data. The storage and merge work +remains the server orchestration described below. ## Java management usage @@ -489,9 +538,9 @@ without that adapter is insufficient. Preparing fresh backing branches and publishing a new mapping is one possible server implementation. The server can update the reference mapping to those prepared versions while retaining any -physical branches needed by tags or merge baselines. The client-visible scoped table address must -continue to resolve correctly. In either case, source and target must remain independently writable: pointing both at -the same mutable table branch would make future source writes modify the target as well. +physical branches needed by tags or merge baselines. The client-visible table address must continue +to resolve correctly. Source and target must remain independently writable: pointing both at the +same mutable table branch would make future source writes modify the target as well. ### Repeated merge @@ -509,19 +558,19 @@ an HTTP success must mean that the planned result is installed. ## Exercise the fixed-table MVP The following workflow requires a server that implements the orchestration above. The client tests -exercise scoped HTTP routing and table loaders; they do not implement database reference storage -or the database merge algorithm. +exercise suffix-based HTTP addressing and table loaders; they do not implement database reference +storage or the database merge algorithm. 1. Create database `training` and two populated managed tables, `features` and `labels`, on `main`. Stop writes and create database branch `experiment` from `main` using tree management. -2. Bind `restCatalog.withReference("training", "experiment")`. List and load `features` and `labels` - by their ordinary names, then write experiment data with the usual batch write API. Their - metadata reads and commits use `/trees/experiment/tables/...`. -3. Stop experiment writes and create database tag `train_v1` from `experiment`. Bind a second - catalog with `restCatalog.withReference("training", "train_v1")`. Load the same logical table - names for training; the service resolves the pinned table versions without a source-branch hint. -4. Advance the experiment tables, then reload and read them through the tag-bound catalog. The - tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. +2. List and load `features` and `labels` from database `training$branch_experiment`, then write + experiment data with the usual batch write API. Their metadata reads and commits use the + existing table paths with the complete suffixed database name. +3. Stop experiment writes and create database tag `train_v1` from `experiment`. Use the same + catalog to access database `training$tag_train_v1`. Load the same logical table names for training; + the service resolves the pinned table versions without a source-branch hint. +4. Advance the experiment tables, then reload and read them through the tag-suffixed database name. + The tagged data and schemas must remain unchanged. Verify that writes through the tag are rejected. 5. With main and experiment writers stopped, merge `train_v1` into `main`. This publishes the evaluated source version. Merging the live `experiment` branch would instead include its newer state. If both sides changed a table, choose a per-table merge mode when appropriate. @@ -531,22 +580,22 @@ or the database merge algorithm. ## Beyond the fixed-table MVP -The REST scope and client binding now identify the selected database view for table listing, +The database name suffix now identifies the selected database view for database lookup, table listing, reads, commits and the ordinary create/alter/drop endpoints. A complete server namespace still needs branch-local membership changes, stable identities across rename, and new identities for drop-and-recreate. The fixed-table server may return `501` for unsupported scoped DDL. -Global table IDs, global listings, rename and the other deferred endpoints need explicit scope -semantics before they can be enabled on a bound client. SQL engine configuration also needs to -preserve the same binding when constructing catalogs. These additions do not require callers to -construct per-table branch names. +Global table IDs, global listings, rename and the other deferred endpoints need explicit reference +semantics before they can be extended. Engine integrations must preserve the full database name in +identifiers and perform database existence checks through the catalog. These additions do not +require callers to construct per-table branch names. ## Validation and implementation sequence The reference tests validate HTTP paths, request bodies, authentication/configuration, pagination, -JSON compatibility, exception propagation and reference preservation through serialized catalogs -and tables. The OpenAPI validator checks that scoped endpoints reuse the corresponding ordinary -request and success-response structures. A stateful test fixture also uses real Paimon data files +JSON compatibility, exception propagation and suffix preservation through serialized tables and +catalog loaders. Tests cover virtual database lookup, mutation guards and malformed or mixed +selectors. The OpenAPI validator checks that reference access uses the ordinary table paths. A stateful test fixture also uses real Paimon data files to exercise batch writes on separate branches, frozen tag reads after source writes, and tag write rejection. This validates client integration with a resolving server; production reference lifecycle, snapshot retention and database merge still require server integration tests. diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index 6f2c9d18cde0..8e372fcb6cff 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -74,7 +74,7 @@ Choose the authentication guide for your service: ## API References - [REST Catalog API](./rest-api): the OpenAPI contract for catalog operations. -- [Database Branches and Tags](./database-versioning): experimental reference management, scoped table APIs, and +- [Database Branches and Tags](./database-versioning): experimental reference management, database-name selectors, and the server MVP design using existing table branches and tags. - [REST Management API](./management-api): permissions, row filters, column masking, and the corresponding Spark SQL procedures. diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index cae9e3b6e18c..c099dff39691 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -62,9 +62,9 @@ In this table, **table-scoped** means endpoints are described in the specification alongside their database-scoped counterparts. See [Database Branches and Tags](./database-versioning) for reference-management examples, merge -modes, and the server MVP design. Supported table operations can select a reference through -`/v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}`, using the existing request and -response structures. Java clients bind the scope with `withReference(database, reference)`. +modes, and the server MVP design. Supported table operations select a reference with a database +name such as `training$branch_experiment` or `training$tag_train_v1`. The existing table paths, +request/response structures and Java methods carry the full database name. ## Partition Compatibility diff --git a/docs/docs/program-api/rest-api.mdx b/docs/docs/program-api/rest-api.mdx index 45864e944493..ec89c6dcb249 100644 --- a/docs/docs/program-api/rest-api.mdx +++ b/docs/docs/program-api/rest-api.mdx @@ -37,7 +37,7 @@ metadata requests without bringing in the full table read/write bundle. | Implement an HTTP client or catalog server | [REST API specification](../concepts/rest/rest-api) | | Administrative endpoints | [Management API](../concepts/rest/management-api) | | Database branch/tag management | [Database Branches and Tags](../concepts/rest/database-versioning#java-management-usage) | -| Tables within a database branch/tag | [Reference-scoped table usage](../concepts/rest/database-versioning#java-table-usage) | +| Tables within a database branch/tag | [Database-name reference selectors](../concepts/rest/database-versioning#java-table-usage) | ## Dependency diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index 6eaaaa6b6794..a7bc8a1029b4 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -225,43 +225,19 @@ function requireExactEnum(contract, schemaName, expectedValues) { function validateCatalogOpenApi() { const contract = validateCommon('rest-catalog-open-api.yaml'); - const databasePath = '/v1/{prefix}/databases/{database}'; - [ - '/tables', - '/table-details', - '/tables/{table}', - '/tables/{table}/commit', - '/tables/{table}/token', - '/tables/{table}/auth', - '/tables/{table}/snapshot', - '/tables/{table}/snapshots', - '/tables/{table}/snapshots/{version}', - '/tables/{table}/schemas', - '/tables/{table}/schemas/{version}', - ].forEach((suffix) => { - const original = contract.spec.paths[databasePath + suffix]; - const scopedPath = databasePath + '/trees/{reference}' + suffix; - const scoped = contract.spec.paths[scopedPath]; - contract.checkSpec(scoped, `Missing reference-scoped table path: ${scopedPath}`); - Object.entries(original).forEach(([method, operation]) => { - if (!HTTP_METHODS.has(method)) { - return; - } - const counterpart = scoped[method]; - contract.checkSpec(counterpart, `Missing ${method} on ${scopedPath}`); - ['requestBody', 'responses'].forEach((field) => { - const value = (op) => field === 'responses' ? op.responses['200'] : op[field]; - contract.checkSpec( - JSON.stringify(value(operation)) === JSON.stringify(value(counterpart)), - `${scopedPath} must reuse the unscoped ${method} ${field} contract`, - ); - }); - contract.requireResponses(counterpart.operationId, ['404', '501']); - if (method !== 'get' && !suffix.endsWith('/auth')) { - contract.requireResponses(counterpart.operationId, ['409']); - } - }); - }); + contract.checkSpec( + !Object.keys(contract.spec.paths).some((path) => /\/trees\/\{[^}]+\}\/(tables|table-details)/.test(path)), + 'Database reference access must reuse ordinary table paths', + ); + const databaseParameter = contract.spec.components.parameters.Database; + contract.checkSpec( + databaseParameter.examples.branch.value === 'training$branch_experiment' && + databaseParameter.examples.tag.value === 'training$tag_train_v1', + 'Database reference examples must use the reserved branch and tag suffixes', + ); + ['getDatabase', 'listTables', 'getTable', 'commitTable', 'getSchema', 'listSchemas'].forEach( + (operationId) => contract.requireResponses(operationId, ['404', '409', '501']), + ); [ 'getConfig', 'createDatabase', diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 133944ea412e..0bf557b2eba1 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -84,11 +84,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ListDatabasesResponse' + $ref: "#/components/schemas/ListDatabasesResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + List physical databases. Branch and tag access names are not additional database entries; + discover references through /databases/{database}/trees. post: tags: - database @@ -104,16 +107,21 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CreateDatabaseRequest' + $ref: "#/components/schemas/CreateDatabaseRequest" responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "409": - $ref: '#/components/responses/DatabaseAlreadyExistErrorResponse' + $ref: "#/components/responses/DatabaseAlreadyExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Create a physical database. The reserved $branch_ and $tag_ suffix markers are not allowed; + create database references through /trees. /v1/{prefix}/databases/{database}: get: tags: @@ -126,24 +134,31 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" responses: "200": description: Get a database by database name. content: application/json: schema: - $ref: '#/components/schemas/GetDatabaseResponse' + $ref: "#/components/schemas/GetDatabaseResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + A database name with $branch_ or $tag_ selects an existing reference. Return + metadata for the virtual database, retaining the full requested name. This lookup supports + engine namespace existence checks. Missing databases or references return 404; a reference + type mismatch returns 409. Never resolve a missing reference to the base database. delete: tags: - database @@ -163,12 +178,17 @@ paths: responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Database reference suffixes are not allowed for database mutation. Use the /trees management + endpoints with the physical database name to manage references. post: tags: - database @@ -189,20 +209,25 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AlterDatabaseRequest' + $ref: "#/components/schemas/AlterDatabaseRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/AlterDatabaseResponse' + $ref: "#/components/schemas/AlterDatabaseResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' + $ref: "#/components/responses/DatabaseNotExistErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + description: >- + Database reference suffixes are not allowed for database mutation. Use the /trees management + endpoints with the physical database name to manage references. /v1/{prefix}/databases/{database}/trees: get: tags: @@ -452,23 +477,54 @@ paths: $ref: '#/components/schemas/ErrorResponse' "500": $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/trees/{reference}/tables: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" + /v1/{prefix}/databases/{database}/register: + post: + tags: + - table + summary: Register table + operationId: RegisterTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: database + in: path + required: true + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterTableRequest' + responses: + "200": + description: Success, no content + "400": + $ref: '#/components/responses/BadRequestErrorResponse' + "401": + $ref: '#/components/responses/UnauthorizedErrorResponse' + "404": + $ref: '#/components/responses/DatabaseNotExistErrorResponse' + "409": + $ref: '#/components/responses/TableAlreadyExistErrorResponse' + "500": + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables: get: tags: - - database-reference - summary: List tables in a database reference - operationId: listTablesInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. + - table + summary: List tables + operationId: listTables parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" - name: maxResults in: query schema: @@ -498,18 +554,29 @@ paths: $ref: "#/components/responses/ForbiddenErrorResponse" "404": $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. post: tags: - - database-reference - summary: Create table in a database reference - operationId: createTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. + - table + summary: Create table + operationId: createTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" requestBody: content: application/json: @@ -532,23 +599,25 @@ paths: $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/table-details: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. + /v1/{prefix}/databases/{database}/table-details: get: tags: - - database-reference - summary: List table details in a database reference - operationId: listTableDetailsInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. + - table + summary: List table details + operationId: listTableDetails parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" - name: maxResults in: query schema: @@ -583,709 +652,137 @@ paths: $ref: "#/components/responses/ForbiddenErrorResponse" "404": $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" "501": $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. + /v1/{prefix}/tables: get: tags: - - database-reference - summary: Get table in a database reference - operationId: getTableInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. Return the logical table name and the - resolved schema, path and storage options. Internal branch aliases may be carried in schema - options. + - table + summary: List tables globally + operationId: ListTablesGlobally + description: list tables paged globally which matches the given database name pattern and table name pattern both. + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: databaseNamePattern + description: A sql LIKE pattern (%) for database names. All databases will be returned if not set or empty. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: tableNamePattern + description: A sql LIKE pattern (%) for table names. All tables will be returned if not set or empty. Currently, only prefix matching is supported. + in: query + schema: + type: string + - name: maxResults + in: query + schema: + type: integer + format: int32 + - name: pageToken + in: query + schema: + type: string + responses: "200": description: OK content: application/json: schema: - $ref: "#/components/schemas/GetTableResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" + $ref: '#/components/schemas/ListTablesGloballyResponse' "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + $ref: '#/components/responses/UnauthorizedErrorResponse' "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - post: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/tables/id/{tableId}: + get: tags: - - database-reference - summary: Alter table in a database reference - operationId: alterTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/AlterTableRequest" + - table + summary: Get table by id + operationId: getTableById + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - name: tableId + in: path + required: true + schema: + type: string responses: "200": - description: Success, no content - "400": - $ref: "#/components/responses/BadRequestErrorResponse" + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetTableResponse' "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" + $ref: '#/components/responses/UnauthorizedErrorResponse' "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" + $ref: '#/components/responses/TableNotExistErrorResponse' "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - delete: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/databases/{database}/tables/{table}: + get: tags: - - database-reference - summary: Drop table in a database reference - operationId: dropTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - responses: - "200": - description: Success, no content - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/commit: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - post: - tags: - - database-reference - summary: Commit table in a database reference - operationId: commitTableInReference - description: >- - Apply this operation to a branch. A tag is immutable and returns 409. Identifiers and table - IDs in the body must agree with the logical table resolved from the path. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/CommitTableRequest" - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/CommitTableResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "409": - $ref: "#/components/responses/ReferenceTableConflictErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/token: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: Get table token in a database reference - operationId: getTableTokenInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. Credentials for a tag must allow reading - without allowing mutation of retained metadata or data. - responses: - "200": - description: DataToken for visit data. - content: - application/json: - schema: - $ref: "#/components/schemas/GetTableDataTokenResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/auth: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - post: - tags: - - database-reference - summary: Auth table query in a database reference - operationId: authTableQueryInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/AuthTableQueryRequest" - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/AuthTableQueryResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshot: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: Get table snapshot in a database reference - operationId: getTableSnapshotInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. An empty captured table returns 404 with - resourceType SNAPSHOT. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetTableSnapshotResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots/{version}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - - name: version - in: path - required: true - schema: - type: string - get: - tags: - - database-reference - summary: Get version snapshot in a database reference - operationId: getVersionSnapshotInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. A tag exposes only its pinned snapshot: - LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetVersionSnapshotResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/snapshots: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: List snapshots in a database reference - operationId: listSnapshotsInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. A tag exposes only its pinned snapshot: - LATEST and EARLIEST select it, and other versions must resolve to that snapshot or return 404. - parameters: - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/ListSnapshotsResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - get: - tags: - - database-reference - summary: List table schemas in a database reference - operationId: listSchemasInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. LATEST selects the captured schema. Schema - history is limited to schemas retained for the captured table version; later source schemas - are not visible. - parameters: - - name: maxResults - in: query - schema: - type: integer - minimum: 0 - - name: pageToken - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/ListSchemasResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - /v1/{prefix}/databases/{database}/trees/{reference}/tables/{table}/schemas/{version}: - description: >- - Table operations within an existing database branch or immutable tag. Uses the same request and - response schemas as the corresponding unscoped table endpoint. - parameters: - - $ref: "#/components/parameters/Prefix" - - $ref: "#/components/parameters/Database" - - $ref: "#/components/parameters/Reference" - - $ref: "#/components/parameters/Table" - - $ref: "#/components/parameters/Version" - get: - tags: - - database-reference - summary: Get table schema in a database reference - operationId: getSchemaInReference - description: >- - Resolve membership and the table version through the reference. A tag returns frozen metadata - and never the latest state of its source branch. LATEST selects the captured schema. Schema - history is limited to schemas retained for the captured table version; later source schemas - are not visible. - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: "#/components/schemas/GetSchemaResponse" - "400": - $ref: "#/components/responses/BadRequestErrorResponse" - "401": - $ref: "#/components/responses/UnauthorizedErrorResponse" - "403": - $ref: "#/components/responses/ForbiddenErrorResponse" - "404": - $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" - "500": - $ref: "#/components/responses/ServerErrorResponse" - "501": - $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" - - /v1/{prefix}/databases/{database}/register: - post: - tags: - - table - summary: Register table - operationId: RegisterTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterTableRequest' - responses: - "200": - description: Success, no content - "400": - $ref: '#/components/responses/BadRequestErrorResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/tables: - get: - tags: - - table - summary: List tables - operationId: listTables - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTablesResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - post: - tags: - - table - summary: Create table - operationId: createTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CreateTableRequest' - responses: - "200": - description: Success, no content - "400": - $ref: '#/components/responses/BadRequestErrorResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/table-details: - get: - tags: - - table - summary: List table details - operationId: listTableDetails - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: tableType - description: Filter tables by table type. All table types will be returned if not set or empty. - in: query - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTableDetailsResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/DatabaseNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/tables: - get: - tags: - - table - summary: List tables globally - operationId: ListTablesGlobally - description: list tables paged globally which matches the given database name pattern and table name pattern both. - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: databaseNamePattern - description: A sql LIKE pattern (%) for database names. All databases will be returned if not set or empty. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: tableNamePattern - description: A sql LIKE pattern (%) for table names. All tables will be returned if not set or empty. Currently, only prefix matching is supported. - in: query - schema: - type: string - - name: maxResults - in: query - schema: - type: integer - format: int32 - - name: pageToken - in: query - schema: - type: string - - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/ListTablesGloballyResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/tables/id/{tableId}: - get: - tags: - - table - summary: Get table by id - operationId: getTableById - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: tableId - in: path - required: true - schema: - type: string - responses: - "200": - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/GetTableResponse' - "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - "404": - $ref: '#/components/responses/TableNotExistErrorResponse' - "500": - $ref: '#/components/responses/ServerErrorResponse' - /v1/{prefix}/databases/{database}/tables/{table}: - get: - tags: - - table - summary: Get table - operationId: getTable - parameters: - - name: prefix - in: path - required: true - schema: - type: string - - name: database - in: path - required: true - schema: - type: string - - name: table - in: path - required: true - schema: - type: string + - table + summary: Get table + operationId: getTable + parameters: + - name: prefix + in: path + required: true + schema: + type: string + - $ref: "#/components/parameters/Database" + - name: table + in: path + required: true + schema: + type: string responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/GetTableResponse' + $ref: "#/components/schemas/GetTableResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Return the requested database name including its suffix, + the logical table name, and resolved schema, path and storage options. Internal branch aliases + may be carried in schema options. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. post: tags: - table @@ -1297,11 +794,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1311,20 +804,30 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AlterTableRequest' + $ref: "#/components/schemas/AlterTableRequest" responses: "200": description: Success, no content "400": - $ref: '#/components/responses/BadRequestErrorResponse' + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" "409": - $ref: '#/components/responses/TableAlreadyExistErrorResponse' + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. delete: tags: - table @@ -1336,11 +839,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1349,12 +848,26 @@ paths: responses: "200": description: Success, no content + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/tables/rename: post: tags: @@ -1397,11 +910,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1411,22 +920,34 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CommitTableRequest' + $ref: "#/components/schemas/CommitTableRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/CommitTableResponse' + $ref: "#/components/schemas/CommitTableResponse" "400": - $ref: '#/components/responses/BadRequestErrorResponse' + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Apply this operation to the selected branch; a tag returns 409. Identifiers in the + body must retain the full database name including its suffix and agree with the path. Table + IDs must match the resolved table. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/rollback: post: tags: @@ -1526,11 +1047,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1542,13 +1059,27 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetTableDataTokenResponse' + $ref: "#/components/schemas/GetTableDataTokenResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Tag credentials must allow reading without permitting + mutation of retained metadata or data. Missing references never fall back to the base + database. A suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/auth: post: tags: @@ -1561,11 +1092,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1575,31 +1102,33 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/AuthTableQueryRequest' + $ref: "#/components/schemas/AuthTableQueryRequest" responses: "200": description: OK content: application/json: schema: - $ref: '#/components/schemas/AuthTableQueryResponse' + $ref: "#/components/schemas/AuthTableQueryResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" "403": - $ref: '#/components/responses/ForbiddenErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. Missing references never fall back to the base database. A + suffix whose type does not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshot: get: tags: @@ -1612,11 +1141,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1628,25 +1153,27 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetTableSnapshotResponse' + $ref: "#/components/schemas/GetTableSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - - SnapshotNotExistException, the requested snapshot does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' - SnapshotNotExist: - $ref: '#/components/examples/SnapshotNotExistError' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. An existing empty table returns 404 with resourceType + SNAPSHOT. Missing references never fall back to the base database. A suffix whose type does + not match the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}: get: tags: @@ -1659,11 +1186,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1680,25 +1203,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/GetVersionSnapshotResponse' + $ref: "#/components/schemas/GetVersionSnapshotResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' - 404: - description: - Not Found - - TableNotExistException, table does not exist - - SnapshotNotExistException, the requested snapshot does not exist - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - examples: - TableNotExist: - $ref: '#/components/examples/TableNotExistError' - SnapshotNotExist: - $ref: '#/components/examples/SnapshotNotExistError' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" + "404": + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST + and EARLIEST select it; other versions must resolve to it or return 404. Missing references + never fall back to the base database. A suffix whose type does not match the reference returns + 409. /v1/{prefix}/databases/{database}/tables/{table}/snapshots: get: tags: @@ -1711,11 +1237,7 @@ paths: required: true schema: type: string - - name: database - in: path - required: true - schema: - type: string + - $ref: "#/components/parameters/Database" - name: table in: path required: true @@ -1736,13 +1258,28 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ListSnapshotsResponse' + $ref: "#/components/schemas/ListSnapshotsResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": - $ref: '#/components/responses/UnauthorizedErrorResponse' + $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - $ref: '#/components/responses/TableNotExistErrorResponse' + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": - $ref: '#/components/responses/ServerErrorResponse' + $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. A database tag exposes only its pinned snapshot: LATEST + and EARLIEST select it; other versions must resolve to it or return 404. Missing references + never fall back to the base database. A suffix whose type does not match the reference returns + 409. /v1/{prefix}/databases/{database}/tables/{table}/schemas: parameters: - $ref: "#/components/parameters/Prefix" @@ -1770,16 +1307,27 @@ paths: application/json: schema: $ref: "#/components/schemas/ListSchemasResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - description: Table or schema does not exist. - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. For a database tag, LATEST selects the captured schema. + History is limited to schemas retained for its captured data, excluding newer source schemas. + Missing references never fall back to the base database. A suffix whose type does not match + the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}: parameters: - $ref: "#/components/parameters/Prefix" @@ -1798,17 +1346,27 @@ paths: application/json: schema: $ref: "#/components/schemas/GetSchemaResponse" + "400": + $ref: "#/components/responses/BadRequestErrorResponse" "401": $ref: "#/components/responses/UnauthorizedErrorResponse" + "403": + $ref: "#/components/responses/ForbiddenErrorResponse" "404": - description: Table or schema does not exist. - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" + $ref: "#/components/responses/ReferenceTableNotExistErrorResponse" + "409": + $ref: "#/components/responses/ReferenceTableConflictErrorResponse" "500": $ref: "#/components/responses/ServerErrorResponse" - + "501": + $ref: "#/components/responses/ReferenceTableNotImplementedErrorResponse" + description: >- + The database path segment may select a database branch or immutable tag using its reserved + suffix. Resolve the table through that reference. Tags expose captured membership and + metadata, never newer source state. For a database tag, LATEST selects the captured schema. + History is limited to schemas retained for its captured data, excluding newer source schemas. + Missing references never fall back to the base database. A suffix whose type does not match + the reference returns 409. /v1/{prefix}/databases/{database}/tables/{table}/partitions: get: tags: @@ -3297,14 +2855,17 @@ components: required: true schema: type: string - Reference: - name: reference - in: path - required: true - schema: - type: string - pattern: ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ - description: An existing database branch or immutable tag. Never falls back to main. + description: >- + Decoded database name. On supported table operations and GET database, + $branch_ selects a writable branch and $tag_ selects + an immutable tag. The markers are case-sensitive and reserved; exactly one valid reference + suffix is allowed. Encode the complete name as one path segment. A suffix is not a physical + database name. + examples: + branch: + value: training$branch_experiment + tag: + value: training$tag_train_v1 Table: name: table in: path @@ -3334,16 +2895,16 @@ components: $ref: "#/components/schemas/ErrorResponse" ReferenceTableConflictErrorResponse: description: >- - The target is an immutable tag, the table already exists, or the operation conflicts with the - selected table state. + The reference type does not match the suffix, the target is an immutable tag, the table already + exists, or the operation conflicts with the selected table state. content: application/json: schema: $ref: "#/components/schemas/ErrorResponse" ReferenceTableNotImplementedErrorResponse: description: >- - The server does not implement this reference-scoped table operation. No fallback to an unscoped - route is allowed. + The server does not implement this operation on a database reference. No fallback to the + physical database is allowed. content: application/json: schema: diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java new file mode 100644 index 000000000000..6b21c26f7241 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseIdentifier.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.catalog.Identifier; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** A REST database name and its optional database-level branch or immutable tag selector. */ +@Experimental +public final class DatabaseIdentifier { + + private static final String BRANCH_SUFFIX = "$branch_"; + private static final String TAG_SUFFIX = "$tag_"; + + private final String databaseName; + @Nullable private final DatabaseReference reference; + + private DatabaseIdentifier(String databaseName, @Nullable DatabaseReference reference) { + this.databaseName = databaseName; + this.reference = reference; + } + + /** + * Parses a decoded database name, such as {@code training$branch_experiment}. + * + *

The suffixes {@code $branch_} and {@code $tag_} are reserved. A name containing either + * marker must have exactly one valid reference suffix. Other dollar signs remain literal. + * Callers retain the original name in table identifiers and encode it as one REST path segment. + */ + public static DatabaseIdentifier parse(String name) { + checkArgument(name != null && !name.trim().isEmpty(), "Database name must not be blank"); + int branch = name.indexOf(BRANCH_SUFFIX); + int tag = name.indexOf(TAG_SUFFIX); + if (branch < 0 && tag < 0) { + return new DatabaseIdentifier(name, null); + } + boolean isBranch = branch >= 0 && (tag < 0 || branch < tag); + int separator = isBranch ? branch : tag; + String database = name.substring(0, separator); + checkArgument(!database.trim().isEmpty(), "Database name must not be blank"); + String reference = + name.substring( + separator + (isBranch ? BRANCH_SUFFIX.length() : TAG_SUFFIX.length())); + return new DatabaseIdentifier( + database, + new DatabaseReference( + isBranch ? DatabaseReferenceType.BRANCH : DatabaseReferenceType.TAG, + reference)); + } + + /** The physical database name, without the reference suffix. */ + public String getDatabaseName() { + return databaseName; + } + + @Nullable + public DatabaseReference getReference() { + return reference; + } + + static void checkNoReference(String database, String operation) { + if (parse(database).getReference() != null) { + throw new UnsupportedOperationException( + operation + " does not support database reference suffixes: " + database); + } + } + + static void checkTableName(String database, String table) { + checkArgument( + parse(database).getReference() == null + || Identifier.create(database, table).getBranchName() == null, + "Table branch suffixes cannot be combined with a database reference"); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java index 353ac2d605cb..845a68deb003 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/DatabaseReference.java @@ -49,14 +49,10 @@ public DatabaseReference( @JsonProperty(FIELD_TYPE) DatabaseReferenceType type, @JsonProperty(FIELD_NAME) String name) { checkArgument(type != null, "Reference type must not be null"); - validateName(name); - this.type = type; - this.name = name; - } - - static void validateName(String name) { checkArgument( name != null && name.matches(NAME_PATTERN), "Invalid reference name: %s", name); + this.type = type; + this.name = name; } @JsonGetter(FIELD_TYPE) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 723bc1e8f7db..e71f674266a7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -257,26 +257,6 @@ public RESTApi(Options options, boolean configRequired) { this.resourcePaths = ResourcePaths.forCatalogProperties(options); } - private RESTApi(RESTApi api, ResourcePaths resourcePaths) { - this.client = api.client; - this.restAuthFunction = api.restAuthFunction; - this.options = api.options; - this.resourcePaths = resourcePaths; - } - - /** - * Returns a client whose table operations address one database branch or immutable tag. - * - *

The original client is unchanged. Table names remain logical names, without a table branch - * suffix. The server resolves the reference and enforces tag immutability. Operations without a - * reference-scoped table route are unsupported on this client; database and reference - * management retain their catalog-wide meaning. - */ - @Experimental - public RESTApi withReference(String database, String reference) { - return new RESTApi(this, resourcePaths.withReference(database, reference)); - } - /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; @@ -340,6 +320,7 @@ public PagedList listDatabasesPaged( * this database */ public void createDatabase(String name, Map properties) { + DatabaseIdentifier.checkNoReference(name, "createDatabase"); CreateDatabaseRequest request = new CreateDatabaseRequest(name, properties); client.post(resourcePaths.databases(), request, restAuthFunction); } @@ -367,6 +348,7 @@ public GetDatabaseResponse getDatabase(String name) { * this database */ public void dropDatabase(String name) { + DatabaseIdentifier.checkNoReference(name, "dropDatabase"); client.delete(resourcePaths.database(name), restAuthFunction); } @@ -381,6 +363,7 @@ public void dropDatabase(String name) { * this database */ public void alterDatabase(String name, List removals, Map updates) { + DatabaseIdentifier.checkNoReference(name, "alterDatabase"); client.post( resourcePaths.database(name), new AlterDatabaseRequest(removals, updates), @@ -934,6 +917,7 @@ public PagedList listSchemasPaged( * creating table */ public void createTable(Identifier identifier, Schema schema) { + DatabaseIdentifier.checkTableName(identifier.getDatabaseName(), identifier.getObjectName()); CreateTableRequest request = new CreateTableRequest(identifier, schema); client.post(resourcePaths.tables(identifier.getDatabaseName()), request, restAuthFunction); } @@ -949,6 +933,8 @@ public void createTable(Identifier identifier, Schema schema) { * renaming table */ public void renameTable(Identifier fromTable, Identifier toTable) { + DatabaseIdentifier.checkNoReference(fromTable.getDatabaseName(), "renameTable"); + DatabaseIdentifier.checkNoReference(toTable.getDatabaseName(), "renameTable"); RenameTableRequest request = new RenameTableRequest(fromTable, toTable); client.post(resourcePaths.renameTable(), request, restAuthFunction); } @@ -2048,6 +2034,8 @@ public PagedList listViewsPagedGlobally( * views */ public void renameView(Identifier fromView, Identifier toView) { + DatabaseIdentifier.checkNoReference(fromView.getDatabaseName(), "renameView"); + DatabaseIdentifier.checkNoReference(toView.getDatabaseName(), "renameView"); RenameTableRequest request = new RenameTableRequest(fromView, toView); client.post(resourcePaths.renameView(), request, restAuthFunction); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index e1b9d8615c2e..22d6ce38d81f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -19,14 +19,11 @@ package org.apache.paimon.rest; import org.apache.paimon.annotation.Experimental; -import org.apache.paimon.catalog.Identifier; import org.apache.paimon.management.PermissionResource; import org.apache.paimon.options.Options; import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; -import javax.annotation.Nullable; - import static org.apache.paimon.rest.RESTUtil.encodeString; import static org.apache.paimon.utils.Preconditions.checkArgument; @@ -67,47 +64,9 @@ public static ResourcePaths forCatalogProperties(Options options) { } private final String prefix; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; public ResourcePaths(String prefix) { - this(encodeString(prefix), null, null); - } - - private ResourcePaths( - String encodedPrefix, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this.prefix = encodedPrefix; - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; - } - - /** Returns paths for table operations within one database branch or immutable tag. */ - @Experimental - public ResourcePaths withReference(String database, String reference) { - checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); - DatabaseReference.validateName(reference); - return new ResourcePaths(prefix, database, reference); - } - - private String tableScope(String database) { - if (referenceName == null) { - return database(database); - } - checkArgument( - referenceDatabase.equals(database), - "Table operation must use reference database %s, not %s", - referenceDatabase, - database); - return databaseTree(database, referenceName); - } - - private void checkUnscoped(String operation) { - if (referenceName != null) { - throw new UnsupportedOperationException( - operation + " is not supported in a database reference scope"); - } + this.prefix = encodeString(prefix); } /** Labels attached to one entity, whose canonical name is encoded as a single segment. */ @@ -139,6 +98,7 @@ private static String encodePathSegment(String value) { @Experimental public String semanticViews(String database) { checkArgument(database != null && !database.trim().isEmpty(), "database must not be blank"); + DatabaseIdentifier.checkNoReference(database, "semanticViews"); return SLASH.join(V1, prefix, DATABASES, encodePathSegment(database), SEMANTIC_VIEWS); } @@ -168,8 +128,8 @@ public String revokePermission() { /** Policy collection nested below its attachment resource. */ @Experimental public String policies(PermissionResource resource) { - checkUnscoped("policies"); resource.validatePolicyAttachment(); + DatabaseIdentifier.checkNoReference(resource.getDatabase(), "policies"); return SLASH.join(table(resource.getDatabase(), resource.getTable()), POLICIES); } @@ -184,12 +144,14 @@ public String databases() { } public String database(String databaseName) { + DatabaseIdentifier.parse(databaseName); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName)); } /** Database-level branches and immutable tags. */ @Experimental public String databaseTrees(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "tree management"); return SLASH.join(database(databaseName), TREES); } @@ -206,38 +168,32 @@ public String mergeDatabaseBranch(String databaseName, String branch) { } public String tables(String databaseName) { - return SLASH.join(tableScope(databaseName), TABLES); + return SLASH.join(database(databaseName), TABLES); } public String tableDetails(String databaseName) { - return SLASH.join(tableScope(databaseName), TABLE_DETAILS); + return SLASH.join(database(databaseName), TABLE_DETAILS); } public String tables() { - checkUnscoped("tables"); return SLASH.join(V1, prefix, TABLES); } public String table(String tableId) { - checkUnscoped("table"); return SLASH.join(V1, prefix, TABLES, ID, encodeString(tableId)); } public String table(String databaseName, String objectName) { - checkArgument( - referenceName == null - || Identifier.create(databaseName, objectName).getBranchName() == null, - "Table branch suffixes cannot be combined with a database reference"); + DatabaseIdentifier.checkTableName(databaseName, objectName); return SLASH.join(tables(databaseName), encodeString(objectName)); } public String renameTable() { - checkUnscoped("renameTable"); return SLASH.join(V1, prefix, TABLES, "rename"); } public String replaceTable(String databaseName, String objectName) { - checkUnscoped("replaceTable"); + DatabaseIdentifier.checkNoReference(databaseName, "replaceTable"); return SLASH.join( V1, prefix, @@ -253,7 +209,7 @@ public String commitTable(String databaseName, String objectName) { } public String rollbackTable(String databaseName, String objectName) { - checkUnscoped("rollbackTable"); + DatabaseIdentifier.checkNoReference(databaseName, "rollbackTable"); return SLASH.join( V1, prefix, @@ -265,7 +221,7 @@ public String rollbackTable(String databaseName, String objectName) { } public String rollbackSchemaTable(String databaseName, String objectName) { - checkUnscoped("rollbackSchemaTable"); + DatabaseIdentifier.checkNoReference(databaseName, "rollbackSchemaTable"); return SLASH.join( V1, prefix, @@ -277,7 +233,7 @@ public String rollbackSchemaTable(String databaseName, String objectName) { } public String registerTable(String databaseName) { - checkUnscoped("registerTable"); + DatabaseIdentifier.checkNoReference(databaseName, "registerTable"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), REGISTER); } @@ -310,7 +266,7 @@ public String authTable(String databaseName, String objectName) { } public String partitions(String databaseName, String objectName) { - checkUnscoped("partitions"); + DatabaseIdentifier.checkNoReference(databaseName, "partitions"); return SLASH.join( V1, prefix, @@ -322,7 +278,7 @@ public String partitions(String databaseName, String objectName) { } public String dropPartitions(String databaseName, String objectName) { - checkUnscoped("dropPartitions"); + DatabaseIdentifier.checkNoReference(databaseName, "dropPartitions"); return SLASH.join( V1, prefix, @@ -335,7 +291,7 @@ public String dropPartitions(String databaseName, String objectName) { } public String markDonePartitions(String databaseName, String objectName) { - checkUnscoped("markDonePartitions"); + DatabaseIdentifier.checkNoReference(databaseName, "markDonePartitions"); return SLASH.join( V1, prefix, @@ -348,7 +304,7 @@ public String markDonePartitions(String databaseName, String objectName) { } public String listPartitionsByNames(String databaseName, String objectName) { - checkUnscoped("listPartitionsByNames"); + DatabaseIdentifier.checkNoReference(databaseName, "listPartitionsByNames"); return SLASH.join( V1, prefix, @@ -361,7 +317,7 @@ public String listPartitionsByNames(String databaseName, String objectName) { } public String listPartitionsByFilter(String databaseName, String objectName) { - checkUnscoped("listPartitionsByFilter"); + DatabaseIdentifier.checkNoReference(databaseName, "listPartitionsByFilter"); return SLASH.join( V1, prefix, @@ -374,7 +330,7 @@ public String listPartitionsByFilter(String databaseName, String objectName) { } public String branches(String databaseName, String objectName) { - checkUnscoped("branches"); + DatabaseIdentifier.checkNoReference(databaseName, "branches"); return SLASH.join( V1, prefix, @@ -386,7 +342,7 @@ public String branches(String databaseName, String objectName) { } public String branch(String databaseName, String objectName, String branchName) { - checkUnscoped("branch"); + DatabaseIdentifier.checkNoReference(databaseName, "branch"); return SLASH.join( V1, prefix, @@ -399,7 +355,7 @@ public String branch(String databaseName, String objectName, String branchName) } public String forwardBranch(String databaseName, String tableName, String branch) { - checkUnscoped("forwardBranch"); + DatabaseIdentifier.checkNoReference(databaseName, "forwardBranch"); return SLASH.join( V1, prefix, @@ -413,7 +369,7 @@ public String forwardBranch(String databaseName, String tableName, String branch } public String tags(String databaseName, String objectName) { - checkUnscoped("tags"); + DatabaseIdentifier.checkNoReference(databaseName, "tags"); return SLASH.join( V1, prefix, @@ -425,7 +381,7 @@ public String tags(String databaseName, String objectName) { } public String consumers(String databaseName, String objectName) { - checkUnscoped("consumers"); + DatabaseIdentifier.checkNoReference(databaseName, "consumers"); return SLASH.join( V1, prefix, @@ -437,7 +393,7 @@ public String consumers(String databaseName, String objectName) { } public String resetConsumer(String databaseName, String objectName) { - checkUnscoped("resetConsumer"); + DatabaseIdentifier.checkNoReference(databaseName, "resetConsumer"); return SLASH.join( V1, prefix, @@ -450,7 +406,7 @@ public String resetConsumer(String databaseName, String objectName) { } public String tag(String databaseName, String objectName, String tagName) { - checkUnscoped("tag"); + DatabaseIdentifier.checkNoReference(databaseName, "tag"); return SLASH.join( V1, prefix, @@ -463,10 +419,12 @@ public String tag(String databaseName, String objectName, String tagName) { } public String views(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "views"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), VIEWS); } public String viewDetails(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "viewDetails"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), VIEW_DETAILS); } @@ -475,6 +433,7 @@ public String views() { } public String view(String databaseName, String viewName) { + DatabaseIdentifier.checkNoReference(databaseName, "view"); return SLASH.join( V1, prefix, DATABASES, encodeString(databaseName), VIEWS, encodeString(viewName)); } @@ -484,6 +443,7 @@ public String renameView() { } public String functions(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "functions"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), FUNCTIONS); } @@ -492,10 +452,12 @@ public String functions() { } public String functionDetails(String databaseName) { + DatabaseIdentifier.checkNoReference(databaseName, "functionDetails"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), FUNCTION_DETAILS); } public String function(String databaseName, String functionName) { + DatabaseIdentifier.checkNoReference(databaseName, "function"); return SLASH.join( V1, prefix, diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java new file mode 100644 index 000000000000..edba6dcb8d54 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/DatabaseIdentifierTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the reserved database reference suffix grammar independently of HTTP encoding. */ +class DatabaseIdentifierTest { + + @Test + void testBranchAndTagSelectors() { + DatabaseIdentifier branch = DatabaseIdentifier.parse("training db$branch_experiment"); + assertThat(branch.getDatabaseName()).isEqualTo("training db"); + assertThat(branch.getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "experiment")); + + DatabaseIdentifier tag = DatabaseIdentifier.parse("training$literal$tag_train_v1"); + assertThat(tag.getDatabaseName()).isEqualTo("training$literal"); + assertThat(tag.getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.TAG, "train_v1")); + + assertThat(DatabaseIdentifier.parse("training$branch_123").getReference()) + .isEqualTo(new DatabaseReference(DatabaseReferenceType.BRANCH, "123")); + } + + @ParameterizedTest + @ValueSource( + strings = { + "training", + "training$literal", + "training$Branch_main", + "training%24branch_main" + }) + void testOrdinaryNamesRemainLiteral(String name) { + DatabaseIdentifier identifier = DatabaseIdentifier.parse(name); + assertThat(identifier.getDatabaseName()).isEqualTo(name); + assertThat(identifier.getReference()).isNull(); + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + " ", + "$branch_main", + "$tag_v1", + "training$branch_", + "training$tag_", + "training$branch_a/b", + "training$tag_..", + "training$branch_a$branch_b", + "training$branch_a$tag_b", + "training$tag_a$branch_b" + }) + void testMalformedSelectorsAreNotLiteralDatabaseNames(String name) { + assertThatThrownBy(() -> DatabaseIdentifier.parse(name)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index e830e40a1011..acf362fc95e9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -115,8 +115,6 @@ public class RESTCatalog implements Catalog { private final RESTApi api; private final CatalogContext context; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; private final boolean dataTokenEnabled; protected final Map tableDefaultOptions; private final @Nullable LocalCacheManager cacheManager; @@ -126,30 +124,7 @@ public RESTCatalog(CatalogContext context) { } public RESTCatalog(CatalogContext context, boolean configRequired) { - this(context, configRequired, null, null); - } - - RESTCatalog( - CatalogContext context, - boolean configRequired, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this( - context, - new RESTApi(context.options(), configRequired), - referenceDatabase, - referenceName); - } - - private RESTCatalog( - CatalogContext context, - RESTApi api, - @Nullable String referenceDatabase, - @Nullable String referenceName) { - this.api = - referenceName == null ? api : api.withReference(referenceDatabase, referenceName); - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; + this.api = new RESTApi(context.options(), configRequired); this.context = CatalogContext.create( api.options(), @@ -168,20 +143,7 @@ public Map options() { @Override public RESTCatalogLoader catalogLoader() { - return new RESTCatalogLoader(context, referenceDatabase, referenceName); - } - - /** - * Returns a separate catalog whose table operations use one database branch or immutable tag. - * - *

The binding is preserved by {@link #catalogLoader()}. Use ordinary logical table names; - * the server resolves their backing versions. No additional configuration request is made. - * Database and reference management are not versioned by this binding. - */ - @Experimental - public RESTCatalog withReference(String database, String reference) { - DatabaseReference.validateName(reference); - return new RESTCatalog(context, api, database, reference); + return new RESTCatalogLoader(context); } @Experimental @@ -262,6 +224,7 @@ public Database getDatabase(String name) throws DatabaseNotExistException { public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade) throws DatabaseNotExistException, DatabaseNotEmptyException { checkNotSystemDatabase(name); + DatabaseIdentifier.checkNoReference(name, "dropDatabase"); try { if (!cascade && !this.listTables(name).isEmpty()) { throw new DatabaseNotEmptyException(name); @@ -571,9 +534,10 @@ public boolean commitSnapshot( Snapshot snapshot, List statistics) throws TableNotExistException { - // CatalogSnapshotCommit supplies the physical storage branch. A database reference - // already selects the write target, so keep its logical table name on the wire. - if (referenceName != null && identifier.getBranchName() != null) { + // CatalogSnapshotCommit supplies the physical storage branch. The database suffix + // already selects the write target; keep the logical table name on the wire. + if (DatabaseIdentifier.parse(identifier.getDatabaseName()).getReference() != null + && identifier.getBranchName() != null) { identifier = new Identifier( identifier.getDatabaseName(), diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java index 05aa369eaa4d..efc5a0b46ca4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalogLoader.java @@ -21,28 +21,15 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.catalog.CatalogLoader; -import javax.annotation.Nullable; - /** Loader to create {@link RESTCatalog}. */ public class RESTCatalogLoader implements CatalogLoader { private static final long serialVersionUID = 1L; private final CatalogContext context; - @Nullable private final String referenceDatabase; - @Nullable private final String referenceName; public RESTCatalogLoader(CatalogContext context) { - this(context, null, null); - } - - RESTCatalogLoader( - CatalogContext context, - @Nullable String referenceDatabase, - @Nullable String referenceName) { this.context = context; - this.referenceDatabase = referenceDatabase; - this.referenceName = referenceName; } public CatalogContext context() { @@ -51,6 +38,6 @@ public CatalogContext context() { @Override public RESTCatalog load() { - return new RESTCatalog(context, false, referenceDatabase, referenceName); + return new RESTCatalog(context, false); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java index 9ab1269b467f..b71d8c93a681 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogReferenceTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.reader.RecordReader; import org.apache.paimon.rest.exceptions.AlreadyExistsException; import org.apache.paimon.rest.requests.CommitTableRequest; +import org.apache.paimon.rest.requests.CreateTableRequest; import org.apache.paimon.rest.responses.GetSchemaResponse; import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.schema.FileSystemSchemaManager; @@ -112,37 +113,38 @@ void tearDown() throws Exception { } @ParameterizedTest - @ValueSource(strings = {"experiment", "train_v1"}) + @ValueSource(strings = {"$branch_experiment", "$tag_train_v1"}) void testTableAndSerializedLoaderKeepReference(String reference) throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, reference); - String scope = DATABASE_PATH + "/trees/" + reference; + String database = DATABASE + reference; + Identifier selected = Identifier.create(database, "features"); + String scope = DATABASE_PATH + reference.replace("$", "%24"); enqueue(200, "{\"tables\":[\"features\",\"labels\"]}"); - assertThat(scoped.listTables(DATABASE)).containsExactly("features", "labels"); + assertThat(catalog.listTables(database)).containsExactly("features", "labels"); takeRequest("GET", scope + "/tables"); - enqueue(200, tableResponse("physical-experiment")); - FileStoreTable table = (FileStoreTable) scoped.getTable(TABLE); - assertThat(table.catalogEnvironment().identifier()).isEqualTo(TABLE); + enqueue(200, tableResponse(database, "physical-experiment", 2)); + FileStoreTable table = (FileStoreTable) catalog.getTable(selected); + assertThat(table.catalogEnvironment().identifier()).isEqualTo(selected); assertThat(table.snapshotManager().branch()).isEqualTo("physical-experiment"); assertThat(table.schema().id()).isEqualTo(2); takeRequest("GET", scope + "/tables/features"); - // A task receives a serialized table. Its snapshot loader must still address the tree. + // A task receives a serialized table. Its identifier must retain the database suffix. FileStoreTable restored = InstantiationUtil.clone(table); enqueue(200, "{\"snapshot\":{\"snapshot\":" + SNAPSHOT_JSON + "}}"); assertThat(restored.snapshotManager().latestSnapshot().id()).isEqualTo(7); takeRequest("GET", scope + "/tables/features/snapshot"); - RESTCatalog loaded = InstantiationUtil.clone(scoped.catalogLoader()).load(); + RESTCatalog loaded = InstantiationUtil.clone(catalog.catalogLoader()).load(); enqueue( 200, RESTApi.toJson( new GetSchemaResponse( TableSchema.create(2, schema("physical-experiment"))))); - assertThat(loaded.loadSchema(TABLE, "LATEST").get().id()).isEqualTo(2); + assertThat(loaded.loadSchema(selected, "LATEST").get().id()).isEqualTo(2); takeRequest("GET", scope + "/tables/features/schemas/LATEST"); - // Binding another catalog must not change the original catalog's route or metadata. + // The same catalog also loads the ordinary database without reference state. enqueue(200, tableResponse("main")); FileStoreTable main = (FileStoreTable) catalog.getTable(TABLE); assertThat(main.snapshotManager().branch()).isEqualTo("main"); @@ -152,10 +154,10 @@ void testTableAndSerializedLoaderKeepReference(String reference) throws Exceptio @Test void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, "experiment"); - enqueue(200, tableResponse("physical-experiment")); - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); - takeRequest("GET", DATABASE_PATH + "/trees/experiment/tables/features"); + Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + enqueue(200, tableResponse(selected.getDatabaseName(), "physical-experiment", 2)); + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); + takeRequest("GET", DATABASE_PATH + "%24branch_experiment/tables/features"); Snapshot snapshot = Snapshot.fromJson(SNAPSHOT_JSON); enqueue(200, "{\"success\":true}"); @@ -170,7 +172,7 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { .isTrue(); } RecordedRequest request = - takeRequest("POST", DATABASE_PATH + "/trees/experiment/tables/features/commit"); + takeRequest("POST", DATABASE_PATH + "%24branch_experiment/tables/features/commit"); CommitTableRequest body = RESTApi.fromJson(request.getBody().readUtf8(), CommitTableRequest.class); assertThat(body.getTableId()).isEqualTo("table-id"); @@ -181,16 +183,18 @@ void testStorageCommitUsesLogicalTableAndExistingBody() throws Exception { @Test void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { - RESTApi scoped = catalog.api().withReference(DATABASE, "train_v1"); - String scope = DATABASE_PATH + "/trees/train_v1"; + RESTApi api = catalog.api(); + String database = DATABASE + "$tag_train_v1"; + Identifier selected = Identifier.create(database, "features"); + String scope = DATABASE_PATH + "%24tag_train_v1"; String tablePath = scope + "/tables/features"; enqueue(200, "{\"tables\":[\"features\"],\"nextPageToken\":\"next\"}"); - assertThat(scoped.listTablesPaged(DATABASE, 1, null, "feat%", null).getNextPageToken()) + assertThat(api.listTablesPaged(database, 1, null, "feat%", null).getNextPageToken()) .isEqualTo("next"); RecordedRequest first = takeRequest("GET", scope + "/tables"); assertThat(first.getRequestUrl().queryParameter("tableNamePattern")).isEqualTo("feat%"); enqueue(200, "{\"tables\":[\"labels\"]}"); - assertThat(scoped.listTablesPaged(DATABASE, 1, "next", null, null).getElements()) + assertThat(api.listTablesPaged(database, 1, "next", null, null).getElements()) .containsExactly("labels"); assertThat( takeRequest("GET", scope + "/tables") @@ -198,55 +202,67 @@ void testReadFollowUpsAndPaginationReuseProtocol() throws Exception { .queryParameter("pageToken")) .isEqualTo("next"); - enqueue(200, "{\"tableDetails\":[" + tableResponse("physical-experiment") + "]}"); - assertThat(scoped.listTableDetails(DATABASE).get(0).getName()).isEqualTo("features"); + enqueue( + 200, + "{\"tableDetails\":[" + tableResponse(database, "physical-experiment", 2) + "]}"); + GetTableResponse details = api.listTableDetails(database).get(0); + assertThat(details.getName()).isEqualTo("features"); + assertThat(details.getDatabase()).isEqualTo(database); takeRequest("GET", scope + "/table-details"); enqueue(200, "{\"snapshot\":" + SNAPSHOT_JSON + "}"); - assertThat(scoped.loadSnapshot(TABLE, "LATEST").id()).isEqualTo(7); + assertThat(api.loadSnapshot(selected, "LATEST").id()).isEqualTo(7); takeRequest("GET", tablePath + "/snapshots/LATEST"); enqueue(200, "{\"snapshots\":[" + SNAPSHOT_JSON + "]}"); - assertThat(scoped.listSnapshotsPaged(TABLE, 10, null).getElements().get(0).id()) + assertThat(api.listSnapshotsPaged(selected, 10, null).getElements().get(0).id()) .isEqualTo(7); takeRequest("GET", tablePath + "/snapshots"); TableSchema schema = TableSchema.create(2, schema("physical-experiment")); enqueue(200, "{\"schemas\":[" + RESTApi.toJson(schema) + "]}"); - assertThat(scoped.listSchemasPaged(TABLE, 10, null).getElements()).containsExactly(schema); + assertThat(api.listSchemasPaged(selected, 10, null).getElements()).containsExactly(schema); takeRequest("GET", tablePath + "/schemas"); enqueue(200, "{\"token\":{\"key\":\"value\"},\"expiresAtMillis\":1234}"); - assertThat(scoped.loadTableToken(TABLE).getToken()).containsEntry("key", "value"); + assertThat(api.loadTableToken(selected).getToken()).containsEntry("key", "value"); takeRequest("GET", tablePath + "/token"); enqueue(200, "{\"filter\":[],\"columnMasking\":{}}"); - scoped.authTableQuery(TABLE, singletonList("id")); + api.authTableQuery(selected, singletonList("id")); assertThat(takeRequest("POST", tablePath + "/auth").getBody().readUtf8()) .isEqualTo("{\"select\":[\"id\"]}"); } @Test void testTableMutationsReuseRequestBodies() throws Exception { - RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); - for (RESTApi api : new RESTApi[] {catalog.api(), scoped}) { + Identifier selected = Identifier.create(DATABASE + "$branch_experiment", "features"); + RESTApi api = catalog.api(); + for (Identifier identifier : new Identifier[] {TABLE, selected}) { enqueue(200, "{}"); - api.createTable(TABLE, schema("main")); + api.createTable(identifier, schema("main")); enqueue(200, "{}"); - api.alterTable(TABLE, singletonList(SchemaChange.setOption("key", "value"))); + api.alterTable(identifier, singletonList(SchemaChange.setOption("key", "value"))); enqueue(200, "{}"); - api.dropTable(TABLE); + api.dropTable(identifier); } RecordedRequest[] original = { takeRequest("POST", DATABASE_PATH + "/tables"), takeRequest("POST", DATABASE_PATH + "/tables/features"), takeRequest("DELETE", DATABASE_PATH + "/tables/features") }; - String scope = DATABASE_PATH + "/trees/experiment"; + String scope = DATABASE_PATH + "%24branch_experiment"; RecordedRequest[] referenced = { takeRequest("POST", scope + "/tables"), takeRequest("POST", scope + "/tables/features"), takeRequest("DELETE", scope + "/tables/features") }; - for (int i = 0; i < original.length; i++) { + CreateTableRequest plain = + RESTApi.fromJson(original[0].getBody().readUtf8(), CreateTableRequest.class); + CreateTableRequest branch = + RESTApi.fromJson(referenced[0].getBody().readUtf8(), CreateTableRequest.class); + assertThat(plain.getIdentifier()).isEqualTo(TABLE); + assertThat(branch.getIdentifier()).isEqualTo(selected); + assertThat(branch.getSchema()).isEqualTo(plain.getSchema()); + for (int i = 1; i < original.length; i++) { assertThat(referenced[i].getBody().readUtf8()) .isEqualTo(original[i].getBody().readUtf8()); } @@ -254,50 +270,77 @@ void testTableMutationsReuseRequestBodies() throws Exception { @Test void testErrorsDoNotFallBackToDefaultBranch() throws Exception { - RESTCatalog scoped = catalog.withReference(DATABASE, "train_v1"); + Identifier selected = Identifier.create(DATABASE + "$tag_train_v1", "features"); enqueue(404, "{\"message\":\"reference missing\",\"code\":404}"); - assertThatThrownBy(() -> scoped.getTable(TABLE)) + assertThatThrownBy(() -> catalog.getTable(selected)) .isInstanceOf(Catalog.TableNotExistException.class); - takeRequest("GET", DATABASE_PATH + "/trees/train_v1/tables/features"); + takeRequest("GET", DATABASE_PATH + "%24tag_train_v1/tables/features"); enqueue(409, "{\"message\":\"tag is immutable\",\"code\":409}"); assertThatThrownBy( () -> - scoped.commitSnapshot( - TABLE, + catalog.commitSnapshot( + selected, "table-id", null, Snapshot.fromJson(SNAPSHOT_JSON), emptyList())) .isInstanceOf(AlreadyExistsException.class) .hasMessageContaining("tag is immutable"); - takeRequest("POST", DATABASE_PATH + "/trees/train_v1/tables/features/commit"); + takeRequest("POST", DATABASE_PATH + "%24tag_train_v1/tables/features/commit"); assertThat(server.getRequestCount()).isEqualTo(3); } @Test - void testUnsupportedSelectorsNeverSendAnUnscopedRequest() { - RESTApi scoped = catalog.api().withReference(DATABASE, "experiment"); - assertThatThrownBy(() -> catalog.withReference(DATABASE, null)) + void testUnsupportedDatabaseOperationsAndMixedSelectorsDoNotSendRequests() { + RESTApi api = catalog.api(); + String database = DATABASE + "$branch_experiment"; + Identifier selected = Identifier.create(database, "features"); + Identifier mixed = new Identifier(database, "features", "other"); + assertThatThrownBy(() -> api.getTable(mixed)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> api.createTable(mixed, schema("main"))) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> scoped.withReference(DATABASE, "../main")) + assertThatThrownBy(() -> api.listTables(DATABASE + "$tag_")) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> scoped.listTables("other")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining(DATABASE); - assertThatThrownBy(() -> scoped.getTable(new Identifier(DATABASE, "features", "other"))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("branch suffixes"); - assertThatThrownBy(() -> scoped.getTableById("table-id")) + assertThatThrownBy(() -> api.renameTable(selected, TABLE)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.renameTable(TABLE, selected)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createBranch(selected, "nested", null)) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.createDatabase(database, java.util.Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy( + () -> + api.alterDatabase( + database, emptyList(), java.util.Collections.emptyMap())) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> api.dropDatabase(database)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.listTablesPagedGlobally(null, null, null, null)) + assertThatThrownBy(() -> catalog.dropDatabase(database, true, false)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.renameTable(TABLE, Identifier.create(DATABASE, "renamed"))) + assertThatThrownBy(() -> catalog.dropDatabase(database, true, true)) .isInstanceOf(UnsupportedOperationException.class); - assertThatThrownBy(() -> scoped.createBranch(TABLE, "nested", null)) + assertThatThrownBy(() -> catalog.treeManagement().getReference(database, "main")) .isInstanceOf(UnsupportedOperationException.class); assertThat(server.getRequestCount()).isEqualTo(1); } + @ParameterizedTest + @ValueSource(strings = {"$branch_experiment", "$tag_train_v1"}) + void testDatabaseLookupPreservesVirtualName(String suffix) throws Exception { + String database = DATABASE + suffix; + enqueue( + 200, + "{\"name\":\"" + database + "\",\"location\":\"file:///training\",\"options\":{}}"); + assertThat(catalog.getDatabase(database).name()).isEqualTo(database); + takeRequest("GET", DATABASE_PATH + suffix.replace("$", "%24")); + enqueue(404, "{\"code\":404,\"message\":\"reference missing\"}"); + assertThatThrownBy(() -> catalog.getDatabase(database)) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + takeRequest("GET", DATABASE_PATH + suffix.replace("$", "%24")); + assertThat(server.getRequestCount()).isEqualTo(3); + } + @Test void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { // A small stateful fixture resolves references; production reference lifecycle is separate. @@ -315,13 +358,22 @@ void testBatchReadWriteAndPinnedTagWithRealDataFiles() throws Exception { public MockResponse dispatch(RecordedRequest request) { try { String route = request.getRequestUrl().encodedPath(); - String prefix = DATABASE_PATH + "/trees/"; + String prefix = "/v1/catalog%2Fid/databases/"; if (!route.startsWith(prefix)) { unexpected.add(route); return response(500, "{}"); } String[] parts = route.substring(prefix.length()).split("/"); - String reference = parts[0]; + DatabaseIdentifier database = + DatabaseIdentifier.parse(RESTUtil.decodeString(parts[0])); + if (!database.getDatabaseName().equals(DATABASE)) { + unexpected.add(route); + return response(500, "{}"); + } + String reference = + database.getReference() == null + ? "main" + : database.getReference().getName(); if (parts.length < 3 || !parts[1].equals("tables") || !parts[2].equals("features")) { @@ -331,7 +383,9 @@ public MockResponse dispatch(RecordedRequest request) { String branch = reference.equals("main") ? "main" : "physical-experiment"; if (request.getMethod().equals("GET") && parts.length == 3) { - return response(200, tableResponse(branch, 0)); + return response( + 200, + tableResponse(RESTUtil.decodeString(parts[0]), branch, 0)); } if (request.getMethod().equals("GET") && parts.length == 4 @@ -375,12 +429,12 @@ public MockResponse dispatch(RecordedRequest request) { } }); - RESTCatalog main = catalog.withReference(DATABASE, "main"); - RESTCatalog experiment = catalog.withReference(DATABASE, "experiment"); + Identifier main = Identifier.create(DATABASE + "$branch_main", "features"); + Identifier experiment = Identifier.create(DATABASE + "$branch_experiment", "features"); writeRows(main, 10); writeRows(experiment, 20); snapshots.put("train_v1", snapshots.get("experiment")); - RESTCatalog tag = catalog.withReference(DATABASE, "train_v1"); + Identifier tag = Identifier.create(DATABASE + "$tag_train_v1", "features"); assertThat(readRows(tag)).containsExactly(20); writeRows(experiment, 30); @@ -395,8 +449,8 @@ public MockResponse dispatch(RecordedRequest request) { assertThat(unexpected).isEmpty(); } - private void writeRows(RESTCatalog scoped, int value) throws Exception { - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + private void writeRows(Identifier selected, int value) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); BatchWriteBuilder builder = table.newBatchWriteBuilder(); try (BatchTableWrite write = builder.newWrite(); BatchTableCommit commit = builder.newCommit()) { @@ -405,8 +459,8 @@ private void writeRows(RESTCatalog scoped, int value) throws Exception { } } - private List readRows(RESTCatalog scoped) throws Exception { - FileStoreTable table = InstantiationUtil.clone((FileStoreTable) scoped.getTable(TABLE)); + private List readRows(Identifier selected) throws Exception { + FileStoreTable table = InstantiationUtil.clone((FileStoreTable) catalog.getTable(selected)); ReadBuilder builder = table.newReadBuilder(); List rows = new ArrayList<>(); try (RecordReader reader = @@ -420,19 +474,20 @@ private Schema schema(String branch) { return Schema.newBuilder() .column("id", DataTypes.INT()) .option("bucket", "-1") + .option("commit.max-retries", "0") .option(BRANCH.key(), branch) .build(); } private String tableResponse(String branch) throws Exception { - return tableResponse(branch, 2); + return tableResponse(DATABASE, branch, 2); } - private String tableResponse(String branch, long schemaId) throws Exception { + private String tableResponse(String database, String branch, long schemaId) throws Exception { return RESTApi.toJson( new GetTableResponse( "table-id", - DATABASE, + database, "features", tempDir.resolve("features").toUri().toString(), false,