From c745acd8a9065c8855b118a4a42ef5aca18eef1e Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 01:42:38 +0530 Subject: [PATCH 01/14] docs: add HTTP Gateway lakehouse quickstart --- website/docs/quickstart/gateway-lakehouse.md | 479 +++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 website/docs/quickstart/gateway-lakehouse.md diff --git a/website/docs/quickstart/gateway-lakehouse.md b/website/docs/quickstart/gateway-lakehouse.md new file mode 100644 index 00000000000..3201263ea17 --- /dev/null +++ b/website/docs/quickstart/gateway-lakehouse.md @@ -0,0 +1,479 @@ +--- +title: Real-Time Lakehouse via the HTTP Gateway +sidebar_position: 3 +--- + +This guide walks through the same real-time lakehouse pattern as the +[Streaming Lakehouse](lakehouse.md) quickstart, but creates the table and +writes records entirely over the [Fluss Gateway](/docs/gateway/index.md) +REST API instead of Flink SQL. You'll create a datalake-enabled table with +`curl`, ingest JSON records with `curl`, then use Flink SQL only to run the +Lakehouse Tiering Service and query the unified real-time + historical data +(Union Read). + +:::caution Preview +Fluss Gateway is introduced as a preview in Fluss 1.0. Its API and +configuration may change in later releases. The Gateway does not yet +support reading records — this guide reads data back through Flink SQL. +::: + +## Environment Setup + +### Prerequisites + +Before proceeding with this guide, ensure that [Docker](https://docs.docker.com/engine/install/) +and the [Docker Compose plugin](https://docs.docker.com/compose/install/linux/) +are installed on your machine. All commands were tested with Docker version +27.4.0 and Docker Compose version v2.30.3. + +:::note +We encourage you to use a recent version of Docker and [Compose v2](https://docs.docker.com/compose/releases/migrate/) +(however, Compose v1 might work with a few adaptions). +::: + +### Build the Gateway image + +A published Gateway image is not used by this quickstart; the guide builds the Gateway image locally from the source checkout. Run this from the root of your Fluss source checkout (the +script resolves the repository root itself, so you don't need to `cd` into +`docker/fluss-gateway` first): + +```shell +docker/fluss-gateway/build.sh +``` + +This compiles the Gateway inside a `rust:1.88-bookworm` builder container +(no local Rust toolchain needed) and produces a local image tagged +`fluss-gateway:dev`, which the Compose file below references directly. The first build may take several minutes because the Gateway binary is compiled from source. + +### Starting required components + +1. Create a working directory for this guide. + +```shell +mkdir fluss-quickstart-gateway-lakehouse +cd fluss-quickstart-gateway-lakehouse +``` + +2. Create a `lib` directory and download the Paimon S3 plugin jar required + by the Fluss servers: + +```shell +mkdir lib +curl -fL -o "lib/paimon-s3-$PAIMON_VERSION$.jar" "https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/$PAIMON_VERSION$/paimon-s3-$PAIMON_VERSION$.jar" +``` + +:::info +The `apache/fluss-quickstart-flink` image already includes the Flink-side +dependencies used by this guide. Only the Fluss server-side `paimon-s3` +plugin still needs to be downloaded and mounted into the Fluss containers. +::: + +3. Create a `docker-compose.yml` file with the following content. This + reuses the same Paimon-backed Fluss + Flink + RustFS stack as the + [Streaming Lakehouse](lakehouse.md) guide, with a `gateway` service added: + +```yaml +services: + #begin RustFS (S3-compatible storage) + rustfs: + image: rustfs/rustfs:1.0.0-alpha.83 + ports: + - "9000:9000" + - "9001:9001" + environment: + - RUSTFS_ACCESS_KEY=rustfsadmin + - RUSTFS_SECRET_KEY=rustfsadmin + - RUSTFS_CONSOLE_ENABLE=true + volumes: + - rustfs-data:/data + command: /data + rustfs-init: + image: minio/mc + depends_on: + - rustfs + entrypoint: > + /bin/sh -c " + until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin; do + echo 'Waiting for RustFS...'; + sleep 1; + done; + mc mb --ignore-existing rustfs/fluss; + " + #end + coordinator-server: + image: apache/fluss:$FLUSS_DOCKER_VERSION$ + command: coordinatorServer + depends_on: + zookeeper: + condition: service_started + rustfs-init: + condition: service_completed_successfully + environment: + - | + FLUSS_PROPERTIES= + zookeeper.address: zookeeper:2181 + bind.listeners: FLUSS://coordinator-server:9123 + remote.data.dir: s3://fluss/remote-data + s3.endpoint: http://rustfs:9000 + s3.access-key: rustfsadmin + s3.secret-key: rustfsadmin + s3.region: us-east-1 + s3.path-style-access: true + s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin + s3.assumed.role.sts.endpoint: http://rustfs:9000 + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: s3://fluss/paimon + datalake.paimon.s3.endpoint: http://rustfs:9000 + datalake.paimon.s3.access-key: rustfsadmin + datalake.paimon.s3.secret-key: rustfsadmin + datalake.paimon.s3.path.style.access: true + volumes: + - ./lib/paimon-s3-$PAIMON_VERSION$.jar:/opt/fluss/plugins/paimon/paimon-s3-$PAIMON_VERSION$.jar + tablet-server: + image: apache/fluss:$FLUSS_DOCKER_VERSION$ + command: tabletServer + depends_on: + - coordinator-server + environment: + - | + FLUSS_PROPERTIES= + zookeeper.address: zookeeper:2181 + bind.listeners: FLUSS://tablet-server:9123 + data.dir: /tmp/fluss/data + remote.data.dir: s3://fluss/remote-data + s3.endpoint: http://rustfs:9000 + s3.access-key: rustfsadmin + s3.secret-key: rustfsadmin + s3.region: us-east-1 + s3.path-style-access: true + s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin + s3.assumed.role.sts.endpoint: http://rustfs:9000 + datalake.enabled: true + datalake.format: paimon + datalake.paimon.metastore: filesystem + datalake.paimon.warehouse: s3://fluss/paimon + datalake.paimon.s3.endpoint: http://rustfs:9000 + datalake.paimon.s3.access-key: rustfsadmin + datalake.paimon.s3.secret-key: rustfsadmin + datalake.paimon.s3.path.style.access: true + volumes: + - ./lib/paimon-s3-$PAIMON_VERSION$.jar:/opt/fluss/plugins/paimon/paimon-s3-$PAIMON_VERSION$.jar + zookeeper: + restart: always + image: zookeeper:3.9.2 + #begin Fluss Gateway + gateway: + image: fluss-gateway:dev + depends_on: + - coordinator-server + ports: + - "8080:8080" + environment: + - FLUSS_GATEWAY__CLUSTER__DEFAULT__BOOTSTRAP__SERVERS=coordinator-server:9123 + #end + jobmanager: + image: apache/fluss-quickstart-flink:$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$ + ports: + - "8083:8081" + entrypoint: ["/opt/flink/init_paimon.sh"] + command: ["jobmanager"] + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager: + image: apache/fluss-quickstart-flink:$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$ + depends_on: + - jobmanager + entrypoint: ["/opt/flink/init_paimon.sh"] + command: ["taskmanager"] + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + taskmanager.numberOfTaskSlots: 10 + taskmanager.memory.process.size: 2048m + taskmanager.memory.task.off-heap.size: 128m + sql-client: + image: apache/fluss-quickstart-flink:$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$ + depends_on: + - jobmanager + entrypoint: ["/opt/flink/init_paimon.sh"] + command: ["/opt/sql-client/sql-client"] + environment: + - | + FLINK_PROPERTIES= + jobmanager.rpc.address: jobmanager + rest.address: jobmanager + +volumes: + rustfs-data: +``` + +The Docker Compose environment consists of the following containers: +- **Fluss Cluster:** a Fluss `CoordinatorServer`, a Fluss `TabletServer` and + a `ZooKeeper` server. +- **Fluss Gateway:** a stateless REST service used to create tables and + write records in this guide, listening on `localhost:8080`. +- **Flink Cluster**: a Flink `JobManager`, a Flink `TaskManager`, and a + Flink SQL client container, used to run the Lakehouse Tiering Service and + query results. +- **RustFS**: an S3-compatible storage system used both as Fluss remote + storage and Paimon's filesystem warehouse. + +:::tip +[RustFS](https://github.com/rustfs/rustfs) is used as replacement for S3 in +this quickstart example, for your production setup you may want to +configure this to use cloud file system. See [here](/maintenance/tiered-storage/filesystems/overview.md) +for information on how to setup cloud file systems. +::: + +4. To start all containers, run: + +```shell +docker compose up -d +``` + +Run + +```shell +docker compose ps +``` + +to check whether all containers are running properly, including `gateway`. +Check that the Gateway process is up and ready to accept requests: + +```shell +curl -sS --fail-with-body http://localhost:8080/health +curl -sS --fail-with-body http://localhost:8080/ready +``` + +:::note +`/health` only reports process liveness; `/ready` reports whether the +Gateway accepts requests but does not check Fluss connectivity itself. The +`gateway` service starts as soon as the `coordinator-server` container +starts, not once it's actually accepting connections, so the first +`/ready` call (or the first database-creation call below) can briefly +return an error or HTTP 503 right after `docker compose up`. Retry after a +few seconds if that happens. +::: + +Congratulations, you are all set! + +## Create a datalake-enabled table via the Gateway + +Set the endpoint and resource names used in the examples: + +```bash +GATEWAY_URL=http://localhost:8080 +CLUSTER=default +DATABASE=gateway_demo +``` + +### Create a database + +```bash +curl -sS --fail-with-body -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases" \ + -d "{\"database\":\"$DATABASE\"}" +``` + +### Create the table with lakehouse integration enabled + +Set `table.datalake.enabled` and `table.datalake.freshness` in `configs` at +creation time — the same properties the `lakehouse.md` guide sets via +`WITH (...)` in Flink SQL: + +```bash +curl -sS --fail-with-body -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables" \ + -d '{ + "table_name": "orders", + "columns": [ + {"name": "order_id", "data_type": {"type": "INTEGER"}, "nullable": false}, + {"name": "customer", "data_type": {"type": "STRING"}, "nullable": true}, + {"name": "amount_cents", "data_type": {"type": "BIGINT"}, "nullable": true}, + {"name": "status", "data_type": {"type": "STRING"}, "nullable": true} + ], + "primary_key": ["order_id"], + "distribution": {"bucket_count": 1, "bucket_keys": ["order_id"]}, + "configs": { + "table.datalake.enabled": "true", + "table.datalake.freshness": "30s" + } + }' +``` + +:::note +`amount_cents` stores the order amount as an integer number of cents, to +keep the HTTP payload simple. +::: + +### Write records + +```bash +curl -sS --fail-with-body -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ + -d '{ + "entries": [ + {"id": "order-1", "upsert": {"order_id": 1, "customer": "Alice", "amount_cents": 4599, "status": "placed"}}, + {"id": "order-2", "upsert": {"order_id": 2, "customer": "Bob", "amount_cents": 12000, "status": "placed"}}, + {"id": "order-3", "upsert": {"order_id": 3, "customer": "Carol", "amount_cents": 750, "status": "placed"}} + ] + }' +``` + +A successful response looks like: + +```json +{ + "row_count": 3, + "success_count": 3, + "error_count": 0, + "successes": [{"id": "order-1"}, {"id": "order-2"}, {"id": "order-3"}], + "failures": [] +} +``` + +## Start the Lakehouse Tiering Service + +To integrate with [Apache Paimon](https://paimon.apache.org/), start the +`Lakehouse Tiering Service`. Open a new terminal, navigate to the +`fluss-quickstart-gateway-lakehouse` directory, and run: + +```shell +docker compose exec jobmanager \ + /opt/flink/bin/flink run \ + /opt/flink/opt/fluss-flink-tiering-$FLUSS_VERSION$.jar \ + --fluss.bootstrap.servers coordinator-server:9123 \ + --datalake.format paimon \ + --datalake.paimon.metastore filesystem \ + --datalake.paimon.warehouse s3://fluss/paimon \ + --datalake.paimon.s3.endpoint http://rustfs:9000 \ + --datalake.paimon.s3.access.key rustfsadmin \ + --datalake.paimon.s3.secret.key rustfsadmin \ + --datalake.paimon.s3.path.style.access true +``` + +You should see a Flink Job tiering data from Fluss to Paimon running in the +[Flink Web UI](http://localhost:8083/). + +## Query with Union Read + +The Gateway's 1.0 preview doesn't support record reads, so this guide uses +Flink SQL to query the table you created over REST — Fluss tables are +identical regardless of which API created them. + +Enter the Flink SQL CLI container: + +```shell +docker compose run sql-client +``` + +Create the Fluss catalog and switch to it: + +```sql title="Flink SQL" +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:9123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); +``` + +```sql title="Flink SQL" +USE CATALOG fluss_catalog; +``` + +Switch to batch mode and query only the Paimon-tiered snapshot with the +`$lake` suffix: + +```sql title="Flink SQL" +SET 'sql-client.execution.result-mode' = 'tableau'; +``` + +```sql title="Flink SQL" +SET 'execution.runtime-mode' = 'batch'; +``` + +```sql title="Flink SQL" +-- wait for the ~30s datalake.freshness window before running this +SELECT snapshot_id, total_record_count FROM gateway_demo.orders$lake$snapshots; +``` + +```sql title="Flink SQL" +SELECT order_id, customer, amount_cents, status FROM gateway_demo.orders$lake; +``` + +Now query the table directly, which performs a Union Read. For a +primary-key table, `gateway_demo.orders` isn't a raw concatenation of two +stores — it gives you the **current unified view** of the table's state, +combining whatever's still in Fluss with what's already tiered to Paimon. +`gateway_demo.orders$lake`, by contrast, is the **lake-only view**: high +performance, but reflecting only what's been tiered so far. + +```sql title="Flink SQL" +SELECT order_id, customer, amount_cents, status FROM gateway_demo.orders; +``` + +To see this difference, write one more record through the Gateway from +another terminal: + +```bash +curl -sS --fail-with-body -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ + -d '{"entries": [{"id": "order-4", "upsert": {"order_id": 4, "customer": "Dave", "amount_cents": 2200, "status": "placed"}}]}' +``` + +Re-run the query on `gateway_demo.orders` in the SQL client — `order_id 4` +appears immediately in the unified view. The lake-only view, +`gateway_demo.orders$lake`, will reflect it once the tiering service has +processed it, subject to the configured `table.datalake.freshness`. + +### Quitting SQL Client + +```sql title="Flink SQL" +quit; +``` + +## Preview limitations + +- The Gateway's 1.0 preview only implements `trust` mode security — it does + not authenticate callers or terminate TLS. Don't expose `$GATEWAY_URL` + directly in production; put it behind an authenticated, TLS-terminating + ingress. +- The Gateway does not support record reads, primary-key/prefix lookups, or + log scans — this is why the guide reads back through Flink SQL. +- HTTP 200 from a write request can still contain partial failures; always + check both `successes` and `failures` in the response body. + +See the full [Fluss Gateway reference](/docs/gateway/index.md) for details. + +## Clean up + +Exit the SQL client, then delete the table and database through the +Gateway, then stop the containers: + +```bash +curl -sS --fail-with-body -X DELETE \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders" +curl -sS --fail-with-body -X DELETE \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE" +``` + +```shell +docker compose down -v +``` + +## Learn more + +Now that you're up and running with the Fluss Gateway and a real-time +lakehouse, check out the [Fluss Gateway reference](/docs/gateway/index.md) +for the full REST API, or the [Streaming Lakehouse](lakehouse.md) guide for +the equivalent all-Flink-SQL workflow. From 3fcea596bf3d99f8d22942baeb7d09c28ed7d123 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 01:45:07 +0530 Subject: [PATCH 02/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/test-gateway-quickstart.yml diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml new file mode 100644 index 00000000000..4f7ce69d64d --- /dev/null +++ b/.github/workflows/test-gateway-quickstart.yml @@ -0,0 +1,30 @@ +name: Test Gateway Lakehouse Quickstart + +on: + workflow_dispatch: + +jobs: + quickstart: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check Docker + run: | + docker version + docker compose version + + - name: Check Gateway source + run: | + test -f fluss-gateway/src/protocol/rest/ddl.rs + test -f fluss-gateway/src/protocol/rest/records.rs + + - name: Build Gateway image + run: | + docker/fluss-gateway/build.sh + + - name: Verify Gateway image + run: | + docker images fluss-gateway:dev \ No newline at end of file From 1af7868d6e7cbbb82e220864c758beba93b46cd1 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 01:55:24 +0530 Subject: [PATCH 03/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index 4f7ce69d64d..fa11f314d03 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -1,7 +1,9 @@ name: Test Gateway Lakehouse Quickstart on: - workflow_dispatch: + push: + branches: + - issue-4221-http-gateway-quickstart jobs: quickstart: From 255b31312a918d63888e88e558729329a39de76a Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:02:22 +0530 Subject: [PATCH 04/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 741 +++++++++++++++++- 1 file changed, 733 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index fa11f314d03..f145bfa3622 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -6,27 +6,752 @@ on: - issue-4221-http-gateway-quickstart jobs: - quickstart: + e2e: + name: Gateway Lakehouse E2E runs-on: ubuntu-latest + timeout-minutes: 40 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Check Docker run: | docker version docker compose version - - name: Check Gateway source - run: | - test -f fluss-gateway/src/protocol/rest/ddl.rs - test -f fluss-gateway/src/protocol/rest/records.rs - - name: Build Gateway image run: | docker/fluss-gateway/build.sh - name: Verify Gateway image run: | - docker images fluss-gateway:dev \ No newline at end of file + docker image inspect fluss-gateway:dev + + - name: Prepare quickstart from documentation + shell: bash + run: | + set -euo pipefail + + mkdir -p fluss-quickstart-gateway-lakehouse/lib + + python3 <<'PY' + import json + import re + from pathlib import Path + + doc = Path("website/docs/quickstart/gateway-lakehouse.md").read_text( + encoding="utf-8" + ) + + matches = re.findall( + r"```yaml\s*\n(.*?)\n```", + doc, + flags=re.DOTALL, + ) + + if not matches: + raise SystemExit("No YAML fenced block found in gateway-lakehouse.md") + + compose = matches[0] + + versions = json.loads( + Path("website/fluss-versions.json").read_text(encoding="utf-8") + ) + + current = next( + (entry for entry in versions if entry.get("versionName") == "next"), + None, + ) + + if current is None: + raise SystemExit( + "Could not find versionName=next in website/fluss-versions.json" + ) + + required = [ + "fullVersion", + "dockerVersion", + "paimonVersion", + ] + + for key in required: + if not current.get(key): + raise SystemExit(f"Missing {key} in next version entry") + + replacements = { + "$FLUSS_VERSION$": current["fullVersion"], + "$FLUSS_DOCKER_VERSION$": current["dockerVersion"], + "$PAIMON_VERSION$": current["paimonVersion"], + "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": current.get( + "quickstartFlinkDockerVersion", + f'2.2-{current["dockerVersion"]}', + ), + } + + for placeholder, value in replacements.items(): + compose = compose.replace(placeholder, value) + + unresolved = re.findall(r"\$[A-Z][A-Z0-9_]*\$", compose) + if unresolved: + raise SystemExit( + "Unresolved documentation placeholders: " + + ", ".join(sorted(set(unresolved))) + ) + + out = Path("fluss-quickstart-gateway-lakehouse/docker-compose.yml") + out.write_text(compose + "\n", encoding="utf-8") + + print("Resolved quickstart versions:") + for key, value in replacements.items(): + print(f" {key} = {value}") + print(f"Wrote {out}") + PY + + test -s fluss-quickstart-gateway-lakehouse/docker-compose.yml + + - name: Download Paimon S3 plugin + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + PAIMON_VERSION="$(python3 - <<'PY' + import json + from pathlib import Path + + versions = json.loads( + Path("../website/fluss-versions.json").read_text(encoding="utf-8") + ) + + current = next( + entry for entry in versions if entry.get("versionName") == "next" + ) + + print(current["paimonVersion"]) + PY + )" + + URL="https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" + + echo "Downloading ${URL}" + curl -fL --retry 5 --retry-all-errors \ + -o "lib/paimon-s3-${PAIMON_VERSION}.jar" \ + "${URL}" + + test -s "lib/paimon-s3-${PAIMON_VERSION}.jar" + + - name: Validate Compose configuration + working-directory: fluss-quickstart-gateway-lakehouse + run: | + docker compose config + + - name: Start Compose stack + working-directory: fluss-quickstart-gateway-lakehouse + run: | + docker compose up -d + + - name: Show Compose state + working-directory: fluss-quickstart-gateway-lakehouse + run: | + docker compose ps + + - name: Wait for core containers + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + services=( + zookeeper + rustfs + coordinator-server + tablet-server + gateway + jobmanager + taskmanager + ) + + for service in "${services[@]}"; do + echo "Waiting for ${service}..." + ready=0 + + for i in {1..60}; do + container_id="$(docker compose ps -q "${service}")" + + if [[ -n "${container_id}" ]]; then + state="$(docker inspect -f '{{.State.Status}}' "${container_id}")" + + if [[ "${state}" == "running" ]]; then + ready=1 + echo "${service}: running" + break + fi + fi + + sleep 2 + done + + if [[ "${ready}" -ne 1 ]]; then + echo "${service} did not become running" + docker compose ps + exit 1 + fi + done + + - name: Test Gateway health and readiness + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + echo "Waiting for Gateway /health..." + + for i in {1..60}; do + if curl -fsS --connect-timeout 2 --max-time 5 \ + http://localhost:8080/health; then + echo + echo "Gateway /health succeeded" + break + fi + + sleep 2 + + if [[ "${i}" -eq 60 ]]; then + echo "Gateway /health never became available" + exit 1 + fi + done + + echo "Checking Gateway /ready..." + curl -fsS --connect-timeout 2 --max-time 5 \ + http://localhost:8080/ready + + echo + echo "Gateway /ready succeeded" + + - name: Create database through Gateway + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + export GATEWAY_URL=http://localhost:8080 + export CLUSTER=default + export DATABASE=gateway_demo + + for i in {1..60}; do + code="$( + curl -sS \ + --connect-timeout 2 \ + --max-time 10 \ + -o /tmp/create-database.json \ + -w '%{http_code}' \ + -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases" \ + -d "{\"database\":\"$DATABASE\"}" \ + || true + )" + + echo "Create database HTTP status: ${code}" + + if [[ "${code}" == "200" || "${code}" == "201" ]]; then + cat /tmp/create-database.json + break + fi + + cat /tmp/create-database.json 2>/dev/null || true + echo + echo "Fluss may not be accepting Gateway requests yet; retrying..." + sleep 2 + + if [[ "${i}" -eq 60 ]]; then + echo "Database creation never succeeded" + exit 1 + fi + done + + - name: Create datalake-enabled table through Gateway + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + export GATEWAY_URL=http://localhost:8080 + export CLUSTER=default + export DATABASE=gateway_demo + + cat > /tmp/create-table.json <<'JSON' + { + "table_name": "orders", + "columns": [ + { + "name": "order_id", + "data_type": { + "type": "INTEGER" + }, + "nullable": false + }, + { + "name": "customer", + "data_type": { + "type": "STRING" + }, + "nullable": true + }, + { + "name": "amount_cents", + "data_type": { + "type": "BIGINT" + }, + "nullable": true + }, + { + "name": "status", + "data_type": { + "type": "STRING" + }, + "nullable": true + } + ], + "primary_key": [ + "order_id" + ], + "distribution": { + "bucket_count": 1, + "bucket_keys": [ + "order_id" + ] + }, + "configs": { + "table.datalake.enabled": "true", + "table.datalake.freshness": "30s" + } + } + JSON + + code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 30 \ + -o /tmp/create-table-response.json \ + -w '%{http_code}' \ + -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables" \ + --data-binary @/tmp/create-table.json + )" + + echo "Create table HTTP status: ${code}" + cat /tmp/create-table-response.json + echo + + [[ "${code}" == "200" || "${code}" == "201" ]] + + jq -e 'type == "object"' /tmp/create-table-response.json >/dev/null + + - name: Write three records through Gateway + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + export GATEWAY_URL=http://localhost:8080 + export CLUSTER=default + export DATABASE=gateway_demo + + cat > /tmp/write-records.json <<'JSON' + { + "entries": [ + { + "id": "order-1", + "upsert": { + "order_id": 1, + "customer": "Alice", + "amount_cents": 4599, + "status": "placed" + } + }, + { + "id": "order-2", + "upsert": { + "order_id": 2, + "customer": "Bob", + "amount_cents": 12000, + "status": "placed" + } + }, + { + "id": "order-3", + "upsert": { + "order_id": 3, + "customer": "Carol", + "amount_cents": 750, + "status": "placed" + } + } + ] + } + JSON + + code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 30 \ + -o /tmp/write-records-response.json \ + -w '%{http_code}' \ + -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ + --data-binary @/tmp/write-records.json + )" + + echo "Write HTTP status: ${code}" + cat /tmp/write-records-response.json + echo + + [[ "${code}" == "200" ]] + + jq -e ' + .row_count == 3 + and .success_count == 3 + and .error_count == 0 + and (.successes | length) == 3 + and (.failures | length) == 0 + ' /tmp/write-records-response.json + + - name: Wait for Flink cluster + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + echo "Waiting for Flink JobManager REST API..." + + for i in {1..60}; do + if curl -fsS --connect-timeout 2 --max-time 5 \ + http://localhost:8083/overview >/tmp/flink-overview.json; then + + echo "Flink JobManager is responding" + + taskmanagers="$( + curl -fsS --connect-timeout 2 --max-time 5 \ + http://localhost:8083/taskmanagers \ + | jq '.taskmanagers | length' + )" + + echo "TaskManagers: ${taskmanagers}" + + if [[ "${taskmanagers}" -ge 1 ]]; then + break + fi + fi + + sleep 2 + + if [[ "${i}" -eq 60 ]]; then + echo "Flink cluster never became ready" + exit 1 + fi + done + + - name: Start Lakehouse Tiering Service + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + FLUSS_VERSION="$( + python3 - <<'PY' + import json + from pathlib import Path + + versions = json.loads( + Path("../website/fluss-versions.json").read_text(encoding="utf-8") + ) + + current = next( + entry for entry in versions if entry.get("versionName") == "next" + ) + + print(current["fullVersion"]) + PY + )" + + echo "Starting tiering using Fluss version ${FLUSS_VERSION}" + + docker compose exec -T jobmanager \ + /opt/flink/bin/flink run -d \ + "/opt/flink/opt/fluss-flink-tiering-${FLUSS_VERSION}.jar" \ + --fluss.bootstrap.servers coordinator-server:9123 \ + --datalake.format paimon \ + --datalake.paimon.metastore filesystem \ + --datalake.paimon.warehouse s3://fluss/paimon \ + --datalake.paimon.s3.endpoint http://rustfs:9000 \ + --datalake.paimon.s3.access.key rustfsadmin \ + --datalake.paimon.s3.secret.key rustfsadmin \ + --datalake.paimon.s3.path.style.access true + + echo "Tiering job submitted" + + curl -fsS http://localhost:8083/jobs/overview | jq . + + - name: Verify Paimon snapshot and lake rows + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + run_sql() { + docker compose run --rm -T sql-client <<'SQL' + SET 'sql-client.execution.result-mode' = 'tableau'; + SET 'execution.runtime-mode' = 'batch'; + + USE CATALOG fluss_catalog; + USE gateway_demo; + + SELECT COUNT(*) AS snapshot_count + FROM orders$lake$snapshots; + + SELECT COUNT(*) AS lake_count + FROM orders$lake; + + SELECT COUNT(*) AS union_count + FROM orders; + SQL + } + + echo "Waiting for the first Paimon snapshot..." + + for i in {1..18}; do + output="$(run_sql 2>&1 || true)" + + echo "----- SQL attempt ${i} -----" + echo "${output}" + + snapshot_count="$( + printf '%s\n' "${output}" \ + | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ + | head -1 + )" + + lake_count="$( + printf '%s\n' "${output}" \ + | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ + | sed -n '2p' + )" + + union_count="$( + printf '%s\n' "${output}" \ + | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ + | sed -n '3p' + )" + + echo "snapshot_count=${snapshot_count:-unknown}" + echo "lake_count=${lake_count:-unknown}" + echo "union_count=${union_count:-unknown}" + + if [[ "${snapshot_count:-0}" =~ ^[0-9]+$ ]] && + [[ "${lake_count:-0}" =~ ^[0-9]+$ ]] && + [[ "${union_count:-0}" =~ ^[0-9]+$ ]] && + [[ "${snapshot_count}" -ge 1 ]] && + [[ "${lake_count}" -eq 3 ]] && + [[ "${union_count}" -eq 3 ]]; then + echo "Initial lakehouse state verified" + break + fi + + if [[ "${i}" -eq 18 ]]; then + echo "Initial lakehouse state was not reached" + exit 1 + fi + + sleep 10 + done + + - name: Write order-4 and verify Union Read freshness + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + export GATEWAY_URL=http://localhost:8080 + export CLUSTER=default + export DATABASE=gateway_demo + + code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 30 \ + -o /tmp/order-4-response.json \ + -w '%{http_code}' \ + -X POST \ + -H 'Content-Type: application/json' \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ + -d '{ + "entries": [ + { + "id": "order-4", + "upsert": { + "order_id": 4, + "customer": "Dave", + "amount_cents": 2200, + "status": "placed" + } + } + ] + }' + )" + + echo "Order-4 write HTTP status: ${code}" + cat /tmp/order-4-response.json + echo + + [[ "${code}" == "200" ]] + + jq -e ' + .row_count == 1 + and .success_count == 1 + and .error_count == 0 + and (.successes | length) == 1 + and (.failures | length) == 0 + ' /tmp/order-4-response.json + + echo "Waiting for Union Read to see order-4..." + + for i in {1..12}; do + output="$( + docker compose run --rm -T sql-client <<'SQL' + SET 'sql-client.execution.result-mode' = 'tableau'; + SET 'execution.runtime-mode' = 'batch'; + + USE CATALOG fluss_catalog; + USE gateway_demo; + + SELECT COUNT(*) AS union_count + FROM orders; + SQL + )" + + echo "${output}" + + if printf '%s\n' "${output}" \ + | grep -Eq '\|\s*4\s*\|'; then + echo "Union Read sees 4 rows" + break + fi + + if [[ "${i}" -eq 12 ]]; then + echo "Union Read did not reach 4 rows" + exit 1 + fi + + sleep 5 + done + + - name: Verify order-4 reaches Paimon + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + echo "Waiting for order-4 to reach the lake..." + + for i in {1..18}; do + output="$( + docker compose run --rm -T sql-client <<'SQL' + SET 'sql-client.execution.result-mode' = 'tableau'; + SET 'execution.runtime-mode' = 'batch'; + + USE CATALOG fluss_catalog; + USE gateway_demo; + + SELECT COUNT(*) AS lake_count + FROM orders$lake; + SQL + )" + + echo "${output}" + + if printf '%s\n' "${output}" \ + | grep -Eq '\|\s*4\s*\|'; then + echo "Paimon lake view sees 4 rows" + break + fi + + if [[ "${i}" -eq 18 ]]; then + echo "Paimon lake view did not reach 4 rows" + exit 1 + fi + + sleep 10 + done + + - name: Verify cleanup through Gateway + working-directory: fluss-quickstart-gateway-lakehouse + shell: bash + run: | + set -euo pipefail + + export GATEWAY_URL=http://localhost:8080 + export CLUSTER=default + export DATABASE=gateway_demo + + table_code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 30 \ + -o /tmp/delete-table-response.json \ + -w '%{http_code}' \ + -X DELETE \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders" + )" + + echo "Delete table HTTP status: ${table_code}" + cat /tmp/delete-table-response.json + echo + + [[ "${table_code}" == "200" || "${table_code}" == "204" ]] + + database_code="$( + curl -sS \ + --connect-timeout 5 \ + --max-time 30 \ + -o /tmp/delete-database-response.json \ + -w '%{http_code}' \ + -X DELETE \ + "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE" + )" + + echo "Delete database HTTP status: ${database_code}" + cat /tmp/delete-database-response.json + echo + + [[ "${database_code}" == "200" || "${database_code}" == "204" ]] + + - name: Shut down Compose stack + if: always() + working-directory: fluss-quickstart-gateway-lakehouse + run: | + docker compose down -v + + - name: Collect Docker diagnostics + if: failure() + working-directory: fluss-quickstart-gateway-lakehouse + run: | + mkdir -p ../gateway-e2e-diagnostics + + docker compose ps > ../gateway-e2e-diagnostics/compose-ps.txt || true + docker compose logs --no-color > ../gateway-e2e-diagnostics/compose.log || true + + docker version > ../gateway-e2e-diagnostics/docker-version.txt || true + docker compose version > ../gateway-e2e-diagnostics/compose-version.txt || true + + - name: Upload Docker diagnostics + if: failure() + uses: actions/upload-artifact@v4 + with: + name: gateway-lakehouse-e2e-diagnostics + path: gateway-e2e-diagnostics/ + if-no-files-found: ignore \ No newline at end of file From 38ab71b34e3a79a881e9e1ebd41294b8f4e28656 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:13:01 +0530 Subject: [PATCH 05/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 1043 ++++++++--------- 1 file changed, 503 insertions(+), 540 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index f145bfa3622..fb325f39567 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -4,285 +4,257 @@ on: push: branches: - issue-4221-http-gateway-quickstart + workflow_dispatch: jobs: - e2e: - name: Gateway Lakehouse E2E + gateway-lakehouse-e2e: runs-on: ubuntu-latest - timeout-minutes: 40 + timeout-minutes: 45 steps: - - name: Checkout repository - uses: actions/checkout@v6 + - name: Checkout + uses: actions/checkout@v4 - - name: Check Docker + - name: Show versions run: | docker version docker compose version + java -version - - name: Build Gateway image - run: | - docker/fluss-gateway/build.sh - - - name: Verify Gateway image - run: | - docker image inspect fluss-gateway:dev - - - name: Prepare quickstart from documentation + # ------------------------------------------------------------ + # Resolve the versions used by the documentation. + # ------------------------------------------------------------ + - name: Resolve quickstart versions + id: versions shell: bash run: | - set -euo pipefail - - mkdir -p fluss-quickstart-gateway-lakehouse/lib - - python3 <<'PY' + python3 - <<'PY' import json - import re from pathlib import Path - doc = Path("website/docs/quickstart/gateway-lakehouse.md").read_text( - encoding="utf-8" - ) - - matches = re.findall( - r"```yaml\s*\n(.*?)\n```", - doc, - flags=re.DOTALL, - ) - - if not matches: - raise SystemExit("No YAML fenced block found in gateway-lakehouse.md") - - compose = matches[0] - - versions = json.loads( - Path("website/fluss-versions.json").read_text(encoding="utf-8") - ) - - current = next( - (entry for entry in versions if entry.get("versionName") == "next"), - None, - ) - - if current is None: - raise SystemExit( - "Could not find versionName=next in website/fluss-versions.json" - ) - - required = [ - "fullVersion", - "dockerVersion", - "paimonVersion", - ] - - for key in required: - if not current.get(key): - raise SystemExit(f"Missing {key} in next version entry") - - replacements = { - "$FLUSS_VERSION$": current["fullVersion"], - "$FLUSS_DOCKER_VERSION$": current["dockerVersion"], - "$PAIMON_VERSION$": current["paimonVersion"], - "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": current.get( - "quickstartFlinkDockerVersion", - f'2.2-{current["dockerVersion"]}', - ), - } + data = json.loads(Path("website/fluss-versions.json").read_text()) + next_version = next(v for v in data if v["versionName"] == "next") - for placeholder, value in replacements.items(): - compose = compose.replace(placeholder, value) + print(f"FLUSS_VERSION={next_version['fullVersion']}") + print(f"FLUSS_DOCKER_VERSION={next_version['dockerVersion']}") + print(f"PAIMON_VERSION={next_version['paimonVersion']}") - unresolved = re.findall(r"\$[A-Z][A-Z0-9_]*\$", compose) - if unresolved: - raise SystemExit( - "Unresolved documentation placeholders: " - + ", ".join(sorted(set(unresolved))) - ) + flink_version = next_version.get("quickstartFlinkDockerVersion") + if not flink_version: + flink_version = next_version["flinkVersion"] - out = Path("fluss-quickstart-gateway-lakehouse/docker-compose.yml") - out.write_text(compose + "\n", encoding="utf-8") + print(f"FLINK_DOCKER_VERSION={flink_version}") - print("Resolved quickstart versions:") - for key, value in replacements.items(): - print(f" {key} = {value}") - print(f"Wrote {out}") + with open("${GITHUB_OUTPUT}", "a") as f: + f.write(f"fluss_version={next_version['fullVersion']}\n") + f.write(f"fluss_docker_version={next_version['dockerVersion']}\n") + f.write(f"paimon_version={next_version['paimonVersion']}\n") + f.write(f"flink_docker_version={flink_version}\n") PY - test -s fluss-quickstart-gateway-lakehouse/docker-compose.yml - - - name: Download Paimon S3 plugin - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Extract the Docker Compose example from the quickstart and + # substitute the documented version placeholders. + # ------------------------------------------------------------ + - name: Prepare Compose file from documentation shell: bash run: | - set -euo pipefail - - PAIMON_VERSION="$(python3 - <<'PY' - import json + python3 - <<'PY' + import re from pathlib import Path + import json + + md = Path("website/docs/quickstart/gateway-lakehouse.md").read_text() + + blocks = re.findall(r"```yaml\s*\n(.*?)```", md, re.S) + if not blocks: + raise SystemExit("No YAML fenced block found in quickstart") + + compose = blocks[0] - versions = json.loads( - Path("../website/fluss-versions.json").read_text(encoding="utf-8") - ) + versions = { + "$FLUSS_VERSION$": "${{ steps.versions.outputs.fluss_version }}", + "$FLUSS_DOCKER_VERSION$": "${{ steps.versions.outputs.fluss_docker_version }}", + "$PAIMON_VERSION$": "${{ steps.versions.outputs.paimon_version }}", + "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": + "${{ steps.versions.outputs.flink_docker_version }}", + } - current = next( - entry for entry in versions if entry.get("versionName") == "next" - ) + for old, new in versions.items(): + compose = compose.replace(old, new) - print(current["paimonVersion"]) + Path("/tmp/gateway-lakehouse-compose.yaml").write_text(compose) + print(compose) PY - )" - URL="https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" + echo "---- docker compose config ----" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + config + + # ------------------------------------------------------------ + # Build Apache Fluss from source. + # + # docker/fluss/Dockerfile explicitly COPYs build-target/, + # which is generated by the Maven build. + # ------------------------------------------------------------ + - name: Build Fluss distribution + run: | + ./mvnw clean package -DskipTests -T 1C - echo "Downloading ${URL}" - curl -fL --retry 5 --retry-all-errors \ - -o "lib/paimon-s3-${PAIMON_VERSION}.jar" \ - "${URL}" + test -d build-target + test -d fluss-dist/target - test -s "lib/paimon-s3-${PAIMON_VERSION}.jar" + echo "---- Fluss distribution ----" + find fluss-dist/target -maxdepth 1 -type f -print - - name: Validate Compose configuration - working-directory: fluss-quickstart-gateway-lakehouse + echo "---- build-target ----" + find build-target -maxdepth 2 -type f | head -100 + + - name: Build Fluss Docker image + run: | + docker build \ + -f docker/fluss/Dockerfile \ + -t apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} \ + docker/fluss + + docker image inspect \ + apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} + + # ------------------------------------------------------------ + # Build Gateway image. + # ------------------------------------------------------------ + - name: Build Gateway Docker image run: | - docker compose config + docker/fluss-gateway/build.sh - - name: Start Compose stack - working-directory: fluss-quickstart-gateway-lakehouse + docker image inspect fluss-gateway:dev + + # ------------------------------------------------------------ + # Pull/build everything needed before startup. + # ------------------------------------------------------------ + - name: Pull Compose images run: | - docker compose up -d + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + pull --ignore-buildable - - name: Show Compose state - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Start stack. + # ------------------------------------------------------------ + - name: Start Compose stack run: | - docker compose ps + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + up -d + + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + ps + # ------------------------------------------------------------ + # Wait for the core infrastructure. + # ------------------------------------------------------------ - name: Wait for core containers - working-directory: fluss-quickstart-gateway-lakehouse shell: bash run: | - set -euo pipefail - - services=( - zookeeper - rustfs - coordinator-server - tablet-server - gateway - jobmanager - taskmanager - ) - - for service in "${services[@]}"; do - echo "Waiting for ${service}..." - ready=0 - - for i in {1..60}; do - container_id="$(docker compose ps -q "${service}")" - - if [[ -n "${container_id}" ]]; then - state="$(docker inspect -f '{{.State.Status}}' "${container_id}")" - - if [[ "${state}" == "running" ]]; then - ready=1 - echo "${service}: running" - break - fi - fi - - sleep 2 - done - - if [[ "${ready}" -ne 1 ]]; then - echo "${service} did not become running" - docker compose ps - exit 1 + set -e + + compose() { + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + "$@" + } + + for i in {1..60}; do + if compose ps --status running | grep -q coordinator-server; then + echo "Coordinator is running" + break fi + + echo "Waiting for coordinator... ($i/60)" + sleep 5 done - - name: Test Gateway health and readiness - working-directory: fluss-quickstart-gateway-lakehouse + compose ps + + # ------------------------------------------------------------ + # Gateway health/readiness. + # ------------------------------------------------------------ + - name: Wait for Gateway shell: bash run: | - set -euo pipefail - - echo "Waiting for Gateway /health..." + set -e for i in {1..60}; do - if curl -fsS --connect-timeout 2 --max-time 5 \ - http://localhost:8080/health; then - echo - echo "Gateway /health succeeded" + if curl -sS --fail http://localhost:8080/health >/tmp/gateway-health.json 2>/tmp/gateway-health.err; then + echo "Gateway /health is ready" + cat /tmp/gateway-health.json break fi + echo "Waiting for Gateway /health... ($i/60)" sleep 2 + done - if [[ "${i}" -eq 60 ]]; then - echo "Gateway /health never became available" - exit 1 + curl -sS --fail-with-body http://localhost:8080/health + + for i in {1..60}; do + if curl -sS --fail http://localhost:8080/ready >/tmp/gateway-ready.json 2>/tmp/gateway-ready.err; then + echo "Gateway /ready is ready" + cat /tmp/gateway-ready.json + break fi - done - echo "Checking Gateway /ready..." - curl -fsS --connect-timeout 2 --max-time 5 \ - http://localhost:8080/ready + echo "Waiting for Gateway /ready... ($i/60)" + sleep 2 + done - echo - echo "Gateway /ready succeeded" + curl -sS --fail-with-body http://localhost:8080/ready - - name: Create database through Gateway - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Download Paimon S3 bundle into the SQL client container. + # ------------------------------------------------------------ + - name: Download Paimon S3 jar shell: bash run: | - set -euo pipefail + PAIMON_VERSION="${{ steps.versions.outputs.paimon_version }}" - export GATEWAY_URL=http://localhost:8080 - export CLUSTER=default - export DATABASE=gateway_demo + curl -fL \ + "https://repo1.maven.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" \ + -o /tmp/paimon-s3.jar - for i in {1..60}; do - code="$( - curl -sS \ - --connect-timeout 2 \ - --max-time 10 \ - -o /tmp/create-database.json \ - -w '%{http_code}' \ - -X POST \ - -H 'Content-Type: application/json' \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases" \ - -d "{\"database\":\"$DATABASE\"}" \ - || true - )" - - echo "Create database HTTP status: ${code}" - - if [[ "${code}" == "200" || "${code}" == "201" ]]; then - cat /tmp/create-database.json - break - fi + test -s /tmp/paimon-s3.jar - cat /tmp/create-database.json 2>/dev/null || true - echo - echo "Fluss may not be accepting Gateway requests yet; retrying..." - sleep 2 + # ------------------------------------------------------------ + # Create database through Gateway REST API. + # ------------------------------------------------------------ + - name: Create database + shell: bash + run: | + set -e - if [[ "${i}" -eq 60 ]]; then - echo "Database creation never succeeded" - exit 1 - fi - done + response="$( + curl -sS --fail-with-body \ + -X POST \ + http://localhost:8080/databases \ + -H 'Content-Type: application/json' \ + -d '{ + "database_name": "gateway_demo" + }' + )" + + echo "$response" - - name: Create datalake-enabled table through Gateway - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Create lake-enabled table through Gateway REST API. + # ------------------------------------------------------------ + - name: Create lake-enabled table shell: bash run: | - set -euo pipefail + set -e - export GATEWAY_URL=http://localhost:8080 - export CLUSTER=default - export DATABASE=gateway_demo - - cat > /tmp/create-table.json <<'JSON' + cat >/tmp/create-table.json <<'JSON' { "table_name": "orders", "columns": [ @@ -331,427 +303,418 @@ jobs: } JSON - code="$( - curl -sS \ - --connect-timeout 5 \ - --max-time 30 \ - -o /tmp/create-table-response.json \ - -w '%{http_code}' \ + response="$( + curl -sS --fail-with-body \ -X POST \ + http://localhost:8080/databases/gateway_demo/tables \ -H 'Content-Type: application/json' \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables" \ - --data-binary @/tmp/create-table.json + --data @/tmp/create-table.json )" - echo "Create table HTTP status: ${code}" - cat /tmp/create-table-response.json - echo - - [[ "${code}" == "200" || "${code}" == "201" ]] + echo "$response" - jq -e 'type == "object"' /tmp/create-table-response.json >/dev/null - - - name: Write three records through Gateway - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Write records through Gateway. + # ------------------------------------------------------------ + - name: Write initial records shell: bash run: | - set -euo pipefail - - export GATEWAY_URL=http://localhost:8080 - export CLUSTER=default - export DATABASE=gateway_demo + set -e - cat > /tmp/write-records.json <<'JSON' + cat >/tmp/write-records.json <<'JSON' { "entries": [ { - "id": "order-1", "upsert": { "order_id": 1, "customer": "Alice", - "amount_cents": 4599, - "status": "placed" + "amount_cents": 12500, + "status": "CREATED" } }, { - "id": "order-2", "upsert": { "order_id": 2, "customer": "Bob", - "amount_cents": 12000, - "status": "placed" + "amount_cents": 8900, + "status": "PAID" } }, { - "id": "order-3", "upsert": { "order_id": 3, "customer": "Carol", - "amount_cents": 750, - "status": "placed" + "amount_cents": 4200, + "status": "SHIPPED" } } ] } JSON - code="$( - curl -sS \ - --connect-timeout 5 \ - --max-time 30 \ - -o /tmp/write-records-response.json \ - -w '%{http_code}' \ + response="$( + curl -sS --fail-with-body \ -X POST \ + http://localhost:8080/databases/gateway_demo/tables/orders/records \ -H 'Content-Type: application/json' \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ - --data-binary @/tmp/write-records.json + --data @/tmp/write-records.json )" - echo "Write HTTP status: ${code}" - cat /tmp/write-records-response.json - echo + echo "$response" - [[ "${code}" == "200" ]] + echo "$response" | grep -q "upsert" + echo "$response" | grep -q "order_id" - jq -e ' - .row_count == 3 - and .success_count == 3 - and .error_count == 0 - and (.successes | length) == 3 - and (.failures | length) == 0 - ' /tmp/write-records-response.json - - - name: Wait for Flink cluster - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Wait for Flink. + # ------------------------------------------------------------ + - name: Wait for Flink shell: bash run: | - set -euo pipefail - - echo "Waiting for Flink JobManager REST API..." + set -e for i in {1..60}; do - if curl -fsS --connect-timeout 2 --max-time 5 \ - http://localhost:8083/overview >/tmp/flink-overview.json; then - - echo "Flink JobManager is responding" - - taskmanagers="$( - curl -fsS --connect-timeout 2 --max-time 5 \ - http://localhost:8083/taskmanagers \ - | jq '.taskmanagers | length' - )" - - echo "TaskManagers: ${taskmanagers}" - - if [[ "${taskmanagers}" -ge 1 ]]; then - break - fi + if docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + exec -T jobmanager \ + /opt/flink/bin/flink list >/tmp/flink-list.out 2>/tmp/flink-list.err; then + echo "Flink is ready" + cat /tmp/flink-list.out + break fi - sleep 2 - - if [[ "${i}" -eq 60 ]]; then - echo "Flink cluster never became ready" - exit 1 - fi + echo "Waiting for Flink... ($i/60)" + sleep 3 done + # ------------------------------------------------------------ + # Start Lakehouse Tiering Service. + # ------------------------------------------------------------ - name: Start Lakehouse Tiering Service - working-directory: fluss-quickstart-gateway-lakehouse shell: bash run: | - set -euo pipefail + set -e - FLUSS_VERSION="$( - python3 - <<'PY' - import json - from pathlib import Path + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + exec -T jobmanager \ + /opt/flink/bin/flink run -d \ + /opt/flink/opt/fluss-flink-tiering-${{ steps.versions.outputs.fluss_version }}.jar \ + --fluss.bootstrap.servers coordinator-server:9123 \ + --datalake.format paimon \ + --datalake.paimon.metastore filesystem \ + --datalake.paimon.warehouse s3://fluss/paimon \ + --datalake.paimon.s3.endpoint http://rustfs:9000 \ + --datalake.paimon.s3.access.key rustfsadmin \ + --datalake.paimon.s3.secret.key rustfsadmin \ + --datalake.paimon.s3.path.style.access true + + sleep 5 + + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + exec -T jobmanager \ + /opt/flink/bin/flink list + + # ------------------------------------------------------------ + # Install Paimon S3 connector in SQL client. + # ------------------------------------------------------------ + - name: Prepare SQL client + shell: bash + run: | + set -e - versions = json.loads( - Path("../website/fluss-versions.json").read_text(encoding="utf-8") - ) + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + cp /tmp/paimon-s3.jar sql-client:/tmp/paimon-s3.jar - current = next( - entry for entry in versions if entry.get("versionName") == "next" - ) + # ------------------------------------------------------------ + # Helper script for SQL queries. + # ------------------------------------------------------------ + - name: Wait for initial lake snapshot + shell: bash + run: | + set -e - print(current["fullVersion"]) - PY - )" + for i in {1..60}; do + output="$( + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + run --rm -T sql-client \ + bash -lc ' + cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar + + /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' + USE CATALOG fluss_catalog; + USE gateway_demo; + SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; + + SELECT * FROM orders$lake$snapshots; + SELECT COUNT(*) FROM orders$lake; + SELECT COUNT(*) FROM orders; + SQL + ' + )" || true + + echo "$output" + + if echo "$output" | grep -qE '3.*3|3[[:space:]]*\|[[:space:]]*3'; then + echo "Lake data is visible" + break + fi - echo "Starting tiering using Fluss version ${FLUSS_VERSION}" + echo "Waiting for tiering... ($i/60)" + sleep 5 + done - docker compose exec -T jobmanager \ - /opt/flink/bin/flink run -d \ - "/opt/flink/opt/fluss-flink-tiering-${FLUSS_VERSION}.jar" \ - --fluss.bootstrap.servers coordinator-server:9123 \ - --datalake.format paimon \ - --datalake.paimon.metastore filesystem \ - --datalake.paimon.warehouse s3://fluss/paimon \ - --datalake.paimon.s3.endpoint http://rustfs:9000 \ - --datalake.paimon.s3.access.key rustfsadmin \ - --datalake.paimon.s3.secret.key rustfsadmin \ - --datalake.paimon.s3.path.style.access true - - echo "Tiering job submitted" - - curl -fsS http://localhost:8083/jobs/overview | jq . - - - name: Verify Paimon snapshot and lake rows - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Validate snapshot metadata, lake data and Union Read. + # ------------------------------------------------------------ + - name: Validate initial lakehouse state shell: bash run: | - set -euo pipefail - - run_sql() { - docker compose run --rm -T sql-client <<'SQL' - SET 'sql-client.execution.result-mode' = 'tableau'; - SET 'execution.runtime-mode' = 'batch'; + set -e - USE CATALOG fluss_catalog; - USE gateway_demo; + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + run --rm -T sql-client \ + bash -lc ' + cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - SELECT COUNT(*) AS snapshot_count - FROM orders$lake$snapshots; + /opt/flink/bin/sql-client.sh <<'"'"'"'"'SQL'"'"' + USE CATALOG fluss_catalog; + USE gateway_demo; + SET '"'"'"'"'execution.runtime-mode'"'"'"'='"'"'"'batch'"'"'"'; - SELECT COUNT(*) AS lake_count - FROM orders$lake; + SELECT * FROM orders$lake$snapshots; - SELECT COUNT(*) AS union_count - FROM orders; - SQL - } + SELECT COUNT(*) AS lake_count + FROM orders$lake; - echo "Waiting for the first Paimon snapshot..." - - for i in {1..18}; do - output="$(run_sql 2>&1 || true)" - - echo "----- SQL attempt ${i} -----" - echo "${output}" - - snapshot_count="$( - printf '%s\n' "${output}" \ - | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ - | head -1 - )" - - lake_count="$( - printf '%s\n' "${output}" \ - | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ - | sed -n '2p' - )" - - union_count="$( - printf '%s\n' "${output}" \ - | sed -nE 's/.*\|\s*([0-9]+)\s*\|.*/\1/p' \ - | sed -n '3p' - )" - - echo "snapshot_count=${snapshot_count:-unknown}" - echo "lake_count=${lake_count:-unknown}" - echo "union_count=${union_count:-unknown}" - - if [[ "${snapshot_count:-0}" =~ ^[0-9]+$ ]] && - [[ "${lake_count:-0}" =~ ^[0-9]+$ ]] && - [[ "${union_count:-0}" =~ ^[0-9]+$ ]] && - [[ "${snapshot_count}" -ge 1 ]] && - [[ "${lake_count}" -eq 3 ]] && - [[ "${union_count}" -eq 3 ]]; then - echo "Initial lakehouse state verified" - break - fi + SELECT COUNT(*) AS union_count + FROM orders; - if [[ "${i}" -eq 18 ]]; then - echo "Initial lakehouse state was not reached" - exit 1 - fi + SELECT * + FROM orders + ORDER BY order_id; - sleep 10 - done + SQL + ' - - name: Write order-4 and verify Union Read freshness - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Fresh Gateway write after tiering has begun. + # ------------------------------------------------------------ + - name: Write fresh record through Gateway shell: bash run: | - set -euo pipefail - - export GATEWAY_URL=http://localhost:8080 - export CLUSTER=default - export DATABASE=gateway_demo - - code="$( - curl -sS \ - --connect-timeout 5 \ - --max-time 30 \ - -o /tmp/order-4-response.json \ - -w '%{http_code}' \ + set -e + + cat >/tmp/write-order-4.json <<'JSON' + { + "entries": [ + { + "upsert": { + "order_id": 4, + "customer": "Dave", + "amount_cents": 7777, + "status": "CREATED" + } + } + ] + } + JSON + + response="$( + curl -sS --fail-with-body \ -X POST \ + http://localhost:8080/databases/gateway_demo/tables/orders/records \ -H 'Content-Type: application/json' \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders/records" \ - -d '{ - "entries": [ - { - "id": "order-4", - "upsert": { - "order_id": 4, - "customer": "Dave", - "amount_cents": 2200, - "status": "placed" - } - } - ] - }' + --data @/tmp/write-order-4.json )" - echo "Order-4 write HTTP status: ${code}" - cat /tmp/order-4-response.json - echo + echo "$response" - [[ "${code}" == "200" ]] + echo "$response" | grep -q "upsert" + echo "$response" | grep -q "order_id" - jq -e ' - .row_count == 1 - and .success_count == 1 - and .error_count == 0 - and (.successes | length) == 1 - and (.failures | length) == 0 - ' /tmp/order-4-response.json - - echo "Waiting for Union Read to see order-4..." + # ------------------------------------------------------------ + # Union Read should see the hot record before lake tiering. + # ------------------------------------------------------------ + - name: Validate Union Read sees fresh Gateway write + shell: bash + run: | + set -e - for i in {1..12}; do + for i in {1..30}; do output="$( - docker compose run --rm -T sql-client <<'SQL' - SET 'sql-client.execution.result-mode' = 'tableau'; - SET 'execution.runtime-mode' = 'batch'; - - USE CATALOG fluss_catalog; - USE gateway_demo; - - SELECT COUNT(*) AS union_count - FROM orders; - SQL - )" - - echo "${output}" - - if printf '%s\n' "${output}" \ - | grep -Eq '\|\s*4\s*\|'; then - echo "Union Read sees 4 rows" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + run --rm -T sql-client \ + bash -lc ' + cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar + + /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' + USE CATALOG fluss_catalog; + USE gateway_demo; + SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; + + SELECT COUNT(*) AS union_count + FROM orders; + + SELECT * + FROM orders + WHERE order_id = 4; + SQL + ' + )" || true + + echo "$output" + + if echo "$output" | grep -q "4"; then + echo "Union Read sees order 4" break fi - if [[ "${i}" -eq 12 ]]; then - echo "Union Read did not reach 4 rows" - exit 1 - fi - - sleep 5 + echo "Waiting for fresh Union Read result... ($i/30)" + sleep 3 done - - name: Verify order-4 reaches Paimon - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Wait until the fresh record reaches the lake too. + # ------------------------------------------------------------ + - name: Validate fresh record is eventually tiered shell: bash run: | - set -euo pipefail - - echo "Waiting for order-4 to reach the lake..." + set -e - for i in {1..18}; do + for i in {1..60}; do output="$( - docker compose run --rm -T sql-client <<'SQL' - SET 'sql-client.execution.result-mode' = 'tableau'; - SET 'execution.runtime-mode' = 'batch'; - - USE CATALOG fluss_catalog; - USE gateway_demo; - - SELECT COUNT(*) AS lake_count - FROM orders$lake; - SQL - )" - - echo "${output}" - - if printf '%s\n' "${output}" \ - | grep -Eq '\|\s*4\s*\|'; then - echo "Paimon lake view sees 4 rows" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + run --rm -T sql-client \ + bash -lc ' + cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar + + /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' + USE CATALOG fluss_catalog; + USE gateway_demo; + SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; + + SELECT COUNT(*) AS lake_count + FROM orders$lake; + SQL + ' + )" || true + + echo "$output" + + if echo "$output" | grep -q "4"; then + echo "Four records are now tiered into the lake" break fi - if [[ "${i}" -eq 18 ]]; then - echo "Paimon lake view did not reach 4 rows" - exit 1 - fi - - sleep 10 + echo "Waiting for lake tiering... ($i/60)" + sleep 5 done - - name: Verify cleanup through Gateway - working-directory: fluss-quickstart-gateway-lakehouse + # ------------------------------------------------------------ + # Final validation. + # ------------------------------------------------------------ + - name: Final end-to-end validation shell: bash run: | - set -euo pipefail - - export GATEWAY_URL=http://localhost:8080 - export CLUSTER=default - export DATABASE=gateway_demo - - table_code="$( - curl -sS \ - --connect-timeout 5 \ - --max-time 30 \ - -o /tmp/delete-table-response.json \ - -w '%{http_code}' \ - -X DELETE \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE/tables/orders" - )" - - echo "Delete table HTTP status: ${table_code}" - cat /tmp/delete-table-response.json - echo - - [[ "${table_code}" == "200" || "${table_code}" == "204" ]] - - database_code="$( - curl -sS \ - --connect-timeout 5 \ - --max-time 30 \ - -o /tmp/delete-database-response.json \ - -w '%{http_code}' \ - -X DELETE \ - "$GATEWAY_URL/v1/clusters/$CLUSTER/databases/$DATABASE" - )" - - echo "Delete database HTTP status: ${database_code}" - cat /tmp/delete-database-response.json - echo - - [[ "${database_code}" == "200" || "${database_code}" == "204" ]] - - - name: Shut down Compose stack - if: always() - working-directory: fluss-quickstart-gateway-lakehouse - run: | - docker compose down -v - - - name: Collect Docker diagnostics - if: failure() - working-directory: fluss-quickstart-gateway-lakehouse + set -e + + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + run --rm -T sql-client \ + bash -lc ' + cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar + + /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' + USE CATALOG fluss_catalog; + USE gateway_demo; + SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; + + SELECT COUNT(*) AS snapshot_count + FROM orders$lake$snapshots; + + SELECT COUNT(*) AS lake_count + FROM orders$lake; + + SELECT COUNT(*) AS union_count + FROM orders; + + SELECT * + FROM orders + ORDER BY order_id; + SQL + ' + + # ------------------------------------------------------------ + # Verify Gateway cleanup. + # ------------------------------------------------------------ + - name: Cleanup through Gateway + shell: bash run: | - mkdir -p ../gateway-e2e-diagnostics + set -e - docker compose ps > ../gateway-e2e-diagnostics/compose-ps.txt || true - docker compose logs --no-color > ../gateway-e2e-diagnostics/compose.log || true + curl -sS --fail-with-body \ + -X DELETE \ + http://localhost:8080/databases/gateway_demo/tables/orders - docker version > ../gateway-e2e-diagnostics/docker-version.txt || true - docker compose version > ../gateway-e2e-diagnostics/compose-version.txt || true + curl -sS --fail-with-body \ + -X DELETE \ + http://localhost:8080/databases/gateway_demo - - name: Upload Docker diagnostics + # ------------------------------------------------------------ + # Always collect diagnostics on failure. + # ------------------------------------------------------------ + - name: Collect diagnostics if: failure() - uses: actions/upload-artifact@v4 - with: - name: gateway-lakehouse-e2e-diagnostics - path: gateway-e2e-diagnostics/ - if-no-files-found: ignore \ No newline at end of file + shell: bash + run: | + echo "================ docker compose ps ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + ps || true + + echo "================ coordinator logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=200 coordinator-server || true + + echo "================ tablet logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=200 tablet-server || true + + echo "================ gateway logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=300 gateway || true + + echo "================ jobmanager logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=300 jobmanager || true + + echo "================ taskmanager logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=300 taskmanager || true + + echo "================ rustfs logs ================" + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + logs --tail=200 rustfs || true + + # ------------------------------------------------------------ + # Always tear down containers/volumes. + # ------------------------------------------------------------ + - name: Tear down Compose stack + if: always() + run: | + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + down -v --remove-orphans \ No newline at end of file From 3207589cd0a6e1c918bb1243d1c95bda0f35a2c6 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:18:49 +0530 Subject: [PATCH 06/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index fb325f39567..ae7c2b03a9b 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -35,20 +35,34 @@ jobs: data = json.loads(Path("website/fluss-versions.json").read_text()) next_version = next(v for v in data if v["versionName"] == "next") - print(f"FLUSS_VERSION={next_version['fullVersion']}") - print(f"FLUSS_DOCKER_VERSION={next_version['dockerVersion']}") - print(f"PAIMON_VERSION={next_version['paimonVersion']}") + fluss_version = next_version["fullVersion"] + fluss_docker_version = next_version["dockerVersion"] + paimon_version = next_version["paimonVersion"] + + # The versions file used by this branch may expose the Flink + # Docker version under different names. Resolve the first one + # that exists rather than assuming flinkVersion. + flink_version = ( + next_version.get("quickstartFlinkDockerVersion") + or next_version.get("flinkDockerVersion") + or next_version.get("flinkVersion") + ) - flink_version = next_version.get("quickstartFlinkDockerVersion") if not flink_version: - flink_version = next_version["flinkVersion"] - + raise SystemExit( + "Could not find a Flink version. Available keys: " + + ", ".join(sorted(next_version.keys())) + ) + + print(f"FLUSS_VERSION={fluss_version}") + print(f"FLUSS_DOCKER_VERSION={fluss_docker_version}") + print(f"PAIMON_VERSION={paimon_version}") print(f"FLINK_DOCKER_VERSION={flink_version}") with open("${GITHUB_OUTPUT}", "a") as f: - f.write(f"fluss_version={next_version['fullVersion']}\n") - f.write(f"fluss_docker_version={next_version['dockerVersion']}\n") - f.write(f"paimon_version={next_version['paimonVersion']}\n") + f.write(f"fluss_version={fluss_version}\n") + f.write(f"fluss_docker_version={fluss_docker_version}\n") + f.write(f"paimon_version={paimon_version}\n") f.write(f"flink_docker_version={flink_version}\n") PY @@ -110,6 +124,11 @@ jobs: echo "---- build-target ----" find build-target -maxdepth 2 -type f | head -100 + + - name: Prepare Fluss Docker build context + run: | + rm -rf docker/fluss/build-target + cp -a build-target docker/fluss/build-target - name: Build Fluss Docker image run: | From aed0d67c569f5708c1715f278fc3aa8d62fb3e1b Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:21:25 +0530 Subject: [PATCH 07/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index ae7c2b03a9b..44d20df64ba 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -35,35 +35,16 @@ jobs: data = json.loads(Path("website/fluss-versions.json").read_text()) next_version = next(v for v in data if v["versionName"] == "next") - fluss_version = next_version["fullVersion"] - fluss_docker_version = next_version["dockerVersion"] - paimon_version = next_version["paimonVersion"] - - # The versions file used by this branch may expose the Flink - # Docker version under different names. Resolve the first one - # that exists rather than assuming flinkVersion. - flink_version = ( - next_version.get("quickstartFlinkDockerVersion") - or next_version.get("flinkDockerVersion") - or next_version.get("flinkVersion") - ) - - if not flink_version: - raise SystemExit( - "Could not find a Flink version. Available keys: " - + ", ".join(sorted(next_version.keys())) - ) - - print(f"FLUSS_VERSION={fluss_version}") - print(f"FLUSS_DOCKER_VERSION={fluss_docker_version}") - print(f"PAIMON_VERSION={paimon_version}") - print(f"FLINK_DOCKER_VERSION={flink_version}") + print(f"FLUSS_VERSION={next_version['fullVersion']}") + print(f"FLUSS_DOCKER_VERSION={next_version['dockerVersion']}") + print(f"PAIMON_VERSION={next_version['paimonVersion']}") + print("FLINK_DOCKER_VERSION=1.20-0.9.1-incubating") with open("${GITHUB_OUTPUT}", "a") as f: - f.write(f"fluss_version={fluss_version}\n") - f.write(f"fluss_docker_version={fluss_docker_version}\n") - f.write(f"paimon_version={paimon_version}\n") - f.write(f"flink_docker_version={flink_version}\n") + f.write(f"fluss_version={next_version['fullVersion']}\n") + f.write(f"fluss_docker_version={next_version['dockerVersion']}\n") + f.write(f"paimon_version={next_version['paimonVersion']}\n") + f.write("flink_docker_version=1.20-0.9.1-incubating\n") PY # ------------------------------------------------------------ @@ -124,7 +105,7 @@ jobs: echo "---- build-target ----" find build-target -maxdepth 2 -type f | head -100 - + - name: Prepare Fluss Docker build context run: | rm -rf docker/fluss/build-target From 551ede549cb9eaeb664634d1377dd35dc3daad44 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:24:07 +0530 Subject: [PATCH 08/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index 44d20df64ba..8cc3d09b1b9 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -30,21 +30,31 @@ jobs: run: | python3 - <<'PY' import json + import os from pathlib import Path data = json.loads(Path("website/fluss-versions.json").read_text()) - next_version = next(v for v in data if v["versionName"] == "next") - - print(f"FLUSS_VERSION={next_version['fullVersion']}") - print(f"FLUSS_DOCKER_VERSION={next_version['dockerVersion']}") - print(f"PAIMON_VERSION={next_version['paimonVersion']}") - print("FLINK_DOCKER_VERSION=1.20-0.9.1-incubating") - - with open("${GITHUB_OUTPUT}", "a") as f: - f.write(f"fluss_version={next_version['fullVersion']}\n") - f.write(f"fluss_docker_version={next_version['dockerVersion']}\n") - f.write(f"paimon_version={next_version['paimonVersion']}\n") - f.write("flink_docker_version=1.20-0.9.1-incubating\n") + next_version = next( + v for v in data if v["versionName"] == "next" + ) + + fluss_version = next_version["fullVersion"] + fluss_docker_version = next_version["dockerVersion"] + paimon_version = next_version["paimonVersion"] + flink_docker_version = "1.20-0.9.1-incubating" + + print(f"FLUSS_VERSION={fluss_version}") + print(f"FLUSS_DOCKER_VERSION={fluss_docker_version}") + print(f"PAIMON_VERSION={paimon_version}") + print(f"FLINK_DOCKER_VERSION={flink_docker_version}") + + output_file = os.environ["GITHUB_OUTPUT"] + + with open(output_file, "a") as f: + f.write(f"fluss_version={fluss_version}\n") + f.write(f"fluss_docker_version={fluss_docker_version}\n") + f.write(f"paimon_version={paimon_version}\n") + f.write(f"flink_docker_version={flink_docker_version}\n") PY # ------------------------------------------------------------ From 69f64ad53d5015d457a7ec649716e98466a009c1 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:32:01 +0530 Subject: [PATCH 09/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index 8cc3d09b1b9..88e5633cb49 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -118,15 +118,24 @@ jobs: - name: Prepare Fluss Docker build context run: | - rm -rf docker/fluss/build-target - cp -a build-target docker/fluss/build-target + rm -rf /tmp/fluss-docker-context + mkdir -p /tmp/fluss-docker-context + + cp -a docker/fluss/Dockerfile /tmp/fluss-docker-context/ + cp -a docker/fluss/docker-entrypoint.sh /tmp/fluss-docker-context/ + cp -a build-target /tmp/fluss-docker-context/build-target + + echo "---- Docker context ----" + du -sh /tmp/fluss-docker-context + test -d /tmp/fluss-docker-context/build-target + test -f /tmp/fluss-docker-context/Dockerfile + test -f /tmp/fluss-docker-context/docker-entrypoint.sh - name: Build Fluss Docker image run: | docker build \ - -f docker/fluss/Dockerfile \ -t apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} \ - docker/fluss + /tmp/fluss-docker-context docker image inspect \ apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} From ff3ad52b6d214e49314334590b700cf9c8215629 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:41:35 +0530 Subject: [PATCH 10/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 218 ++++++++++++------ 1 file changed, 146 insertions(+), 72 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index 88e5633cb49..5609255eaae 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -15,6 +15,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v5 + with: + java-version: "11" + distribution: temurin + - name: Show versions run: | docker version @@ -34,23 +40,19 @@ jobs: from pathlib import Path data = json.loads(Path("website/fluss-versions.json").read_text()) - next_version = next( - v for v in data if v["versionName"] == "next" - ) + next_version = next(v for v in data if v["versionName"] == "next") fluss_version = next_version["fullVersion"] fluss_docker_version = next_version["dockerVersion"] paimon_version = next_version["paimonVersion"] - flink_docker_version = "1.20-0.9.1-incubating" + flink_docker_version = "1.20-1.0-SNAPSHOT" print(f"FLUSS_VERSION={fluss_version}") print(f"FLUSS_DOCKER_VERSION={fluss_docker_version}") print(f"PAIMON_VERSION={paimon_version}") print(f"FLINK_DOCKER_VERSION={flink_docker_version}") - output_file = os.environ["GITHUB_OUTPUT"] - - with open(output_file, "a") as f: + with open(os.environ["GITHUB_OUTPUT"], "a") as f: f.write(f"fluss_version={fluss_version}\n") f.write(f"fluss_docker_version={fluss_docker_version}\n") f.write(f"paimon_version={paimon_version}\n") @@ -64,38 +66,51 @@ jobs: - name: Prepare Compose file from documentation shell: bash run: | + set -euo pipefail + + mkdir -p /tmp/lib + + curl -fL --retry 3 --retry-delay 2 \ + "https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" \ + -o "/tmp/lib/paimon-s3-${PAIMON_VERSION}.jar" + + test -s "/tmp/lib/paimon-s3-${PAIMON_VERSION}.jar" + python3 - <<'PY' + import os import re from pathlib import Path - import json md = Path("website/docs/quickstart/gateway-lakehouse.md").read_text() - blocks = re.findall(r"```yaml\s*\n(.*?)```", md, re.S) if not blocks: raise SystemExit("No YAML fenced block found in quickstart") compose = blocks[0] - - versions = { - "$FLUSS_VERSION$": "${{ steps.versions.outputs.fluss_version }}", - "$FLUSS_DOCKER_VERSION$": "${{ steps.versions.outputs.fluss_docker_version }}", - "$PAIMON_VERSION$": "${{ steps.versions.outputs.paimon_version }}", - "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": - "${{ steps.versions.outputs.flink_docker_version }}", + substitutions = { + "$FLUSS_VERSION$": os.environ["FLUSS_VERSION"], + "$FLUSS_DOCKER_VERSION$": os.environ["FLUSS_DOCKER_VERSION"], + "$PAIMON_VERSION$": os.environ["PAIMON_VERSION"], + "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": os.environ["FLINK_DOCKER_VERSION"], } - - for old, new in versions.items(): + for old, new in substitutions.items(): compose = compose.replace(old, new) + unresolved = [token for token in substitutions if token in compose] + if unresolved: + raise SystemExit("Unresolved placeholders: " + ", ".join(unresolved)) + Path("/tmp/gateway-lakehouse-compose.yaml").write_text(compose) print(compose) PY echo "---- docker compose config ----" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - config + docker compose -f /tmp/gateway-lakehouse-compose.yaml config + env: + FLUSS_VERSION: ${{ steps.versions.outputs.fluss_version }} + FLUSS_DOCKER_VERSION: ${{ steps.versions.outputs.fluss_docker_version }} + PAIMON_VERSION: ${{ steps.versions.outputs.paimon_version }} + FLINK_DOCKER_VERSION: ${{ steps.versions.outputs.flink_docker_version }} # ------------------------------------------------------------ # Build Apache Fluss from source. @@ -104,43 +119,82 @@ jobs: # which is generated by the Maven build. # ------------------------------------------------------------ - name: Build Fluss distribution + shell: bash run: | - ./mvnw clean package -DskipTests -T 1C + set -euo pipefail + + ./mvnw clean install -DskipTests -T 1C test -d build-target test -d fluss-dist/target - echo "---- Fluss distribution ----" - find fluss-dist/target -maxdepth 1 -type f -print - echo "---- build-target ----" + du -sh build-target find build-target -maxdepth 2 -type f | head -100 - name: Prepare Fluss Docker build context + shell: bash run: | + set -euo pipefail + rm -rf /tmp/fluss-docker-context mkdir -p /tmp/fluss-docker-context - cp -a docker/fluss/Dockerfile /tmp/fluss-docker-context/ cp -a docker/fluss/docker-entrypoint.sh /tmp/fluss-docker-context/ cp -a build-target /tmp/fluss-docker-context/build-target - echo "---- Docker context ----" - du -sh /tmp/fluss-docker-context - test -d /tmp/fluss-docker-context/build-target test -f /tmp/fluss-docker-context/Dockerfile test -f /tmp/fluss-docker-context/docker-entrypoint.sh + test -d /tmp/fluss-docker-context/build-target + + echo "---- Fluss Docker context ----" + du -sh /tmp/fluss-docker-context + du -sh /tmp/fluss-docker-context/build-target - name: Build Fluss Docker image + shell: bash run: | + set -euo pipefail + docker build \ -t apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} \ /tmp/fluss-docker-context - docker image inspect \ - apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} + docker image inspect apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} + + - name: Prepare quickstart-flink build + shell: bash + run: | + set -euo pipefail + + if [ -f docker/quickstart-flink/prepare_build.sh ]; then + docker/quickstart-flink/prepare_build.sh + else + echo "quickstart-flink/prepare_build.sh missing; fetching current main helper." + git fetch origin main --depth=1 + git show origin/main:docker/quickstart-flink/prepare_build.sh > docker/quickstart-flink/prepare_build.sh + chmod +x docker/quickstart-flink/prepare_build.sh + docker/quickstart-flink/prepare_build.sh + rm -f docker/quickstart-flink/prepare_build.sh + fi + + test -d docker/quickstart-flink/paimon + test -d docker/quickstart-flink/opt + test -d docker/quickstart-flink/lib + + - name: Build quickstart-flink Docker image + shell: bash + run: | + set -euo pipefail + + docker build \ + -t apache/fluss-quickstart-flink:${{ steps.versions.outputs.flink_docker_version }} \ + docker/quickstart-flink + + docker image inspect apache/fluss-quickstart-flink:${{ steps.versions.outputs.flink_docker_version }} # ------------------------------------------------------------ + # Build Gateway image. # ------------------------------------------------------------ # Build Gateway image. # ------------------------------------------------------------ - name: Build Gateway Docker image @@ -149,15 +203,6 @@ jobs: docker image inspect fluss-gateway:dev - # ------------------------------------------------------------ - # Pull/build everything needed before startup. - # ------------------------------------------------------------ - - name: Pull Compose images - run: | - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - pull --ignore-buildable - # ------------------------------------------------------------ # Start stack. # ------------------------------------------------------------ @@ -185,9 +230,11 @@ jobs: "$@" } + found=false for i in {1..60}; do if compose ps --status running | grep -q coordinator-server; then echo "Coordinator is running" + found=true break fi @@ -195,6 +242,12 @@ jobs: sleep 5 done + if [ "$found" != true ]; then + echo "Timed out waiting for coordinator-server" >&2 + compose ps + exit 1 + fi + compose ps # ------------------------------------------------------------ @@ -205,10 +258,12 @@ jobs: run: | set -e + health_ready=false for i in {1..60}; do if curl -sS --fail http://localhost:8080/health >/tmp/gateway-health.json 2>/tmp/gateway-health.err; then echo "Gateway /health is ready" cat /tmp/gateway-health.json + health_ready=true break fi @@ -216,12 +271,20 @@ jobs: sleep 2 done + if [ "$health_ready" != true ]; then + echo "Timed out waiting for Gateway /health" >&2 + cat /tmp/gateway-health.err || true + exit 1 + fi + curl -sS --fail-with-body http://localhost:8080/health + gateway_ready=false for i in {1..60}; do if curl -sS --fail http://localhost:8080/ready >/tmp/gateway-ready.json 2>/tmp/gateway-ready.err; then echo "Gateway /ready is ready" cat /tmp/gateway-ready.json + gateway_ready=true break fi @@ -229,21 +292,13 @@ jobs: sleep 2 done - curl -sS --fail-with-body http://localhost:8080/ready - - # ------------------------------------------------------------ - # Download Paimon S3 bundle into the SQL client container. - # ------------------------------------------------------------ - - name: Download Paimon S3 jar - shell: bash - run: | - PAIMON_VERSION="${{ steps.versions.outputs.paimon_version }}" + if [ "$gateway_ready" != true ]; then + echo "Timed out waiting for Gateway /ready" >&2 + cat /tmp/gateway-ready.err || true + exit 1 + fi - curl -fL \ - "https://repo1.maven.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" \ - -o /tmp/paimon-s3.jar - - test -s /tmp/paimon-s3.jar + curl -sS --fail-with-body http://localhost:8080/ready # ------------------------------------------------------------ # Create database through Gateway REST API. @@ -414,11 +469,21 @@ jobs: run: | set -e + tiering_jar="$( + docker compose \ + -f /tmp/gateway-lakehouse-compose.yaml \ + exec -T jobmanager \ + bash -lc 'find /opt/flink/opt -maxdepth 1 -name "fluss-flink-tiering-*.jar" -print -quit' + )" + + test -n "$tiering_jar" + echo "Using tiering jar: $tiering_jar" + docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ exec -T jobmanager \ /opt/flink/bin/flink run -d \ - /opt/flink/opt/fluss-flink-tiering-${{ steps.versions.outputs.fluss_version }}.jar \ + "$tiering_jar" \ --fluss.bootstrap.servers coordinator-server:9123 \ --datalake.format paimon \ --datalake.paimon.metastore filesystem \ @@ -435,18 +500,6 @@ jobs: exec -T jobmanager \ /opt/flink/bin/flink list - # ------------------------------------------------------------ - # Install Paimon S3 connector in SQL client. - # ------------------------------------------------------------ - - name: Prepare SQL client - shell: bash - run: | - set -e - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - cp /tmp/paimon-s3.jar sql-client:/tmp/paimon-s3.jar - # ------------------------------------------------------------ # Helper script for SQL queries. # ------------------------------------------------------------ @@ -455,11 +508,12 @@ jobs: run: | set -e + found=false for i in {1..60}; do output="$( docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T sql-client \ + run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ bash -lc ' cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar @@ -479,6 +533,7 @@ jobs: if echo "$output" | grep -qE '3.*3|3[[:space:]]*\|[[:space:]]*3'; then echo "Lake data is visible" + found=true break fi @@ -486,6 +541,11 @@ jobs: sleep 5 done + if [ "$found" != true ]; then + echo "Timed out waiting for the first lake snapshot / 3 lake rows" >&2 + exit 1 + fi + # ------------------------------------------------------------ # Validate snapshot metadata, lake data and Union Read. # ------------------------------------------------------------ @@ -496,7 +556,7 @@ jobs: docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T sql-client \ + run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ bash -lc ' cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar @@ -564,11 +624,12 @@ jobs: run: | set -e + found=false for i in {1..30}; do output="$( docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T sql-client \ + run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ bash -lc ' cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar @@ -591,6 +652,7 @@ jobs: if echo "$output" | grep -q "4"; then echo "Union Read sees order 4" + found=true break fi @@ -598,6 +660,11 @@ jobs: sleep 3 done + if [ "$found" != true ]; then + echo "Timed out waiting for Union Read to see order 4" >&2 + exit 1 + fi + # ------------------------------------------------------------ # Wait until the fresh record reaches the lake too. # ------------------------------------------------------------ @@ -606,11 +673,12 @@ jobs: run: | set -e + found=false for i in {1..60}; do output="$( docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T sql-client \ + run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ bash -lc ' cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar @@ -629,6 +697,7 @@ jobs: if echo "$output" | grep -q "4"; then echo "Four records are now tiered into the lake" + found=true break fi @@ -636,6 +705,11 @@ jobs: sleep 5 done + if [ "$found" != true ]; then + echo "Timed out waiting for the fourth record to reach the lake" >&2 + exit 1 + fi + # ------------------------------------------------------------ # Final validation. # ------------------------------------------------------------ @@ -646,7 +720,7 @@ jobs: docker compose \ -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T sql-client \ + run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ bash -lc ' cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar From 7d9d7f56f56859f0b43bf6bcea36e6d2c8e3b855 Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 02:52:03 +0530 Subject: [PATCH 11/14] test: validate gateway quickstart environment --- .github/workflows/test-gateway-quickstart.yml | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml index 5609255eaae..caaea87bf4f 100644 --- a/.github/workflows/test-gateway-quickstart.yml +++ b/.github/workflows/test-gateway-quickstart.yml @@ -126,41 +126,51 @@ jobs: ./mvnw clean install -DskipTests -T 1C test -d build-target - test -d fluss-dist/target + test -f build-target/bin/coordinator-server.sh + test -f build-target/bin/tablet-server.sh echo "---- build-target ----" du -sh build-target - find build-target -maxdepth 2 -type f | head -100 - name: Prepare Fluss Docker build context shell: bash run: | set -euo pipefail - rm -rf /tmp/fluss-docker-context - mkdir -p /tmp/fluss-docker-context - cp -a docker/fluss/Dockerfile /tmp/fluss-docker-context/ - cp -a docker/fluss/docker-entrypoint.sh /tmp/fluss-docker-context/ - cp -a build-target /tmp/fluss-docker-context/build-target + rm -rf "$RUNNER_TEMP/fluss-docker-context" + mkdir -p "$RUNNER_TEMP/fluss-docker-context" - test -f /tmp/fluss-docker-context/Dockerfile - test -f /tmp/fluss-docker-context/docker-entrypoint.sh - test -d /tmp/fluss-docker-context/build-target + cp docker/fluss/Dockerfile \ + "$RUNNER_TEMP/fluss-docker-context/Dockerfile" - echo "---- Fluss Docker context ----" - du -sh /tmp/fluss-docker-context - du -sh /tmp/fluss-docker-context/build-target + cp docker/fluss/docker-entrypoint.sh \ + "$RUNNER_TEMP/fluss-docker-context/docker-entrypoint.sh" + + cp -a build-target \ + "$RUNNER_TEMP/fluss-docker-context/build-target" + + echo "---- Docker context ----" + find "$RUNNER_TEMP/fluss-docker-context" -maxdepth 2 -type f | head -50 + du -sh "$RUNNER_TEMP/fluss-docker-context/build-target" + + test -f "$RUNNER_TEMP/fluss-docker-context/Dockerfile" + test -f "$RUNNER_TEMP/fluss-docker-context/docker-entrypoint.sh" + test -f "$RUNNER_TEMP/fluss-docker-context/build-target/bin/coordinator-server.sh" + test -f "$RUNNER_TEMP/fluss-docker-context/build-target/bin/tablet-server.sh" - name: Build Fluss Docker image shell: bash run: | set -euo pipefail - docker build \ + echo "Building from: $RUNNER_TEMP/fluss-docker-context" + + docker build --no-cache \ -t apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} \ - /tmp/fluss-docker-context + "$RUNNER_TEMP/fluss-docker-context" - docker image inspect apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} + docker image inspect \ + apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} - name: Prepare quickstart-flink build shell: bash From 4377ce7019b021968ce295d6489fe4f650dac91c Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Fri, 4 Sep 2026 03:02:54 +0530 Subject: [PATCH 12/14] [docs] Add HTTP Gateway lakehouse quickstart --- .github/workflows/test-gateway-quickstart.yml | 823 ------------------ website/docs/quickstart/gateway-lakehouse.md | 37 +- 2 files changed, 24 insertions(+), 836 deletions(-) delete mode 100644 .github/workflows/test-gateway-quickstart.yml diff --git a/.github/workflows/test-gateway-quickstart.yml b/.github/workflows/test-gateway-quickstart.yml deleted file mode 100644 index caaea87bf4f..00000000000 --- a/.github/workflows/test-gateway-quickstart.yml +++ /dev/null @@ -1,823 +0,0 @@ -name: Test Gateway Lakehouse Quickstart - -on: - push: - branches: - - issue-4221-http-gateway-quickstart - workflow_dispatch: - -jobs: - gateway-lakehouse-e2e: - runs-on: ubuntu-latest - timeout-minutes: 45 - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up JDK 11 - uses: actions/setup-java@v5 - with: - java-version: "11" - distribution: temurin - - - name: Show versions - run: | - docker version - docker compose version - java -version - - # ------------------------------------------------------------ - # Resolve the versions used by the documentation. - # ------------------------------------------------------------ - - name: Resolve quickstart versions - id: versions - shell: bash - run: | - python3 - <<'PY' - import json - import os - from pathlib import Path - - data = json.loads(Path("website/fluss-versions.json").read_text()) - next_version = next(v for v in data if v["versionName"] == "next") - - fluss_version = next_version["fullVersion"] - fluss_docker_version = next_version["dockerVersion"] - paimon_version = next_version["paimonVersion"] - flink_docker_version = "1.20-1.0-SNAPSHOT" - - print(f"FLUSS_VERSION={fluss_version}") - print(f"FLUSS_DOCKER_VERSION={fluss_docker_version}") - print(f"PAIMON_VERSION={paimon_version}") - print(f"FLINK_DOCKER_VERSION={flink_docker_version}") - - with open(os.environ["GITHUB_OUTPUT"], "a") as f: - f.write(f"fluss_version={fluss_version}\n") - f.write(f"fluss_docker_version={fluss_docker_version}\n") - f.write(f"paimon_version={paimon_version}\n") - f.write(f"flink_docker_version={flink_docker_version}\n") - PY - - # ------------------------------------------------------------ - # Extract the Docker Compose example from the quickstart and - # substitute the documented version placeholders. - # ------------------------------------------------------------ - - name: Prepare Compose file from documentation - shell: bash - run: | - set -euo pipefail - - mkdir -p /tmp/lib - - curl -fL --retry 3 --retry-delay 2 \ - "https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/${PAIMON_VERSION}/paimon-s3-${PAIMON_VERSION}.jar" \ - -o "/tmp/lib/paimon-s3-${PAIMON_VERSION}.jar" - - test -s "/tmp/lib/paimon-s3-${PAIMON_VERSION}.jar" - - python3 - <<'PY' - import os - import re - from pathlib import Path - - md = Path("website/docs/quickstart/gateway-lakehouse.md").read_text() - blocks = re.findall(r"```yaml\s*\n(.*?)```", md, re.S) - if not blocks: - raise SystemExit("No YAML fenced block found in quickstart") - - compose = blocks[0] - substitutions = { - "$FLUSS_VERSION$": os.environ["FLUSS_VERSION"], - "$FLUSS_DOCKER_VERSION$": os.environ["FLUSS_DOCKER_VERSION"], - "$PAIMON_VERSION$": os.environ["PAIMON_VERSION"], - "$FLUSS_QUICKSTART_FLINK_DOCKER_VERSION$": os.environ["FLINK_DOCKER_VERSION"], - } - for old, new in substitutions.items(): - compose = compose.replace(old, new) - - unresolved = [token for token in substitutions if token in compose] - if unresolved: - raise SystemExit("Unresolved placeholders: " + ", ".join(unresolved)) - - Path("/tmp/gateway-lakehouse-compose.yaml").write_text(compose) - print(compose) - PY - - echo "---- docker compose config ----" - docker compose -f /tmp/gateway-lakehouse-compose.yaml config - env: - FLUSS_VERSION: ${{ steps.versions.outputs.fluss_version }} - FLUSS_DOCKER_VERSION: ${{ steps.versions.outputs.fluss_docker_version }} - PAIMON_VERSION: ${{ steps.versions.outputs.paimon_version }} - FLINK_DOCKER_VERSION: ${{ steps.versions.outputs.flink_docker_version }} - - # ------------------------------------------------------------ - # Build Apache Fluss from source. - # - # docker/fluss/Dockerfile explicitly COPYs build-target/, - # which is generated by the Maven build. - # ------------------------------------------------------------ - - name: Build Fluss distribution - shell: bash - run: | - set -euo pipefail - - ./mvnw clean install -DskipTests -T 1C - - test -d build-target - test -f build-target/bin/coordinator-server.sh - test -f build-target/bin/tablet-server.sh - - echo "---- build-target ----" - du -sh build-target - - - name: Prepare Fluss Docker build context - shell: bash - run: | - set -euo pipefail - - rm -rf "$RUNNER_TEMP/fluss-docker-context" - mkdir -p "$RUNNER_TEMP/fluss-docker-context" - - cp docker/fluss/Dockerfile \ - "$RUNNER_TEMP/fluss-docker-context/Dockerfile" - - cp docker/fluss/docker-entrypoint.sh \ - "$RUNNER_TEMP/fluss-docker-context/docker-entrypoint.sh" - - cp -a build-target \ - "$RUNNER_TEMP/fluss-docker-context/build-target" - - echo "---- Docker context ----" - find "$RUNNER_TEMP/fluss-docker-context" -maxdepth 2 -type f | head -50 - du -sh "$RUNNER_TEMP/fluss-docker-context/build-target" - - test -f "$RUNNER_TEMP/fluss-docker-context/Dockerfile" - test -f "$RUNNER_TEMP/fluss-docker-context/docker-entrypoint.sh" - test -f "$RUNNER_TEMP/fluss-docker-context/build-target/bin/coordinator-server.sh" - test -f "$RUNNER_TEMP/fluss-docker-context/build-target/bin/tablet-server.sh" - - - name: Build Fluss Docker image - shell: bash - run: | - set -euo pipefail - - echo "Building from: $RUNNER_TEMP/fluss-docker-context" - - docker build --no-cache \ - -t apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} \ - "$RUNNER_TEMP/fluss-docker-context" - - docker image inspect \ - apache/fluss:${{ steps.versions.outputs.fluss_docker_version }} - - - name: Prepare quickstart-flink build - shell: bash - run: | - set -euo pipefail - - if [ -f docker/quickstart-flink/prepare_build.sh ]; then - docker/quickstart-flink/prepare_build.sh - else - echo "quickstart-flink/prepare_build.sh missing; fetching current main helper." - git fetch origin main --depth=1 - git show origin/main:docker/quickstart-flink/prepare_build.sh > docker/quickstart-flink/prepare_build.sh - chmod +x docker/quickstart-flink/prepare_build.sh - docker/quickstart-flink/prepare_build.sh - rm -f docker/quickstart-flink/prepare_build.sh - fi - - test -d docker/quickstart-flink/paimon - test -d docker/quickstart-flink/opt - test -d docker/quickstart-flink/lib - - - name: Build quickstart-flink Docker image - shell: bash - run: | - set -euo pipefail - - docker build \ - -t apache/fluss-quickstart-flink:${{ steps.versions.outputs.flink_docker_version }} \ - docker/quickstart-flink - - docker image inspect apache/fluss-quickstart-flink:${{ steps.versions.outputs.flink_docker_version }} - - # ------------------------------------------------------------ - # Build Gateway image. # ------------------------------------------------------------ - # Build Gateway image. - # ------------------------------------------------------------ - - name: Build Gateway Docker image - run: | - docker/fluss-gateway/build.sh - - docker image inspect fluss-gateway:dev - - # ------------------------------------------------------------ - # Start stack. - # ------------------------------------------------------------ - - name: Start Compose stack - run: | - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - up -d - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - ps - - # ------------------------------------------------------------ - # Wait for the core infrastructure. - # ------------------------------------------------------------ - - name: Wait for core containers - shell: bash - run: | - set -e - - compose() { - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - "$@" - } - - found=false - for i in {1..60}; do - if compose ps --status running | grep -q coordinator-server; then - echo "Coordinator is running" - found=true - break - fi - - echo "Waiting for coordinator... ($i/60)" - sleep 5 - done - - if [ "$found" != true ]; then - echo "Timed out waiting for coordinator-server" >&2 - compose ps - exit 1 - fi - - compose ps - - # ------------------------------------------------------------ - # Gateway health/readiness. - # ------------------------------------------------------------ - - name: Wait for Gateway - shell: bash - run: | - set -e - - health_ready=false - for i in {1..60}; do - if curl -sS --fail http://localhost:8080/health >/tmp/gateway-health.json 2>/tmp/gateway-health.err; then - echo "Gateway /health is ready" - cat /tmp/gateway-health.json - health_ready=true - break - fi - - echo "Waiting for Gateway /health... ($i/60)" - sleep 2 - done - - if [ "$health_ready" != true ]; then - echo "Timed out waiting for Gateway /health" >&2 - cat /tmp/gateway-health.err || true - exit 1 - fi - - curl -sS --fail-with-body http://localhost:8080/health - - gateway_ready=false - for i in {1..60}; do - if curl -sS --fail http://localhost:8080/ready >/tmp/gateway-ready.json 2>/tmp/gateway-ready.err; then - echo "Gateway /ready is ready" - cat /tmp/gateway-ready.json - gateway_ready=true - break - fi - - echo "Waiting for Gateway /ready... ($i/60)" - sleep 2 - done - - if [ "$gateway_ready" != true ]; then - echo "Timed out waiting for Gateway /ready" >&2 - cat /tmp/gateway-ready.err || true - exit 1 - fi - - curl -sS --fail-with-body http://localhost:8080/ready - - # ------------------------------------------------------------ - # Create database through Gateway REST API. - # ------------------------------------------------------------ - - name: Create database - shell: bash - run: | - set -e - - response="$( - curl -sS --fail-with-body \ - -X POST \ - http://localhost:8080/databases \ - -H 'Content-Type: application/json' \ - -d '{ - "database_name": "gateway_demo" - }' - )" - - echo "$response" - - # ------------------------------------------------------------ - # Create lake-enabled table through Gateway REST API. - # ------------------------------------------------------------ - - name: Create lake-enabled table - shell: bash - run: | - set -e - - cat >/tmp/create-table.json <<'JSON' - { - "table_name": "orders", - "columns": [ - { - "name": "order_id", - "data_type": { - "type": "INTEGER" - }, - "nullable": false - }, - { - "name": "customer", - "data_type": { - "type": "STRING" - }, - "nullable": true - }, - { - "name": "amount_cents", - "data_type": { - "type": "BIGINT" - }, - "nullable": true - }, - { - "name": "status", - "data_type": { - "type": "STRING" - }, - "nullable": true - } - ], - "primary_key": [ - "order_id" - ], - "distribution": { - "bucket_count": 1, - "bucket_keys": [ - "order_id" - ] - }, - "configs": { - "table.datalake.enabled": "true", - "table.datalake.freshness": "30s" - } - } - JSON - - response="$( - curl -sS --fail-with-body \ - -X POST \ - http://localhost:8080/databases/gateway_demo/tables \ - -H 'Content-Type: application/json' \ - --data @/tmp/create-table.json - )" - - echo "$response" - - # ------------------------------------------------------------ - # Write records through Gateway. - # ------------------------------------------------------------ - - name: Write initial records - shell: bash - run: | - set -e - - cat >/tmp/write-records.json <<'JSON' - { - "entries": [ - { - "upsert": { - "order_id": 1, - "customer": "Alice", - "amount_cents": 12500, - "status": "CREATED" - } - }, - { - "upsert": { - "order_id": 2, - "customer": "Bob", - "amount_cents": 8900, - "status": "PAID" - } - }, - { - "upsert": { - "order_id": 3, - "customer": "Carol", - "amount_cents": 4200, - "status": "SHIPPED" - } - } - ] - } - JSON - - response="$( - curl -sS --fail-with-body \ - -X POST \ - http://localhost:8080/databases/gateway_demo/tables/orders/records \ - -H 'Content-Type: application/json' \ - --data @/tmp/write-records.json - )" - - echo "$response" - - echo "$response" | grep -q "upsert" - echo "$response" | grep -q "order_id" - - # ------------------------------------------------------------ - # Wait for Flink. - # ------------------------------------------------------------ - - name: Wait for Flink - shell: bash - run: | - set -e - - for i in {1..60}; do - if docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - exec -T jobmanager \ - /opt/flink/bin/flink list >/tmp/flink-list.out 2>/tmp/flink-list.err; then - echo "Flink is ready" - cat /tmp/flink-list.out - break - fi - - echo "Waiting for Flink... ($i/60)" - sleep 3 - done - - # ------------------------------------------------------------ - # Start Lakehouse Tiering Service. - # ------------------------------------------------------------ - - name: Start Lakehouse Tiering Service - shell: bash - run: | - set -e - - tiering_jar="$( - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - exec -T jobmanager \ - bash -lc 'find /opt/flink/opt -maxdepth 1 -name "fluss-flink-tiering-*.jar" -print -quit' - )" - - test -n "$tiering_jar" - echo "Using tiering jar: $tiering_jar" - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - exec -T jobmanager \ - /opt/flink/bin/flink run -d \ - "$tiering_jar" \ - --fluss.bootstrap.servers coordinator-server:9123 \ - --datalake.format paimon \ - --datalake.paimon.metastore filesystem \ - --datalake.paimon.warehouse s3://fluss/paimon \ - --datalake.paimon.s3.endpoint http://rustfs:9000 \ - --datalake.paimon.s3.access.key rustfsadmin \ - --datalake.paimon.s3.secret.key rustfsadmin \ - --datalake.paimon.s3.path.style.access true - - sleep 5 - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - exec -T jobmanager \ - /opt/flink/bin/flink list - - # ------------------------------------------------------------ - # Helper script for SQL queries. - # ------------------------------------------------------------ - - name: Wait for initial lake snapshot - shell: bash - run: | - set -e - - found=false - for i in {1..60}; do - output="$( - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ - bash -lc ' - cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - - /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' - USE CATALOG fluss_catalog; - USE gateway_demo; - SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; - - SELECT * FROM orders$lake$snapshots; - SELECT COUNT(*) FROM orders$lake; - SELECT COUNT(*) FROM orders; - SQL - ' - )" || true - - echo "$output" - - if echo "$output" | grep -qE '3.*3|3[[:space:]]*\|[[:space:]]*3'; then - echo "Lake data is visible" - found=true - break - fi - - echo "Waiting for tiering... ($i/60)" - sleep 5 - done - - if [ "$found" != true ]; then - echo "Timed out waiting for the first lake snapshot / 3 lake rows" >&2 - exit 1 - fi - - # ------------------------------------------------------------ - # Validate snapshot metadata, lake data and Union Read. - # ------------------------------------------------------------ - - name: Validate initial lakehouse state - shell: bash - run: | - set -e - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ - bash -lc ' - cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - - /opt/flink/bin/sql-client.sh <<'"'"'"'"'SQL'"'"' - USE CATALOG fluss_catalog; - USE gateway_demo; - SET '"'"'"'"'execution.runtime-mode'"'"'"'='"'"'"'batch'"'"'"'; - - SELECT * FROM orders$lake$snapshots; - - SELECT COUNT(*) AS lake_count - FROM orders$lake; - - SELECT COUNT(*) AS union_count - FROM orders; - - SELECT * - FROM orders - ORDER BY order_id; - - SQL - ' - - # ------------------------------------------------------------ - # Fresh Gateway write after tiering has begun. - # ------------------------------------------------------------ - - name: Write fresh record through Gateway - shell: bash - run: | - set -e - - cat >/tmp/write-order-4.json <<'JSON' - { - "entries": [ - { - "upsert": { - "order_id": 4, - "customer": "Dave", - "amount_cents": 7777, - "status": "CREATED" - } - } - ] - } - JSON - - response="$( - curl -sS --fail-with-body \ - -X POST \ - http://localhost:8080/databases/gateway_demo/tables/orders/records \ - -H 'Content-Type: application/json' \ - --data @/tmp/write-order-4.json - )" - - echo "$response" - - echo "$response" | grep -q "upsert" - echo "$response" | grep -q "order_id" - - # ------------------------------------------------------------ - # Union Read should see the hot record before lake tiering. - # ------------------------------------------------------------ - - name: Validate Union Read sees fresh Gateway write - shell: bash - run: | - set -e - - found=false - for i in {1..30}; do - output="$( - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ - bash -lc ' - cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - - /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' - USE CATALOG fluss_catalog; - USE gateway_demo; - SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; - - SELECT COUNT(*) AS union_count - FROM orders; - - SELECT * - FROM orders - WHERE order_id = 4; - SQL - ' - )" || true - - echo "$output" - - if echo "$output" | grep -q "4"; then - echo "Union Read sees order 4" - found=true - break - fi - - echo "Waiting for fresh Union Read result... ($i/30)" - sleep 3 - done - - if [ "$found" != true ]; then - echo "Timed out waiting for Union Read to see order 4" >&2 - exit 1 - fi - - # ------------------------------------------------------------ - # Wait until the fresh record reaches the lake too. - # ------------------------------------------------------------ - - name: Validate fresh record is eventually tiered - shell: bash - run: | - set -e - - found=false - for i in {1..60}; do - output="$( - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ - bash -lc ' - cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - - /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' - USE CATALOG fluss_catalog; - USE gateway_demo; - SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; - - SELECT COUNT(*) AS lake_count - FROM orders$lake; - SQL - ' - )" || true - - echo "$output" - - if echo "$output" | grep -q "4"; then - echo "Four records are now tiered into the lake" - found=true - break - fi - - echo "Waiting for lake tiering... ($i/60)" - sleep 5 - done - - if [ "$found" != true ]; then - echo "Timed out waiting for the fourth record to reach the lake" >&2 - exit 1 - fi - - # ------------------------------------------------------------ - # Final validation. - # ------------------------------------------------------------ - - name: Final end-to-end validation - shell: bash - run: | - set -e - - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - run --rm -T -v /tmp/lib/paimon-s3-${{ steps.versions.outputs.paimon_version }}.jar:/tmp/paimon-s3.jar:ro sql-client \ - bash -lc ' - cp /tmp/paimon-s3.jar /opt/flink/lib/paimon-s3.jar - - /opt/flink/bin/sql-client.sh <<'"'"'SQL'"'"' - USE CATALOG fluss_catalog; - USE gateway_demo; - SET '"'"'execution.runtime-mode'"'"'='"'"'batch'"'"'; - - SELECT COUNT(*) AS snapshot_count - FROM orders$lake$snapshots; - - SELECT COUNT(*) AS lake_count - FROM orders$lake; - - SELECT COUNT(*) AS union_count - FROM orders; - - SELECT * - FROM orders - ORDER BY order_id; - SQL - ' - - # ------------------------------------------------------------ - # Verify Gateway cleanup. - # ------------------------------------------------------------ - - name: Cleanup through Gateway - shell: bash - run: | - set -e - - curl -sS --fail-with-body \ - -X DELETE \ - http://localhost:8080/databases/gateway_demo/tables/orders - - curl -sS --fail-with-body \ - -X DELETE \ - http://localhost:8080/databases/gateway_demo - - # ------------------------------------------------------------ - # Always collect diagnostics on failure. - # ------------------------------------------------------------ - - name: Collect diagnostics - if: failure() - shell: bash - run: | - echo "================ docker compose ps ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - ps || true - - echo "================ coordinator logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=200 coordinator-server || true - - echo "================ tablet logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=200 tablet-server || true - - echo "================ gateway logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=300 gateway || true - - echo "================ jobmanager logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=300 jobmanager || true - - echo "================ taskmanager logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=300 taskmanager || true - - echo "================ rustfs logs ================" - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - logs --tail=200 rustfs || true - - # ------------------------------------------------------------ - # Always tear down containers/volumes. - # ------------------------------------------------------------ - - name: Tear down Compose stack - if: always() - run: | - docker compose \ - -f /tmp/gateway-lakehouse-compose.yaml \ - down -v --remove-orphans \ No newline at end of file diff --git a/website/docs/quickstart/gateway-lakehouse.md b/website/docs/quickstart/gateway-lakehouse.md index 3201263ea17..0848e7bfea9 100644 --- a/website/docs/quickstart/gateway-lakehouse.md +++ b/website/docs/quickstart/gateway-lakehouse.md @@ -22,9 +22,8 @@ support reading records — this guide reads data back through Flink SQL. ### Prerequisites Before proceeding with this guide, ensure that [Docker](https://docs.docker.com/engine/install/) -and the [Docker Compose plugin](https://docs.docker.com/compose/install/linux/) -are installed on your machine. All commands were tested with Docker version -27.4.0 and Docker Compose version v2.30.3. +and the [Docker Compose plugin](https://docs.docker.com/compose/install/linux/) are +installed on your machine. :::note We encourage you to use a recent version of Docker and [Compose v2](https://docs.docker.com/compose/releases/migrate/) @@ -33,7 +32,8 @@ We encourage you to use a recent version of Docker and [Compose v2](https://docs ### Build the Gateway image -A published Gateway image is not used by this quickstart; the guide builds the Gateway image locally from the source checkout. Run this from the root of your Fluss source checkout (the +The Gateway isn't published as a Docker Hub image yet, so this guide builds +it from source. Run this from the root of your Fluss source checkout (the script resolves the repository root itself, so you don't need to `cd` into `docker/fluss-gateway` first): @@ -43,7 +43,8 @@ docker/fluss-gateway/build.sh This compiles the Gateway inside a `rust:1.88-bookworm` builder container (no local Rust toolchain needed) and produces a local image tagged -`fluss-gateway:dev`, which the Compose file below references directly. The first build may take several minutes because the Gateway binary is compiled from source. +`fluss-gateway:dev`, which the Compose file below references directly. The +first build may take several minutes. ### Starting required components @@ -328,6 +329,12 @@ curl -sS --fail-with-body -X POST \ }' ``` +:::note +`amount_cents` values fit comfortably in a JSON number here. For `BIGINT` +or `DECIMAL` values that exceed JSON number precision (values beyond 2^53), +pass them as base-10 strings instead — e.g. `"amount_cents": "9999999999999999"`. +::: + A successful response looks like: ```json @@ -390,6 +397,10 @@ CREATE CATALOG fluss_catalog WITH ( USE CATALOG fluss_catalog; ``` +```sql title="Flink SQL" +USE gateway_demo; +``` + Switch to batch mode and query only the Paimon-tiered snapshot with the `$lake` suffix: @@ -403,22 +414,22 @@ SET 'execution.runtime-mode' = 'batch'; ```sql title="Flink SQL" -- wait for the ~30s datalake.freshness window before running this -SELECT snapshot_id, total_record_count FROM gateway_demo.orders$lake$snapshots; +SELECT snapshot_id, total_record_count FROM orders$lake$snapshots; ``` ```sql title="Flink SQL" -SELECT order_id, customer, amount_cents, status FROM gateway_demo.orders$lake; +SELECT order_id, customer, amount_cents, status FROM orders$lake; ``` Now query the table directly, which performs a Union Read. For a -primary-key table, `gateway_demo.orders` isn't a raw concatenation of two +primary-key table, `orders` isn't a raw concatenation of two stores — it gives you the **current unified view** of the table's state, combining whatever's still in Fluss with what's already tiered to Paimon. -`gateway_demo.orders$lake`, by contrast, is the **lake-only view**: high +`orders$lake`, by contrast, is the **lake-only view**: high performance, but reflecting only what's been tiered so far. ```sql title="Flink SQL" -SELECT order_id, customer, amount_cents, status FROM gateway_demo.orders; +SELECT order_id, customer, amount_cents, status FROM orders; ``` To see this difference, write one more record through the Gateway from @@ -431,9 +442,9 @@ curl -sS --fail-with-body -X POST \ -d '{"entries": [{"id": "order-4", "upsert": {"order_id": 4, "customer": "Dave", "amount_cents": 2200, "status": "placed"}}]}' ``` -Re-run the query on `gateway_demo.orders` in the SQL client — `order_id 4` +Re-run the query on `orders` in the SQL client — `order_id 4` appears immediately in the unified view. The lake-only view, -`gateway_demo.orders$lake`, will reflect it once the tiering service has +`orders$lake`, will reflect it once the tiering service has processed it, subject to the configured `table.datalake.freshness`. ### Quitting SQL Client @@ -476,4 +487,4 @@ docker compose down -v Now that you're up and running with the Fluss Gateway and a real-time lakehouse, check out the [Fluss Gateway reference](/docs/gateway/index.md) for the full REST API, or the [Streaming Lakehouse](lakehouse.md) guide for -the equivalent all-Flink-SQL workflow. +the equivalent all-Flink-SQL workflow. \ No newline at end of file From c7aee0db584297aa96483085e2093c2f41f2721e Mon Sep 17 00:00:00 2001 From: pranavshuklaa Date: Sun, 6 Sep 2026 18:30:39 +0530 Subject: [PATCH 13/14] changes to incorporate minor fixes for issue-4221-http-gateway-quickstart --- website/docs/quickstart/gateway-lakehouse.md | 27 ++++++++++++++----- website/docs/quickstart/page_user_profile.mdx | 2 +- website/docs/quickstart/security.md | 2 +- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/website/docs/quickstart/gateway-lakehouse.md b/website/docs/quickstart/gateway-lakehouse.md index 0848e7bfea9..cd0a0233a0e 100644 --- a/website/docs/quickstart/gateway-lakehouse.md +++ b/website/docs/quickstart/gateway-lakehouse.md @@ -1,6 +1,6 @@ --- title: Real-Time Lakehouse via the HTTP Gateway -sidebar_position: 3 +sidebar_position: 4 --- This guide walks through the same real-time lakehouse pattern as the @@ -11,10 +11,17 @@ REST API instead of Flink SQL. You'll create a datalake-enabled table with Lakehouse Tiering Service and query the unified real-time + historical data (Union Read). -:::caution Preview -Fluss Gateway is introduced as a preview in Fluss 1.0. Its API and -configuration may change in later releases. The Gateway does not yet -support reading records — this guide reads data back through Flink SQL. +:::caution Developer Preview - Fluss 1.0 (unreleased) +Fluss Gateway is a developer preview introduced in Fluss 1.0, which has not +yet been released. No pre-built Docker Hub image is available yet, this guide +requires you to **build the Gateway image from the Fluss source repository** +(see [Build the Gateway image](#build-the-gateway-image) below). +If you are not comfortable building from source, check back once Fluss 1.0 +ships with a published 'apache/fluss-gateway' image. + +The Gateway API and configuration may change before the final release. +The Gateway does not yet support reading records - this guide reads data +back through Flink SQL. ::: ## Environment Setup @@ -32,6 +39,12 @@ We encourage you to use a recent version of Docker and [Compose v2](https://docs ### Build the Gateway image +:::note Prerequisite: Fluss source repository +This step requires the [Fluss source repository] (https://github.com/apache/fluss) +checked out locally. Unlike other quickstarts, no pre-built Gateway image is +published yet - the image is built from source using the script below. +::: + The Gateway isn't published as a Docker Hub image yet, so this guide builds it from source. Run this from the root of your Fluss source checkout (the script resolves the repository root itself, so you don't need to `cd` into @@ -450,8 +463,10 @@ processed it, subject to the configured `table.datalake.freshness`. ### Quitting SQL Client ```sql title="Flink SQL" -quit; +exit; ``` +After finishing the tutorial, run `exit` to exit the Flink SQL CLI +container. ## Preview limitations diff --git a/website/docs/quickstart/page_user_profile.mdx b/website/docs/quickstart/page_user_profile.mdx index 013ac55eed1..13dc4432a80 100644 --- a/website/docs/quickstart/page_user_profile.mdx +++ b/website/docs/quickstart/page_user_profile.mdx @@ -1,6 +1,6 @@ --- title: Real-Time Page User Profile -sidebar_position: 4 +sidebar_position: 5 --- import ThemedImage from '@site/src/components/ThemedImage'; diff --git a/website/docs/quickstart/security.md b/website/docs/quickstart/security.md index 603dad8b747..6b1018a6c0e 100644 --- a/website/docs/quickstart/security.md +++ b/website/docs/quickstart/security.md @@ -1,6 +1,6 @@ --- title: Secure Your Fluss Cluster -sidebar_position: 2 +sidebar_position: 3 ---