From 03a1dd6fa57292429c213bd66f3b5a9d30eed42e Mon Sep 17 00:00:00 2001 From: Elad Barkay Date: Tue, 29 Sep 2026 17:41:46 +0300 Subject: [PATCH] ci: replace MinIO with RustFS in the Spark, Trino and Dremio test stacks MinIO removed its images from Docker Hub (Sept 11) and then from quay.io (Sept 24), so the spark, trino and dremio CI jobs fail at image pull. Swap the S3-compatible storage in the three docker-compose stacks for RustFS (rustfs/rustfs:1.0.0) and the mc bucket-setup containers for the RustFS client (rustfs/rc:v0.1.36), pinned to exact versions. Rename the minio services, hosts, containers, volumes, credentials and seeder env vars to rustfs, and update the S3 endpoints in the Spark, Trino and Dremio configs and the CI wait step accordingly. Setup containers now wait on a storage healthcheck instead of a fixed sleep or TCP probe, and bucket creation is idempotent. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-warehouse.yml | 12 ++--- integration_tests/docker-compose-dremio.yml | 44 +++++++++---------- integration_tests/docker-compose-spark.yml | 34 +++++++------- integration_tests/docker-compose-trino.yml | 42 ++++++++++-------- .../docker/dremio/dremio-setup.sh | 2 +- .../docker/spark/spark-defaults.conf | 8 ++-- .../docker/trino/catalog/iceberg.properties | 6 +-- integration_tests/tests/data_seeder.py | 28 ++++++------ 8 files changed, 91 insertions(+), 85 deletions(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index c167516bc..ba304cea7 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -126,7 +126,7 @@ jobs: run: | docker compose -f docker-compose-dremio.yml up -d # Wait for Dremio to be healthy (one-shot containers like - # minio-setup exit immediately, so --wait would fail). + # rustfs-setup exit immediately, so --wait would fail). echo "Waiting for Dremio to become healthy..." timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} dremio 2>/dev/null)" = "healthy" ]; do sleep 5; done' echo "Dremio is healthy." @@ -154,13 +154,13 @@ jobs: run: | docker compose -f docker-compose-spark.yml build docker compose -f docker-compose-spark.yml up -d - echo "Waiting for MinIO setup to complete..." + echo "Waiting for RustFS setup to complete..." timeout 60 bash -c ' - until [ "$(docker inspect -f "{{.State.Status}}" spark-minio-setup 2>/dev/null)" = "exited" ]; do sleep 2; done - EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-minio-setup 2>/dev/null) - if [ "$EXIT_CODE" != "0" ]; then echo "MinIO setup failed with exit code $EXIT_CODE"; exit 1; fi + until [ "$(docker inspect -f "{{.State.Status}}" spark-rustfs-setup 2>/dev/null)" = "exited" ]; do sleep 2; done + EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-rustfs-setup 2>/dev/null) + if [ "$EXIT_CODE" != "0" ]; then echo "RustFS setup failed with exit code $EXIT_CODE"; exit 1; fi ' - echo "MinIO is ready." + echo "RustFS is ready." echo "Waiting for Spark Thrift Server to become healthy..." timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} spark-thrift 2>/dev/null)" = "healthy" ]; do sleep 5; done' echo "Spark Thrift Server is healthy." diff --git a/integration_tests/docker-compose-dremio.yml b/integration_tests/docker-compose-dremio.yml index 1799937c9..7cb880db7 100644 --- a/integration_tests/docker-compose-dremio.yml +++ b/integration_tests/docker-compose-dremio.yml @@ -8,39 +8,39 @@ services: ports: - 19120:19120 - # Minio Storage Server - minio: - image: minio/minio:latest + # RustFS Storage Server + rustfs: + image: rustfs/rustfs:1.0.0 container_name: storage environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=storage - - MINIO_REGION_NAME=us-east-1 - - MINIO_REGION=us-east-1 + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password networks: - dremio-lakehouse ports: - 9001:9001 - 9000:9000 - command: ["server", "/data", "--console-address", ":9001"] + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s volumes: - - minio_data:/data + - rustfs_data:/data - minio-setup: - image: minio/mc - container_name: minio-setup + rustfs-setup: + image: rustfs/rc:v0.1.36 + container_name: rustfs-setup depends_on: - - minio + rustfs: + condition: service_healthy entrypoint: > /bin/sh -c " - until (echo > /dev/tcp/minio/9000) >/dev/null 2>&1; do - echo 'Waiting for MinIO...'; - sleep 2; - done; - mc alias set myminio http://minio:9000 admin password; - mc mb myminio/datalake; - mc ls myminio; + set -e; + rc alias set local http://rustfs:9000 admin password; + rc bucket create --ignore-existing local/datalake; + rc ls local; " networks: - dremio-lakehouse @@ -89,4 +89,4 @@ networks: volumes: dremio_data: - minio_data: + rustfs_data: diff --git a/integration_tests/docker-compose-spark.yml b/integration_tests/docker-compose-spark.yml index 50da8a615..1d12cbf10 100644 --- a/integration_tests/docker-compose-spark.yml +++ b/integration_tests/docker-compose-spark.yml @@ -12,7 +12,7 @@ services: depends_on: spark-hive-metastore: condition: service_started - minio-setup: + rustfs-setup: condition: service_completed_successfully command: > --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 @@ -39,40 +39,40 @@ services: - POSTGRES_PASSWORD=dbt - POSTGRES_DB=metastore - minio: - image: minio/minio:latest - container_name: spark-minio + # S3-compatible storage for external seed tables. + rustfs: + image: rustfs/rustfs:1.0.0 + container_name: spark-rustfs ports: - "9000:9000" - "9001:9001" environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - command: ["server", "/data", "--console-address", ":9001"] + - RUSTFS_ACCESS_KEY=rustfs + - RUSTFS_SECRET_KEY=rustfs123 healthcheck: - test: ["CMD-SHELL", "mc ready local || exit 1"] + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] interval: 5s timeout: 5s retries: 10 start_period: 5s volumes: - - minio-data:/data + - rustfs-data:/data - minio-setup: - image: minio/mc - container_name: spark-minio-setup + rustfs-setup: + image: rustfs/rc:v0.1.36 + container_name: spark-rustfs-setup depends_on: - minio: + rustfs: condition: service_healthy entrypoint: > /bin/sh -c " set -e; - mc alias set myminio http://minio:9000 minioadmin minioadmin; - mc mb --ignore-existing myminio/spark-seeds; - echo 'MinIO bucket spark-seeds created.'; + rc alias set local http://rustfs:9000 rustfs rustfs123; + rc bucket create --ignore-existing local/spark-seeds; + echo 'Bucket spark-seeds created.'; " volumes: spark-warehouse: hive-metastore: - minio-data: + rustfs-data: diff --git a/integration_tests/docker-compose-trino.yml b/integration_tests/docker-compose-trino.yml index 51cea59a1..187ee7947 100644 --- a/integration_tests/docker-compose-trino.yml +++ b/integration_tests/docker-compose-trino.yml @@ -27,9 +27,9 @@ services: HIVE_METASTORE_USER: hive HIVE_METASTORE_PASSWORD: hive HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/ - S3_ENDPOINT: http://minio:9000 - S3_ACCESS_KEY: minio - S3_SECRET_KEY: minio123 + S3_ENDPOINT: http://rustfs:9000 + S3_ACCESS_KEY: rustfs + S3_SECRET_KEY: rustfs123 S3_PATH_STYLE_ACCESS: "true" # NOTE - don't remove all these empty vars, the hive metastore startup script actually verifies all of them exist... @@ -51,26 +51,32 @@ services: depends_on: - metastore_db - minio: - hostname: minio - image: "minio/minio:RELEASE.2022-05-26T05-48-41Z" - container_name: minio + rustfs: + hostname: rustfs + image: "rustfs/rustfs:1.0.0" + container_name: rustfs ports: - "9000:9000" - "9001:9001" environment: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - command: server /data --console-address ":9001" + RUSTFS_ACCESS_KEY: rustfs + RUSTFS_SECRET_KEY: rustfs123 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s - # This job will create the "datalake" bucket on Minio - mc-job: - image: "minio/mc:RELEASE.2022-05-09T04-08-26Z" + # This job will create the "datalake" bucket on RustFS + rustfs-setup: + image: "rustfs/rc:v0.1.36" entrypoint: | - /bin/bash -c " - sleep 5; - /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123; - /usr/bin/mc mb --quiet myminio/datalake + /bin/sh -c " + set -e; + rc alias set local http://rustfs:9000 rustfs rustfs123; + rc bucket create --ignore-existing local/datalake " depends_on: - - minio + rustfs: + condition: service_healthy diff --git a/integration_tests/docker/dremio/dremio-setup.sh b/integration_tests/docker/dremio/dremio-setup.sh index 7dec29605..dd3e48e96 100644 --- a/integration_tests/docker/dremio/dremio-setup.sh +++ b/integration_tests/docker/dremio/dremio-setup.sh @@ -28,6 +28,6 @@ echo "Obtained Dremio auth token" curl -s -X PUT "http://dremio:9047/apiv2/source/S3Source" \ -H "Content-Type: application/json" \ -H "Authorization: _dremio$AUTH_TOKEN" \ - --data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" + --data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"rustfs:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" echo "S3 Source created in Dremio" \ No newline at end of file diff --git a/integration_tests/docker/spark/spark-defaults.conf b/integration_tests/docker/spark/spark-defaults.conf index f7c143f3b..9a57c5d64 100644 --- a/integration_tests/docker/spark/spark-defaults.conf +++ b/integration_tests/docker/spark/spark-defaults.conf @@ -13,10 +13,10 @@ spark.default.parallelism 2 spark.ui.enabled false spark.sql.adaptive.enabled true -# S3A configuration for MinIO -spark.hadoop.fs.s3a.endpoint http://minio:9000 -spark.hadoop.fs.s3a.access.key minioadmin -spark.hadoop.fs.s3a.secret.key minioadmin +# S3A configuration for RustFS +spark.hadoop.fs.s3a.endpoint http://rustfs:9000 +spark.hadoop.fs.s3a.access.key rustfs +spark.hadoop.fs.s3a.secret.key rustfs123 spark.hadoop.fs.s3a.path.style.access true spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.connection.ssl.enabled false diff --git a/integration_tests/docker/trino/catalog/iceberg.properties b/integration_tests/docker/trino/catalog/iceberg.properties index c43f07e7f..d3f62f6b4 100644 --- a/integration_tests/docker/trino/catalog/iceberg.properties +++ b/integration_tests/docker/trino/catalog/iceberg.properties @@ -1,9 +1,9 @@ connector.name=iceberg hive.metastore.uri=thrift://hive-metastore:9083 -hive.s3.endpoint=http://minio:9000 +hive.s3.endpoint=http://rustfs:9000 hive.s3.path-style-access=true -hive.s3.aws-access-key=minio -hive.s3.aws-secret-key=minio123 +hive.s3.aws-access-key=rustfs +hive.s3.aws-secret-key=rustfs123 hive.metastore-cache-ttl=0s hive.metastore-refresh-interval=5s hive.metastore-timeout=60s diff --git a/integration_tests/tests/data_seeder.py b/integration_tests/tests/data_seeder.py index 232afb665..e4e1440ee 100644 --- a/integration_tests/tests/data_seeder.py +++ b/integration_tests/tests/data_seeder.py @@ -245,7 +245,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None] class SparkS3CsvSeeder: - """Seeder for Spark that uploads CSVs to MinIO (S3) and creates external tables. + """Seeder for Spark that uploads CSVs to RustFS (S3) and creates external tables. Bypasses ``dbt seed`` entirely — Spark reads the CSV natively via ``CREATE TABLE ... USING CSV``. This avoids the ``_fix_binding`` bug @@ -262,16 +262,16 @@ class SparkS3CsvSeeder: that empty cells are also read as NULL. The S3 CSV files are **not** deleted after the test — they live in - ephemeral MinIO storage that is destroyed with ``docker compose down``. + ephemeral RustFS storage that is destroyed with ``docker compose down``. The external table continues to reference the S3 path throughout the test lifecycle. """ - # MinIO connection defaults (matching docker-compose-spark.yml). - _MINIO_ENDPOINT = os.environ.get("MINIO_ENDPOINT", "http://127.0.0.1:9000") - _MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") # noqa: S105 - _MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") # noqa: S105 - _S3_BUCKET = os.environ.get("MINIO_BUCKET", "spark-seeds") + # RustFS connection defaults (matching docker-compose-spark.yml). + _RUSTFS_ENDPOINT = os.environ.get("RUSTFS_ENDPOINT", "http://127.0.0.1:9000") + _RUSTFS_ACCESS_KEY = os.environ.get("RUSTFS_ACCESS_KEY", "rustfs") # noqa: S105 + _RUSTFS_SECRET_KEY = os.environ.get("RUSTFS_SECRET_KEY", "rustfs123") # noqa: S105 + _S3_BUCKET = os.environ.get("RUSTFS_BUCKET", "spark-seeds") # Spark Thrift Server connection defaults. _THRIFT_HOST = os.environ.get("SPARK_THRIFT_HOST", "127.0.0.1") @@ -287,14 +287,14 @@ def __init__( self._seeds_dir_path = seeds_dir_path def _get_s3_client(self): # type: ignore[no-untyped-def] - """Return a boto3 S3 client configured for the local MinIO endpoint.""" + """Return a boto3 S3 client configured for the local RustFS endpoint.""" import boto3 return boto3.client( "s3", - endpoint_url=self._MINIO_ENDPOINT, - aws_access_key_id=self._MINIO_ACCESS_KEY, - aws_secret_access_key=self._MINIO_SECRET_KEY, + endpoint_url=self._RUSTFS_ENDPOINT, + aws_access_key_id=self._RUSTFS_ACCESS_KEY, + aws_secret_access_key=self._RUSTFS_SECRET_KEY, ) @contextmanager @@ -366,14 +366,14 @@ def _infer_spark_schema(self, data: List[dict]) -> str: @contextmanager def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None]: - """Upload CSV to MinIO and create a Spark external table. + """Upload CSV to RustFS and create a Spark external table. The CSV is also written locally so dbt discovers the seed node for ``{{ ref() }}`` resolution. The local CSV is cleaned up when the context manager exits to prevent dbt compilation errors (duplicate resource names). The S3 object is **not** deleted — the external table references it throughout the test, - and MinIO storage is ephemeral (destroyed with + and RustFS storage is ephemeral (destroyed with ``docker compose down``). """ if not data: @@ -384,7 +384,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None] fq_table = f"`{self._schema}`.`{table_name}`" try: - # Upload CSV to MinIO. + # Upload CSV to RustFS. s3 = self._get_s3_client() s3.upload_file(str(seed_path), self._S3_BUCKET, s3_key)