diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index c167516bc..ba304cea7 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -126,7 +126,7 @@ jobs: run: | docker compose -f docker-compose-dremio.yml up -d # Wait for Dremio to be healthy (one-shot containers like - # minio-setup exit immediately, so --wait would fail). + # rustfs-setup exit immediately, so --wait would fail). echo "Waiting for Dremio to become healthy..." timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} dremio 2>/dev/null)" = "healthy" ]; do sleep 5; done' echo "Dremio is healthy." @@ -154,13 +154,13 @@ jobs: run: | docker compose -f docker-compose-spark.yml build docker compose -f docker-compose-spark.yml up -d - echo "Waiting for MinIO setup to complete..." + echo "Waiting for RustFS setup to complete..." timeout 60 bash -c ' - until [ "$(docker inspect -f "{{.State.Status}}" spark-minio-setup 2>/dev/null)" = "exited" ]; do sleep 2; done - EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-minio-setup 2>/dev/null) - if [ "$EXIT_CODE" != "0" ]; then echo "MinIO setup failed with exit code $EXIT_CODE"; exit 1; fi + until [ "$(docker inspect -f "{{.State.Status}}" spark-rustfs-setup 2>/dev/null)" = "exited" ]; do sleep 2; done + EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-rustfs-setup 2>/dev/null) + if [ "$EXIT_CODE" != "0" ]; then echo "RustFS setup failed with exit code $EXIT_CODE"; exit 1; fi ' - echo "MinIO is ready." + echo "RustFS is ready." echo "Waiting for Spark Thrift Server to become healthy..." timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} spark-thrift 2>/dev/null)" = "healthy" ]; do sleep 5; done' echo "Spark Thrift Server is healthy." diff --git a/integration_tests/docker-compose-dremio.yml b/integration_tests/docker-compose-dremio.yml index 1799937c9..7cb880db7 100644 --- a/integration_tests/docker-compose-dremio.yml +++ b/integration_tests/docker-compose-dremio.yml @@ -8,39 +8,39 @@ services: ports: - 19120:19120 - # Minio Storage Server - minio: - image: minio/minio:latest + # RustFS Storage Server + rustfs: + image: rustfs/rustfs:1.0.0 container_name: storage environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password - - MINIO_DOMAIN=storage - - MINIO_REGION_NAME=us-east-1 - - MINIO_REGION=us-east-1 + - RUSTFS_ACCESS_KEY=admin + - RUSTFS_SECRET_KEY=password networks: - dremio-lakehouse ports: - 9001:9001 - 9000:9000 - command: ["server", "/data", "--console-address", ":9001"] + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s volumes: - - minio_data:/data + - rustfs_data:/data - minio-setup: - image: minio/mc - container_name: minio-setup + rustfs-setup: + image: rustfs/rc:v0.1.36 + container_name: rustfs-setup depends_on: - - minio + rustfs: + condition: service_healthy entrypoint: > /bin/sh -c " - until (echo > /dev/tcp/minio/9000) >/dev/null 2>&1; do - echo 'Waiting for MinIO...'; - sleep 2; - done; - mc alias set myminio http://minio:9000 admin password; - mc mb myminio/datalake; - mc ls myminio; + set -e; + rc alias set local http://rustfs:9000 admin password; + rc bucket create --ignore-existing local/datalake; + rc ls local; " networks: - dremio-lakehouse @@ -89,4 +89,4 @@ networks: volumes: dremio_data: - minio_data: + rustfs_data: diff --git a/integration_tests/docker-compose-spark.yml b/integration_tests/docker-compose-spark.yml index 50da8a615..1d12cbf10 100644 --- a/integration_tests/docker-compose-spark.yml +++ b/integration_tests/docker-compose-spark.yml @@ -12,7 +12,7 @@ services: depends_on: spark-hive-metastore: condition: service_started - minio-setup: + rustfs-setup: condition: service_completed_successfully command: > --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 @@ -39,40 +39,40 @@ services: - POSTGRES_PASSWORD=dbt - POSTGRES_DB=metastore - minio: - image: minio/minio:latest - container_name: spark-minio + # S3-compatible storage for external seed tables. + rustfs: + image: rustfs/rustfs:1.0.0 + container_name: spark-rustfs ports: - "9000:9000" - "9001:9001" environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - command: ["server", "/data", "--console-address", ":9001"] + - RUSTFS_ACCESS_KEY=rustfs + - RUSTFS_SECRET_KEY=rustfs123 healthcheck: - test: ["CMD-SHELL", "mc ready local || exit 1"] + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] interval: 5s timeout: 5s retries: 10 start_period: 5s volumes: - - minio-data:/data + - rustfs-data:/data - minio-setup: - image: minio/mc - container_name: spark-minio-setup + rustfs-setup: + image: rustfs/rc:v0.1.36 + container_name: spark-rustfs-setup depends_on: - minio: + rustfs: condition: service_healthy entrypoint: > /bin/sh -c " set -e; - mc alias set myminio http://minio:9000 minioadmin minioadmin; - mc mb --ignore-existing myminio/spark-seeds; - echo 'MinIO bucket spark-seeds created.'; + rc alias set local http://rustfs:9000 rustfs rustfs123; + rc bucket create --ignore-existing local/spark-seeds; + echo 'Bucket spark-seeds created.'; " volumes: spark-warehouse: hive-metastore: - minio-data: + rustfs-data: diff --git a/integration_tests/docker-compose-trino.yml b/integration_tests/docker-compose-trino.yml index 51cea59a1..187ee7947 100644 --- a/integration_tests/docker-compose-trino.yml +++ b/integration_tests/docker-compose-trino.yml @@ -27,9 +27,9 @@ services: HIVE_METASTORE_USER: hive HIVE_METASTORE_PASSWORD: hive HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/ - S3_ENDPOINT: http://minio:9000 - S3_ACCESS_KEY: minio - S3_SECRET_KEY: minio123 + S3_ENDPOINT: http://rustfs:9000 + S3_ACCESS_KEY: rustfs + S3_SECRET_KEY: rustfs123 S3_PATH_STYLE_ACCESS: "true" # NOTE - don't remove all these empty vars, the hive metastore startup script actually verifies all of them exist... @@ -51,26 +51,32 @@ services: depends_on: - metastore_db - minio: - hostname: minio - image: "minio/minio:RELEASE.2022-05-26T05-48-41Z" - container_name: minio + rustfs: + hostname: rustfs + image: "rustfs/rustfs:1.0.0" + container_name: rustfs ports: - "9000:9000" - "9001:9001" environment: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 - command: server /data --console-address ":9001" + RUSTFS_ACCESS_KEY: rustfs + RUSTFS_SECRET_KEY: rustfs123 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 5s - # This job will create the "datalake" bucket on Minio - mc-job: - image: "minio/mc:RELEASE.2022-05-09T04-08-26Z" + # This job will create the "datalake" bucket on RustFS + rustfs-setup: + image: "rustfs/rc:v0.1.36" entrypoint: | - /bin/bash -c " - sleep 5; - /usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123; - /usr/bin/mc mb --quiet myminio/datalake + /bin/sh -c " + set -e; + rc alias set local http://rustfs:9000 rustfs rustfs123; + rc bucket create --ignore-existing local/datalake " depends_on: - - minio + rustfs: + condition: service_healthy diff --git a/integration_tests/docker/dremio/dremio-setup.sh b/integration_tests/docker/dremio/dremio-setup.sh index 7dec29605..dd3e48e96 100644 --- a/integration_tests/docker/dremio/dremio-setup.sh +++ b/integration_tests/docker/dremio/dremio-setup.sh @@ -28,6 +28,6 @@ echo "Obtained Dremio auth token" curl -s -X PUT "http://dremio:9047/apiv2/source/S3Source" \ -H "Content-Type: application/json" \ -H "Authorization: _dremio$AUTH_TOKEN" \ - --data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" + --data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"rustfs:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" echo "S3 Source created in Dremio" \ No newline at end of file diff --git a/integration_tests/docker/spark/spark-defaults.conf b/integration_tests/docker/spark/spark-defaults.conf index f7c143f3b..9a57c5d64 100644 --- a/integration_tests/docker/spark/spark-defaults.conf +++ b/integration_tests/docker/spark/spark-defaults.conf @@ -13,10 +13,10 @@ spark.default.parallelism 2 spark.ui.enabled false spark.sql.adaptive.enabled true -# S3A configuration for MinIO -spark.hadoop.fs.s3a.endpoint http://minio:9000 -spark.hadoop.fs.s3a.access.key minioadmin -spark.hadoop.fs.s3a.secret.key minioadmin +# S3A configuration for RustFS +spark.hadoop.fs.s3a.endpoint http://rustfs:9000 +spark.hadoop.fs.s3a.access.key rustfs +spark.hadoop.fs.s3a.secret.key rustfs123 spark.hadoop.fs.s3a.path.style.access true spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem spark.hadoop.fs.s3a.connection.ssl.enabled false diff --git a/integration_tests/docker/trino/catalog/iceberg.properties b/integration_tests/docker/trino/catalog/iceberg.properties index c43f07e7f..d3f62f6b4 100644 --- a/integration_tests/docker/trino/catalog/iceberg.properties +++ b/integration_tests/docker/trino/catalog/iceberg.properties @@ -1,9 +1,9 @@ connector.name=iceberg hive.metastore.uri=thrift://hive-metastore:9083 -hive.s3.endpoint=http://minio:9000 +hive.s3.endpoint=http://rustfs:9000 hive.s3.path-style-access=true -hive.s3.aws-access-key=minio -hive.s3.aws-secret-key=minio123 +hive.s3.aws-access-key=rustfs +hive.s3.aws-secret-key=rustfs123 hive.metastore-cache-ttl=0s hive.metastore-refresh-interval=5s hive.metastore-timeout=60s diff --git a/integration_tests/tests/data_seeder.py b/integration_tests/tests/data_seeder.py index 232afb665..e4e1440ee 100644 --- a/integration_tests/tests/data_seeder.py +++ b/integration_tests/tests/data_seeder.py @@ -245,7 +245,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None] class SparkS3CsvSeeder: - """Seeder for Spark that uploads CSVs to MinIO (S3) and creates external tables. + """Seeder for Spark that uploads CSVs to RustFS (S3) and creates external tables. Bypasses ``dbt seed`` entirely — Spark reads the CSV natively via ``CREATE TABLE ... USING CSV``. This avoids the ``_fix_binding`` bug @@ -262,16 +262,16 @@ class SparkS3CsvSeeder: that empty cells are also read as NULL. The S3 CSV files are **not** deleted after the test — they live in - ephemeral MinIO storage that is destroyed with ``docker compose down``. + ephemeral RustFS storage that is destroyed with ``docker compose down``. The external table continues to reference the S3 path throughout the test lifecycle. """ - # MinIO connection defaults (matching docker-compose-spark.yml). - _MINIO_ENDPOINT = os.environ.get("MINIO_ENDPOINT", "http://127.0.0.1:9000") - _MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") # noqa: S105 - _MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") # noqa: S105 - _S3_BUCKET = os.environ.get("MINIO_BUCKET", "spark-seeds") + # RustFS connection defaults (matching docker-compose-spark.yml). + _RUSTFS_ENDPOINT = os.environ.get("RUSTFS_ENDPOINT", "http://127.0.0.1:9000") + _RUSTFS_ACCESS_KEY = os.environ.get("RUSTFS_ACCESS_KEY", "rustfs") # noqa: S105 + _RUSTFS_SECRET_KEY = os.environ.get("RUSTFS_SECRET_KEY", "rustfs123") # noqa: S105 + _S3_BUCKET = os.environ.get("RUSTFS_BUCKET", "spark-seeds") # Spark Thrift Server connection defaults. _THRIFT_HOST = os.environ.get("SPARK_THRIFT_HOST", "127.0.0.1") @@ -287,14 +287,14 @@ def __init__( self._seeds_dir_path = seeds_dir_path def _get_s3_client(self): # type: ignore[no-untyped-def] - """Return a boto3 S3 client configured for the local MinIO endpoint.""" + """Return a boto3 S3 client configured for the local RustFS endpoint.""" import boto3 return boto3.client( "s3", - endpoint_url=self._MINIO_ENDPOINT, - aws_access_key_id=self._MINIO_ACCESS_KEY, - aws_secret_access_key=self._MINIO_SECRET_KEY, + endpoint_url=self._RUSTFS_ENDPOINT, + aws_access_key_id=self._RUSTFS_ACCESS_KEY, + aws_secret_access_key=self._RUSTFS_SECRET_KEY, ) @contextmanager @@ -366,14 +366,14 @@ def _infer_spark_schema(self, data: List[dict]) -> str: @contextmanager def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None]: - """Upload CSV to MinIO and create a Spark external table. + """Upload CSV to RustFS and create a Spark external table. The CSV is also written locally so dbt discovers the seed node for ``{{ ref() }}`` resolution. The local CSV is cleaned up when the context manager exits to prevent dbt compilation errors (duplicate resource names). The S3 object is **not** deleted — the external table references it throughout the test, - and MinIO storage is ephemeral (destroyed with + and RustFS storage is ephemeral (destroyed with ``docker compose down``). """ if not data: @@ -384,7 +384,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None] fq_table = f"`{self._schema}`.`{table_name}`" try: - # Upload CSV to MinIO. + # Upload CSV to RustFS. s3 = self._get_s3_client() s3.upload_file(str(seed_path), self._S3_BUCKET, s3_key)