Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/test-warehouse.yml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ jobs:
run: |
docker compose -f docker-compose-dremio.yml up -d
# Wait for Dremio to be healthy (one-shot containers like
# minio-setup exit immediately, so --wait would fail).
# rustfs-setup exit immediately, so --wait would fail).
echo "Waiting for Dremio to become healthy..."
timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} dremio 2>/dev/null)" = "healthy" ]; do sleep 5; done'
echo "Dremio is healthy."
Expand Down Expand Up @@ -154,13 +154,13 @@ jobs:
run: |
docker compose -f docker-compose-spark.yml build
docker compose -f docker-compose-spark.yml up -d
echo "Waiting for MinIO setup to complete..."
echo "Waiting for RustFS setup to complete..."
timeout 60 bash -c '
until [ "$(docker inspect -f "{{.State.Status}}" spark-minio-setup 2>/dev/null)" = "exited" ]; do sleep 2; done
EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-minio-setup 2>/dev/null)
if [ "$EXIT_CODE" != "0" ]; then echo "MinIO setup failed with exit code $EXIT_CODE"; exit 1; fi
until [ "$(docker inspect -f "{{.State.Status}}" spark-rustfs-setup 2>/dev/null)" = "exited" ]; do sleep 2; done
EXIT_CODE=$(docker inspect -f "{{.State.ExitCode}}" spark-rustfs-setup 2>/dev/null)
if [ "$EXIT_CODE" != "0" ]; then echo "RustFS setup failed with exit code $EXIT_CODE"; exit 1; fi
'
echo "MinIO is ready."
echo "RustFS is ready."
echo "Waiting for Spark Thrift Server to become healthy..."
timeout 180 bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} spark-thrift 2>/dev/null)" = "healthy" ]; do sleep 5; done'
echo "Spark Thrift Server is healthy."
Expand Down
44 changes: 22 additions & 22 deletions integration_tests/docker-compose-dremio.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,39 +8,39 @@ services:
ports:
- 19120:19120

# Minio Storage Server
minio:
image: minio/minio:latest
# RustFS Storage Server
rustfs:
image: rustfs/rustfs:1.0.0
container_name: storage
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
- MINIO_DOMAIN=storage
- MINIO_REGION_NAME=us-east-1
- MINIO_REGION=us-east-1
- RUSTFS_ACCESS_KEY=admin
- RUSTFS_SECRET_KEY=password
networks:
- dremio-lakehouse
ports:
- 9001:9001
- 9000:9000
command: ["server", "/data", "--console-address", ":9001"]
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
volumes:
- minio_data:/data
- rustfs_data:/data

minio-setup:
image: minio/mc
container_name: minio-setup
rustfs-setup:
image: rustfs/rc:v0.1.36
container_name: rustfs-setup
depends_on:
- minio
rustfs:
condition: service_healthy
entrypoint: >
/bin/sh -c "
until (echo > /dev/tcp/minio/9000) >/dev/null 2>&1; do
echo 'Waiting for MinIO...';
sleep 2;
done;
mc alias set myminio http://minio:9000 admin password;
mc mb myminio/datalake;
mc ls myminio;
set -e;
rc alias set local http://rustfs:9000 admin password;
rc bucket create --ignore-existing local/datalake;
rc ls local;
"
networks:
- dremio-lakehouse
Expand Down Expand Up @@ -89,4 +89,4 @@ networks:

volumes:
dremio_data:
minio_data:
rustfs_data:
34 changes: 17 additions & 17 deletions integration_tests/docker-compose-spark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ services:
depends_on:
spark-hive-metastore:
condition: service_started
minio-setup:
rustfs-setup:
condition: service_completed_successfully
command: >
--class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
Expand All @@ -39,40 +39,40 @@ services:
- POSTGRES_PASSWORD=dbt
- POSTGRES_DB=metastore

minio:
image: minio/minio:latest
container_name: spark-minio
# S3-compatible storage for external seed tables.
rustfs:
image: rustfs/rustfs:1.0.0
container_name: spark-rustfs
ports:
- "9000:9000"
- "9001:9001"
environment:
- MINIO_ROOT_USER=minioadmin
- MINIO_ROOT_PASSWORD=minioadmin
command: ["server", "/data", "--console-address", ":9001"]
- RUSTFS_ACCESS_KEY=rustfs
- RUSTFS_SECRET_KEY=rustfs123
healthcheck:
test: ["CMD-SHELL", "mc ready local || exit 1"]
test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s
volumes:
- minio-data:/data
- rustfs-data:/data

minio-setup:
image: minio/mc
container_name: spark-minio-setup
rustfs-setup:
image: rustfs/rc:v0.1.36
container_name: spark-rustfs-setup
depends_on:
minio:
rustfs:
condition: service_healthy
entrypoint: >
/bin/sh -c "
set -e;
mc alias set myminio http://minio:9000 minioadmin minioadmin;
mc mb --ignore-existing myminio/spark-seeds;
echo 'MinIO bucket spark-seeds created.';
rc alias set local http://rustfs:9000 rustfs rustfs123;
rc bucket create --ignore-existing local/spark-seeds;
echo 'Bucket spark-seeds created.';
"

volumes:
spark-warehouse:
hive-metastore:
minio-data:
rustfs-data:
42 changes: 24 additions & 18 deletions integration_tests/docker-compose-trino.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ services:
HIVE_METASTORE_USER: hive
HIVE_METASTORE_PASSWORD: hive
HIVE_METASTORE_WAREHOUSE_DIR: s3://datalake/
S3_ENDPOINT: http://minio:9000
S3_ACCESS_KEY: minio
S3_SECRET_KEY: minio123
S3_ENDPOINT: http://rustfs:9000
S3_ACCESS_KEY: rustfs
S3_SECRET_KEY: rustfs123
S3_PATH_STYLE_ACCESS: "true"

# NOTE - don't remove all these empty vars, the hive metastore startup script actually verifies all of them exist...
Expand All @@ -51,26 +51,32 @@ services:
depends_on:
- metastore_db

minio:
hostname: minio
image: "minio/minio:RELEASE.2022-05-26T05-48-41Z"
container_name: minio
rustfs:
hostname: rustfs
image: "rustfs/rustfs:1.0.0"
container_name: rustfs
ports:
- "9000:9000"
- "9001:9001"
environment:
MINIO_ACCESS_KEY: minio
MINIO_SECRET_KEY: minio123
command: server /data --console-address ":9001"
RUSTFS_ACCESS_KEY: rustfs
RUSTFS_SECRET_KEY: rustfs123
healthcheck:
test: ["CMD-SHELL", "curl -sf http://localhost:9000/health || exit 1"]
interval: 5s
timeout: 5s
retries: 10
start_period: 5s

# This job will create the "datalake" bucket on Minio
mc-job:
image: "minio/mc:RELEASE.2022-05-09T04-08-26Z"
# This job will create the "datalake" bucket on RustFS
rustfs-setup:
image: "rustfs/rc:v0.1.36"
entrypoint: |
/bin/bash -c "
sleep 5;
/usr/bin/mc config --quiet host add myminio http://minio:9000 minio minio123;
/usr/bin/mc mb --quiet myminio/datalake
/bin/sh -c "
set -e;
rc alias set local http://rustfs:9000 rustfs rustfs123;
rc bucket create --ignore-existing local/datalake
"
depends_on:
- minio
rustfs:
condition: service_healthy
2 changes: 1 addition & 1 deletion integration_tests/docker/dremio/dremio-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@ echo "Obtained Dremio auth token"
curl -s -X PUT "http://dremio:9047/apiv2/source/S3Source" \
-H "Content-Type: application/json" \
-H "Authorization: _dremio$AUTH_TOKEN" \
--data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}"
--data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"rustfs:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}"

echo "S3 Source created in Dremio"
8 changes: 4 additions & 4 deletions integration_tests/docker/spark/spark-defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ spark.default.parallelism 2
spark.ui.enabled false
spark.sql.adaptive.enabled true

# S3A configuration for MinIO
spark.hadoop.fs.s3a.endpoint http://minio:9000
spark.hadoop.fs.s3a.access.key minioadmin
spark.hadoop.fs.s3a.secret.key minioadmin
# S3A configuration for RustFS
spark.hadoop.fs.s3a.endpoint http://rustfs:9000
spark.hadoop.fs.s3a.access.key rustfs
spark.hadoop.fs.s3a.secret.key rustfs123
spark.hadoop.fs.s3a.path.style.access true
spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem
spark.hadoop.fs.s3a.connection.ssl.enabled false
6 changes: 3 additions & 3 deletions integration_tests/docker/trino/catalog/iceberg.properties
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
connector.name=iceberg
hive.metastore.uri=thrift://hive-metastore:9083
hive.s3.endpoint=http://minio:9000
hive.s3.endpoint=http://rustfs:9000
hive.s3.path-style-access=true
hive.s3.aws-access-key=minio
hive.s3.aws-secret-key=minio123
hive.s3.aws-access-key=rustfs
hive.s3.aws-secret-key=rustfs123
hive.metastore-cache-ttl=0s
hive.metastore-refresh-interval=5s
hive.metastore-timeout=60s
28 changes: 14 additions & 14 deletions integration_tests/tests/data_seeder.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None]


class SparkS3CsvSeeder:
"""Seeder for Spark that uploads CSVs to MinIO (S3) and creates external tables.
"""Seeder for Spark that uploads CSVs to RustFS (S3) and creates external tables.

Bypasses ``dbt seed`` entirely — Spark reads the CSV natively via
``CREATE TABLE ... USING CSV``. This avoids the ``_fix_binding`` bug
Expand All @@ -262,16 +262,16 @@ class SparkS3CsvSeeder:
that empty cells are also read as NULL.

The S3 CSV files are **not** deleted after the test — they live in
ephemeral MinIO storage that is destroyed with ``docker compose down``.
ephemeral RustFS storage that is destroyed with ``docker compose down``.
The external table continues to reference the S3 path throughout the
test lifecycle.
"""

# MinIO connection defaults (matching docker-compose-spark.yml).
_MINIO_ENDPOINT = os.environ.get("MINIO_ENDPOINT", "http://127.0.0.1:9000")
_MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY", "minioadmin") # noqa: S105
_MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY", "minioadmin") # noqa: S105
_S3_BUCKET = os.environ.get("MINIO_BUCKET", "spark-seeds")
# RustFS connection defaults (matching docker-compose-spark.yml).
_RUSTFS_ENDPOINT = os.environ.get("RUSTFS_ENDPOINT", "http://127.0.0.1:9000")
_RUSTFS_ACCESS_KEY = os.environ.get("RUSTFS_ACCESS_KEY", "rustfs") # noqa: S105
_RUSTFS_SECRET_KEY = os.environ.get("RUSTFS_SECRET_KEY", "rustfs123") # noqa: S105
_S3_BUCKET = os.environ.get("RUSTFS_BUCKET", "spark-seeds")

# Spark Thrift Server connection defaults.
_THRIFT_HOST = os.environ.get("SPARK_THRIFT_HOST", "127.0.0.1")
Expand All @@ -287,14 +287,14 @@ def __init__(
self._seeds_dir_path = seeds_dir_path

def _get_s3_client(self): # type: ignore[no-untyped-def]
"""Return a boto3 S3 client configured for the local MinIO endpoint."""
"""Return a boto3 S3 client configured for the local RustFS endpoint."""
import boto3

return boto3.client(
"s3",
endpoint_url=self._MINIO_ENDPOINT,
aws_access_key_id=self._MINIO_ACCESS_KEY,
aws_secret_access_key=self._MINIO_SECRET_KEY,
endpoint_url=self._RUSTFS_ENDPOINT,
aws_access_key_id=self._RUSTFS_ACCESS_KEY,
aws_secret_access_key=self._RUSTFS_SECRET_KEY,
)

@contextmanager
Expand Down Expand Up @@ -366,14 +366,14 @@ def _infer_spark_schema(self, data: List[dict]) -> str:

@contextmanager
def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None]:
"""Upload CSV to MinIO and create a Spark external table.
"""Upload CSV to RustFS and create a Spark external table.

The CSV is also written locally so dbt discovers the seed node
for ``{{ ref() }}`` resolution. The local CSV is cleaned up
when the context manager exits to prevent dbt compilation
errors (duplicate resource names). The S3 object is **not**
deleted — the external table references it throughout the test,
and MinIO storage is ephemeral (destroyed with
and RustFS storage is ephemeral (destroyed with
``docker compose down``).
"""
if not data:
Expand All @@ -384,7 +384,7 @@ def seed(self, data: List[dict], table_name: str) -> Generator[None, None, None]
fq_table = f"`{self._schema}`.`{table_name}`"

try:
# Upload CSV to MinIO.
# Upload CSV to RustFS.
s3 = self._get_s3_client()
s3.upload_file(str(seed_path), self._S3_BUCKET, s3_key)

Expand Down
Loading