diff --git a/.github/scripts/generate_ci_config.sh b/.github/scripts/generate_ci_config.sh index fa1fc7f..d31a4dd 100755 --- a/.github/scripts/generate_ci_config.sh +++ b/.github/scripts/generate_ci_config.sh @@ -3,7 +3,7 @@ set -xeuo pipefail if [[ $# -lt 2 ]]; then cat >&2 <<'USAGE' -Usage: generate_ci_config.sh [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] +Usage: generate_ci_config.sh [--minio] [--aws-s3] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst] USAGE exit 1 fi @@ -12,6 +12,7 @@ OUTPUT=$1 BASE_CONFIG=$2 shift 2 MINIO=false +AWS_S3=false JUPYTERHUB=false WEKO=false FLOWABLE=false @@ -23,6 +24,9 @@ for arg in "$@"; do --minio) MINIO=true ;; + --aws-s3) + AWS_S3=true + ;; --jupyterhub) JUPYTERHUB=true ;; @@ -46,6 +50,11 @@ for arg in "$@"; do done +if [[ "${MINIO}" == "true" && "${AWS_S3}" == "true" ]]; then + echo "--minio and --aws-s3 both write storages_s3 and cannot be used together" >&2 + exit 1 +fi + cp "${BASE_CONFIG}" "${OUTPUT}" if [[ "${MINIO}" == "true" ]]; then @@ -74,13 +83,66 @@ s3compat_test_bucket_name_2: '${S3COMPAT_BUCKET_NAME_2}' s3compat_type_name_1: '${S3COMPAT_SERVICE_NAME}' s3compat_type_name_2: '${S3COMPAT_SERVICE_NAME}' EOF -else +elif [[ "${AWS_S3}" != "true" ]]; then cat >> "${OUTPUT}" <<'EOF' storages_s3: [] EOF fi +if [[ "${AWS_S3}" == "true" ]]; then + required_aws_s3_vars=( + AWS_S3_ACCESS_KEY_1 + AWS_S3_SECRET_KEY_1 + AWS_S3_ACCESS_KEY_2 + AWS_S3_SECRET_KEY_2 + AWS_S3_LEGACY_REGION + AWS_S3_LEGACY_BUCKET_NAME + AWS_S3_V4_REGION + AWS_S3_V4_BUCKET_NAME + ) + + missing_aws_s3_vars=() + for var_name in "${required_aws_s3_vars[@]}"; do + if [[ -z "${!var_name:-}" ]]; then + missing_aws_s3_vars+=("${var_name}") + fi + done + + if [[ ${#missing_aws_s3_vars[@]} -gt 0 ]]; then + echo "AWS S3 test credentials are not set: ${missing_aws_s3_vars[*]}" >&2 + exit 1 + fi + + if [[ "${AWS_S3_ACCESS_KEY_1}" == "${AWS_S3_ACCESS_KEY_2}" ]]; then + echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different" >&2 + exit 1 + fi + + if [[ "${AWS_S3_SECRET_KEY_1}" == "${AWS_S3_SECRET_KEY_2}" ]]; then + echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different" >&2 + exit 1 + fi + + cat >> "${OUTPUT}" <&2 diff --git a/.github/scripts/mask_sensitive_artifacts.sh b/.github/scripts/mask_sensitive_artifacts.sh new file mode 100644 index 0000000..7596f91 --- /dev/null +++ b/.github/scripts/mask_sensitive_artifacts.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [ ...]" >&2 + exit 1 +fi + +base_dir=$(pwd) + +mask_value() { + local value="$1" + local length=${#value} + if (( length <= 8 )); then + printf '****' + return + fi + + local prefix="${value:0:4}" + local suffix="${value: -4}" + local mask_length=$((length - 8)) + local stars + stars=$(printf '%*s' "$mask_length" '' | tr ' ' '*') + printf '%s%s%s' "$prefix" "$stars" "$suffix" +} + +replace_in_file() { + local file="$1" + local value="$2" + local masked="$3" + + VALUE="$value" MASKED="$masked" perl -0pi -e 's/\Q$ENV{VALUE}\E/$ENV{MASKED}/g' "$file" +} + +replace_in_tree() { + local target_dir="$1" + local value="$2" + local masked="$3" + + find "$target_dir" -type f \ + \( -name '*.ipynb' -o -name '*.log' -o -name '*.json' -o -name '*.har' -o -name '*.html' -o -name '*.js' -o -name '*.txt' -o -name '*.yaml' -o -name '*.yml' \) \ + -print0 | + while IFS= read -r -d '' file; do + replace_in_file "$file" "$value" "$masked" + done + + find "$target_dir" -type f -name 'har.zip' -print0 | + while IFS= read -r -d '' zip_file; do + local tmp_dir + local output_zip + tmp_dir=$(mktemp -d) + case "$zip_file" in + /*) + output_zip="$zip_file" + ;; + *) + output_zip="$base_dir/$zip_file" + ;; + esac + unzip -q "$zip_file" -d "$tmp_dir" + find "$tmp_dir" -type f -print0 | + while IFS= read -r -d '' file; do + replace_in_file "$file" "$value" "$masked" + done + (cd "$tmp_dir" && zip -qr "$output_zip" .) + rm -rf "$tmp_dir" + done +} + +while IFS='=' read -r name value; do + case "$name" in + *ACCESS_KEY*|*SECRET_KEY*|*PASSWORD*|*TOKEN*) + ;; + *) + continue + ;; + esac + + if [[ -z "$value" || ${#value} -lt 8 ]]; then + continue + fi + + masked=$(mask_value "$value") + echo "::add-mask::$value" + + for target_dir in "$@"; do + if [[ -d "$target_dir" ]]; then + replace_in_tree "$target_dir" "$value" "$masked" + fi + done +done < <(env) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 5dfa497..7f7031c 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -162,6 +162,19 @@ jobs: weko_enabled: false jupyterhub_enabled: false flowable_enabled: false + - name: user-aws-s3 + display_name: "User Tests (AWS S3 SigV4 Regions)" + include_admin: false + skip_admin: true + skip_metadata: true + skip_default_storage: true + skip_login: true + skip_130mb_upload: true + minio_enabled: false + aws_s3_enabled: true + weko_enabled: false + jupyterhub_enabled: false + flowable_enabled: false - name: admin-minio display_name: "Admin Tests (MinIO)" include_admin: true @@ -215,6 +228,7 @@ jobs: name: E2E ${{ matrix.test-group.display_name }} env: MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }} + AWS_S3_ENABLED: ${{ matrix.test-group.aws_s3_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }} S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }} WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }} @@ -226,6 +240,14 @@ jobs: MINIO_ENDPOINT: http://minio:9000 MINIO_REGION: us-east-1 MINIO_SERVICE_NAME: MinIO (CI) + AWS_S3_ACCESS_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_1 || '' }} + AWS_S3_SECRET_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_1 || '' }} + AWS_S3_ACCESS_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_2 || '' }} + AWS_S3_SECRET_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_2 || '' }} + AWS_S3_LEGACY_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_REGION || '' }} + AWS_S3_LEGACY_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_BUCKET_NAME || '' }} + AWS_S3_V4_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_REGION || '' }} + AWS_S3_V4_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_BUCKET_NAME || '' }} steps: - name: Checkout test repository @@ -1022,6 +1044,28 @@ jobs: docker-compose exec -T web python3 -m scripts.register_erad_metadata /tmp/erad_sample.csv echo "e-Rad data registered successfully" + - name: Validate and mask sensitive test credentials + working-directory: e2e-tests + run: | + set -euo pipefail + + bash .github/scripts/mask_sensitive_artifacts.sh /tmp/nonexistent-mask-target + + if [ "${AWS_S3_ENABLED}" = "true" ]; then + if [ -z "${AWS_S3_ACCESS_KEY_1}" ] || [ -z "${AWS_S3_SECRET_KEY_1}" ] || [ -z "${AWS_S3_ACCESS_KEY_2}" ] || [ -z "${AWS_S3_SECRET_KEY_2}" ]; then + echo "AWS S3 test credentials are not fully configured. Set AWS_S3_ACCESS_KEY_1, AWS_S3_SECRET_KEY_1, AWS_S3_ACCESS_KEY_2, and AWS_S3_SECRET_KEY_2 secrets." + exit 1 + fi + if [ "${AWS_S3_ACCESS_KEY_1}" = "${AWS_S3_ACCESS_KEY_2}" ]; then + echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different." + exit 1 + fi + if [ "${AWS_S3_SECRET_KEY_1}" = "${AWS_S3_SECRET_KEY_2}" ]; then + echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different." + exit 1 + fi + fi + - name: Prepare test configuration working-directory: e2e-tests run: | @@ -1078,6 +1122,9 @@ jobs: if [ "${MINIO_ENABLED}" = "true" ]; then args+=(--minio) fi + if [ "${AWS_S3_ENABLED}" = "true" ]; then + args+=(--aws-s3) + fi if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then args+=(--jupyterhub) fi @@ -1152,6 +1199,12 @@ jobs: echo "ticket=$TICKET" >> $GITHUB_OUTPUT echo "Extracted ticket: $TICKET" + - name: Mask sensitive values in test artifacts + if: always() + working-directory: e2e-tests + run: | + bash .github/scripts/mask_sensitive_artifacts.sh result result-failed + - name: Generate Excel summary if: always() working-directory: e2e-tests