feat(ci): AWS S3 SigV4リージョンのE2Eテストを追加

.github/scripts/generate_ci_config.sh

@@ -3,7 +3,7 @@ set -xeuo pipefail
 
 if [[ $# -lt 2 ]]; then
   cat >&2 <<'USAGE'
-Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst]
+Usage: generate_ci_config.sh <output_path> <base_config_yaml> [--minio] [--aws-s3] [--jupyterhub] [--weko] [--flowable] [--s3compatsigv4] [--s3compatsigv4-inst]
 USAGE
   exit 1
 fi
@@ -12,6 +12,7 @@ OUTPUT=$1
 BASE_CONFIG=$2
 shift 2
 MINIO=false
+AWS_S3=false
 JUPYTERHUB=false
 WEKO=false
 FLOWABLE=false
@@ -23,6 +24,9 @@ for arg in "$@"; do
     --minio)
       MINIO=true
       ;;
+    --aws-s3)
+      AWS_S3=true
+      ;;
     --jupyterhub)
       JUPYTERHUB=true
       ;;
@@ -46,6 +50,11 @@ for arg in "$@"; do
 
 done
 
+if [[ "${MINIO}" == "true" && "${AWS_S3}" == "true" ]]; then
+  echo "--minio and --aws-s3 both write storages_s3 and cannot be used together" >&2
+  exit 1
+fi
+
 cp "${BASE_CONFIG}" "${OUTPUT}"
 
 if [[ "${MINIO}" == "true" ]]; then
@@ -74,13 +83,66 @@ s3compat_test_bucket_name_2: '${S3COMPAT_BUCKET_NAME_2}'
 s3compat_type_name_1: '${S3COMPAT_SERVICE_NAME}'
 s3compat_type_name_2: '${S3COMPAT_SERVICE_NAME}'
 EOF
-else
+elif [[ "${AWS_S3}" != "true" ]]; then
   cat >> "${OUTPUT}" <<'EOF'
 
 storages_s3: []
 EOF
 fi
 
+if [[ "${AWS_S3}" == "true" ]]; then
+  required_aws_s3_vars=(
+    AWS_S3_ACCESS_KEY_1
+    AWS_S3_SECRET_KEY_1
+    AWS_S3_ACCESS_KEY_2
+    AWS_S3_SECRET_KEY_2
+    AWS_S3_LEGACY_REGION
+    AWS_S3_LEGACY_BUCKET_NAME
+    AWS_S3_V4_REGION
+    AWS_S3_V4_BUCKET_NAME
+  )
+
+  missing_aws_s3_vars=()
+  for var_name in "${required_aws_s3_vars[@]}"; do
+    if [[ -z "${!var_name:-}" ]]; then
+      missing_aws_s3_vars+=("${var_name}")
+    fi
+  done
+
+  if [[ ${#missing_aws_s3_vars[@]} -gt 0 ]]; then
+    echo "AWS S3 test credentials are not set: ${missing_aws_s3_vars[*]}" >&2
+    exit 1
+  fi
+
+  if [[ "${AWS_S3_ACCESS_KEY_1}" == "${AWS_S3_ACCESS_KEY_2}" ]]; then
+    echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different" >&2
+    exit 1
+  fi
+
+  if [[ "${AWS_S3_SECRET_KEY_1}" == "${AWS_S3_SECRET_KEY_2}" ]]; then
+    echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different" >&2
+    exit 1
+  fi
+
+  cat >> "${OUTPUT}" <<EOF
+
+storages_s3:
+  - id: 's3'
+    name: 'Amazon S3'
+    skip_too_many_files_check: true
+
+s3_access_key_1: '${AWS_S3_ACCESS_KEY_1}'
+s3_secret_access_key_1: '${AWS_S3_SECRET_KEY_1}'
+s3_default_region_1: '${AWS_S3_LEGACY_REGION}'
+s3_test_bucket_name_1: '${AWS_S3_LEGACY_BUCKET_NAME}'
+
+s3_access_key_2: '${AWS_S3_ACCESS_KEY_2}'
+s3_secret_access_key_2: '${AWS_S3_SECRET_KEY_2}'
+s3_default_region_2: '${AWS_S3_V4_REGION}'
+s3_test_bucket_name_2: '${AWS_S3_V4_BUCKET_NAME}'
+EOF
+fi
+
 if [[ "${JUPYTERHUB}" == "true" ]]; then
   if [[ -z "${TLJH_URL:-}" || -z "${TLJH_USERNAME:-}" || -z "${TLJH_PASSWORD:-}" ]]; then
     echo "TLJH connection information is not set" >&2

.github/scripts/mask_sensitive_artifacts.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <artifact-dir> [<artifact-dir> ...]" >&2
+  exit 1
+fi
+
+base_dir=$(pwd)
+
+mask_value() {
+  local value="$1"
+  local length=${#value}
+  if (( length <= 8 )); then
+    printf '****'
+    return
+  fi
+
+  local prefix="${value:0:4}"
+  local suffix="${value: -4}"
+  local mask_length=$((length - 8))
+  local stars
+  stars=$(printf '%*s' "$mask_length" '' | tr ' ' '*')
+  printf '%s%s%s' "$prefix" "$stars" "$suffix"
+}
+
+replace_in_file() {
+  local file="$1"
+  local value="$2"
+  local masked="$3"
+
+  VALUE="$value" MASKED="$masked" perl -0pi -e 's/\Q$ENV{VALUE}\E/$ENV{MASKED}/g' "$file"
+}
+
+replace_in_tree() {
+  local target_dir="$1"
+  local value="$2"
+  local masked="$3"
+
+  find "$target_dir" -type f \
+    \( -name '*.ipynb' -o -name '*.log' -o -name '*.json' -o -name '*.har' -o -name '*.html' -o -name '*.js' -o -name '*.txt' -o -name '*.yaml' -o -name '*.yml' \) \
+    -print0 |
+    while IFS= read -r -d '' file; do
+      replace_in_file "$file" "$value" "$masked"
+    done
+
+  find "$target_dir" -type f -name 'har.zip' -print0 |
+    while IFS= read -r -d '' zip_file; do
+      local tmp_dir
+      local output_zip
+      tmp_dir=$(mktemp -d)
+      case "$zip_file" in
+        /*)
+          output_zip="$zip_file"
+          ;;
+        *)
+          output_zip="$base_dir/$zip_file"
+          ;;
+      esac
+      unzip -q "$zip_file" -d "$tmp_dir"
+      find "$tmp_dir" -type f -print0 |
+        while IFS= read -r -d '' file; do
+          replace_in_file "$file" "$value" "$masked"
+        done
+      (cd "$tmp_dir" && zip -qr "$output_zip" .)
+      rm -rf "$tmp_dir"
+    done
+}
+
+while IFS='=' read -r name value; do
+  case "$name" in
+    *ACCESS_KEY*|*SECRET_KEY*|*PASSWORD*|*TOKEN*)
+      ;;
+    *)
+      continue
+      ;;
+  esac
+
+  if [[ -z "$value" || ${#value} -lt 8 ]]; then
+    continue
+  fi
+
+  masked=$(mask_value "$value")
+  echo "::add-mask::$value"
+
+  for target_dir in "$@"; do
+    if [[ -d "$target_dir" ]]; then
+      replace_in_tree "$target_dir" "$value" "$masked"
+    fi
+  done
+done < <(env)

.github/workflows/e2e-test.yml

@@ -162,6 +162,19 @@ jobs:
             weko_enabled: false
             jupyterhub_enabled: false
             flowable_enabled: false
+          - name: user-aws-s3
+            display_name: "User Tests (AWS S3 SigV4 Regions)"
+            include_admin: false
+            skip_admin: true
+            skip_metadata: true
+            skip_default_storage: true
+            skip_login: true
+            skip_130mb_upload: true
+            minio_enabled: false
+            aws_s3_enabled: true
+            weko_enabled: false
+            jupyterhub_enabled: false
+            flowable_enabled: false
           - name: admin-minio
             display_name: "Admin Tests (MinIO)"
             include_admin: true
@@ -215,6 +228,7 @@ jobs:
     name: E2E ${{ matrix.test-group.display_name }}
     env:
       MINIO_ENABLED: ${{ matrix.test-group.minio_enabled == true && 'true' || 'false' }}
+      AWS_S3_ENABLED: ${{ matrix.test-group.aws_s3_enabled == true && 'true' || 'false' }}
       S3COMPATSIGV4_ENABLED: ${{ matrix.test-group.s3compatsigv4_enabled == true && 'true' || 'false' }}
       S3COMPATSIGV4_INST_ENABLED: ${{ matrix.test-group.s3compatsigv4_institutional_storage == true && 'true' || 'false' }}
       WEKO_ENABLED: ${{ matrix.test-group.weko_enabled == true && 'true' || 'false' }}
@@ -226,6 +240,14 @@ jobs:
       MINIO_ENDPOINT: http://minio:9000
       MINIO_REGION: us-east-1
       MINIO_SERVICE_NAME: MinIO (CI)
+      AWS_S3_ACCESS_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_1 || '' }}
+      AWS_S3_SECRET_KEY_1: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_1 || '' }}
+      AWS_S3_ACCESS_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_ACCESS_KEY_2 || '' }}
+      AWS_S3_SECRET_KEY_2: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_SECRET_KEY_2 || '' }}
+      AWS_S3_LEGACY_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_REGION || '' }}
+      AWS_S3_LEGACY_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_LEGACY_BUCKET_NAME || '' }}
+      AWS_S3_V4_REGION: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_REGION || '' }}
+      AWS_S3_V4_BUCKET_NAME: ${{ matrix.test-group.aws_s3_enabled == true && secrets.AWS_S3_V4_BUCKET_NAME || '' }}
 
     steps:
     - name: Checkout test repository
@@ -1022,6 +1044,28 @@ jobs:
         docker-compose exec -T web python3 -m scripts.register_erad_metadata /tmp/erad_sample.csv
         echo "e-Rad data registered successfully"
 
+    - name: Validate and mask sensitive test credentials
+      working-directory: e2e-tests
+      run: |
+        set -euo pipefail
+
+        bash .github/scripts/mask_sensitive_artifacts.sh /tmp/nonexistent-mask-target
+
+        if [ "${AWS_S3_ENABLED}" = "true" ]; then
+          if [ -z "${AWS_S3_ACCESS_KEY_1}" ] || [ -z "${AWS_S3_SECRET_KEY_1}" ] || [ -z "${AWS_S3_ACCESS_KEY_2}" ] || [ -z "${AWS_S3_SECRET_KEY_2}" ]; then
+            echo "AWS S3 test credentials are not fully configured. Set AWS_S3_ACCESS_KEY_1, AWS_S3_SECRET_KEY_1, AWS_S3_ACCESS_KEY_2, and AWS_S3_SECRET_KEY_2 secrets."
+            exit 1
+          fi
+          if [ "${AWS_S3_ACCESS_KEY_1}" = "${AWS_S3_ACCESS_KEY_2}" ]; then
+            echo "AWS_S3_ACCESS_KEY_1 and AWS_S3_ACCESS_KEY_2 must be different."
+            exit 1
+          fi
+          if [ "${AWS_S3_SECRET_KEY_1}" = "${AWS_S3_SECRET_KEY_2}" ]; then
+            echo "AWS_S3_SECRET_KEY_1 and AWS_S3_SECRET_KEY_2 must be different."
+            exit 1
+          fi
+        fi
+
     - name: Prepare test configuration
       working-directory: e2e-tests
       run: |
@@ -1078,6 +1122,9 @@ jobs:
         if [ "${MINIO_ENABLED}" = "true" ]; then
           args+=(--minio)
         fi
+        if [ "${AWS_S3_ENABLED}" = "true" ]; then
+          args+=(--aws-s3)
+        fi
         if [ "${JUPYTERHUB_ENABLED}" = "true" ]; then
           args+=(--jupyterhub)
         fi
@@ -1152,6 +1199,12 @@ jobs:
         echo "ticket=$TICKET" >> $GITHUB_OUTPUT
         echo "Extracted ticket: $TICKET"
 
+    - name: Mask sensitive values in test artifacts
+      if: always()
+      working-directory: e2e-tests
+      run: |
+        bash .github/scripts/mask_sensitive_artifacts.sh result result-failed
+
     - name: Generate Excel summary
       if: always()
       working-directory: e2e-tests