diff --git a/README.md b/README.md index ae310c0..6c30954 100644 --- a/README.md +++ b/README.md @@ -474,6 +474,13 @@ outputs. sqrbx-update lazygit # update, or explicitly install, one tool sqrbx-update --list # list all tools and current versions +Setup and `sqrbx-update` read GitHub release metadata through the GitHub API, +which allows only 60 unauthenticated requests per hour per IP address. On a +shared NAT or CI runner, authenticate those requests to avoid HTTP 403 rate +limits: export `GH_TOKEN` (or `GITHUB_TOKEN`), or run `gh auth login` inside +the Box. The token is optional, is sent only to the GitHub API, and a rejected +token falls back to unauthenticated requests. + ### Full rebuild (from the host) sqrbx-rebuild diff --git a/SECURITY.md b/SECURITY.md index 3701a7e..39d20cc 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -169,6 +169,17 @@ Unsupported architectures and invalid Tool-tier/destination combinations fail before network or destination mutation. Extracted archives reject escaping links, special files, and ambiguous executable matches before promotion. +GitHub API metadata requests are optionally authenticated to raise the +unauthenticated rate limit. The token comes from `GH_TOKEN`, then +`GITHUB_TOKEN`, then an already-authenticated `gh` CLI (`gh auth token`, only +for the default `https://api.github.com` base; never prompted). It is sent as +an `Authorization: Bearer` header only to the configured HTTPS API base, never +to artifact downloads or redirect targets, and reaches curl through a stdin +config rather than its command line. It is not logged or written to the +metadata cache. An HTTP 401 drops the token for the rest of that run and +retries once unauthenticated. Authentication changes only rate limits; it does +not relax digest verification. + The GitHub CLI and Eza APT repositories are configured only while the image builds (then removed). Their signing keys are not trusted on download alone: the Dockerfile pins each key's expected full 40-hex primary-key fingerprint set diff --git a/scripts/lib/tool-lib.sh b/scripts/lib/tool-lib.sh index 74067e4..eb94ff3 100644 --- a/scripts/lib/tool-lib.sh +++ b/scripts/lib/tool-lib.sh @@ -376,8 +376,68 @@ _sb_gh_cache_failure() { /bin/mv -fT -- "$stage" "$path" 2>/dev/null || rm -f -- "$stage" 2>/dev/null || true } +# Optional GitHub API authentication raises the unauthenticated 60 requests per +# hour limit. Precedence: GH_TOKEN, GITHUB_TOKEN, then an already-authenticated +# gh CLI (evaluated lazily, at most once per shell; never prompts). The token is +# sent only to the configured HTTPS SB_GITHUB_API_BASE, reaches curl through a +# stdin config rather than argv, and is never logged or written to the metadata +# cache. curl does not forward a custom Authorization header to another host +# when following a redirect. +_SB_GH_TOKEN="" +_SB_GH_TOKEN_RESOLVED=false +_SB_GH_TOKEN_DISABLED=false + +_sb_gh_resolve_token() { + local token="" source="" + [ "$_SB_GH_TOKEN_RESOLVED" = false ] || return 0 + _SB_GH_TOKEN_RESOLVED=true + _SB_GH_TOKEN="" + if [ -n "${GH_TOKEN:-}" ]; then + token=$GH_TOKEN; source=GH_TOKEN + elif [ -n "${GITHUB_TOKEN:-}" ]; then + token=$GITHUB_TOKEN; source=GITHUB_TOKEN + elif [ "${SB_GITHUB_API_BASE%/}" = "https://api.github.com" ] && command -v gh >/dev/null 2>&1; then + # gh's stored credential belongs to github.com; never offer it to a + # custom API base. Without a stored login this fails without prompting. + token=$(GH_PROMPT_DISABLED=1 gh auth token --hostname github.com /dev/null) || token="" + source="gh auth token" + fi + [ -n "$token" ] || return 0 + # Restrict to token characters so the value cannot alter curl's config. + if ! [[ "$token" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "Warning: ignoring malformed GitHub token from ${source}; using unauthenticated GitHub API requests" >&2 + return 0 + fi + _SB_GH_TOKEN=$token +} + +# Succeed (with _SB_GH_TOKEN set) only when url targets the configured HTTPS +# API base and a usable token exists. Runs in the current shell so lazy token +# resolution is remembered. +_sb_gh_token_applies() { + local url="$1" base="${SB_GITHUB_API_BASE%/}" + [ "$_SB_GH_TOKEN_DISABLED" = false ] || return 1 + [[ "$base" == https://?* ]] || return 1 + [[ "$url" == "$base/"* ]] || return 1 + _sb_gh_resolve_token + [ -n "$_SB_GH_TOKEN" ] +} + +# One metadata GET. With authentication, the header is supplied through a curl +# config here-string on stdin so the token never appears in argv or ps output. +_sb_gh_curl_metadata() { + local url="$1" authenticated="$2" + if [ "$authenticated" = true ]; then + curl -K - -sSL -w '\n%{http_code}' "$url" 2>/dev/null \ + <<<"header = \"Authorization: Bearer ${_SB_GH_TOKEN}\"" + else + curl -sSL -w '\n%{http_code}' "$url" 2>/dev/null + fi +} + _sb_gh_api_get() { local url="$1" context="$2" response curl_rc http_code body message cache_path failed_path + local authenticated=false if [ -n "${_SB_GH_BODY_CACHE[$url]+x}" ]; then SB_GH_API_BODY=${_SB_GH_BODY_CACHE[$url]} printf '%s\n' "$SB_GH_API_BODY" @@ -400,8 +460,19 @@ _sb_gh_api_get() { _SB_GH_FAILED_CACHE[$url]=1 return 1 fi - response=$(curl -sSL -w '\n%{http_code}' "$url" 2>/dev/null) + ! _sb_gh_token_applies "$url" || authenticated=true + response=$(_sb_gh_curl_metadata "$url" "$authenticated") curl_rc=$? + if [ "$authenticated" = true ] && [ "$curl_rc" -eq 0 ] && [ "${response##*$'\n'}" = "401" ]; then + # A rejected token must not block public metadata: drop it for the rest + # of this shell and retry once without credentials. + echo "Warning: GitHub API rejected the configured token (HTTP 401) for ${context}; retrying unauthenticated" >&2 + _SB_GH_TOKEN_DISABLED=true + _SB_GH_TOKEN="" + authenticated=false + response=$(_sb_gh_curl_metadata "$url" false) + curl_rc=$? + fi if [ "$curl_rc" -ne 0 ]; then _SB_GH_FAILED_CACHE[$url]=1 _sb_gh_cache_failure "$url" @@ -440,8 +511,10 @@ _sb_gh_api_get() { _SB_GH_FAILED_CACHE[$url]=1 _sb_gh_cache_failure "$url" message=$(printf '%s' "$body" | jq -r '.message // empty' 2>/dev/null || true) - if [ "$http_code" = "403" ]; then - echo "Error: GitHub API returned HTTP 403 for ${context}${message:+: $message} (possible rate limit)" >&2 + if { [ "$http_code" = "403" ] || [ "$http_code" = "429" ]; } && [ "$authenticated" = false ]; then + echo "Error: GitHub API returned HTTP ${http_code} for ${context}${message:+: $message} (possible unauthenticated rate limit; set GH_TOKEN or run 'gh auth login' to raise it)" >&2 + elif [ "$http_code" = "403" ] || [ "$http_code" = "429" ]; then + echo "Error: GitHub API returned HTTP ${http_code} for ${context}${message:+: $message} (possible rate limit)" >&2 else echo "Error: GitHub API returned HTTP ${http_code} for ${context}${message:+: $message}" >&2 fi diff --git a/tests/test-tool-lib.sh b/tests/test-tool-lib.sh index 07961ae..390b2d6 100755 --- a/tests/test-tool-lib.sh +++ b/tests/test-tool-lib.sh @@ -421,6 +421,122 @@ if ( [ "$(grep -c 'api.test/' "$CACHE_CASE/curl.log")" -eq 1 ] ); then ok "release metadata cache survives command substitution and exact-version install"; else not_ok "release metadata cache survives command substitution and exact-version install"; fi +# ── Optional GitHub API authentication ─────────────────────────────── +# The auth mock curl records every argv element and, only when given a config +# on stdin (-K -), that config. AUTH_STATUS_WITH_TOKEN overrides the API status +# for authenticated requests (401 fallback). Downloads copy PAYLOAD. +AUTH_TOKEN_VALUE=ghp_SyntheticSecretToken0123456789 +run_auth_case() { + local name=$1 body=$2 + local case_dir="$TMP/auth-$name" + mkdir -p "$case_dir/home" "$case_dir/cache" + ( + unset GH_TOKEN GITHUB_TOKEN + export HOME="$case_dir/home" SB_TOOLS_YAML="$REGISTRY" SB_DPKG_ARCH=amd64 + export SB_GITHUB_API_BASE=https://api.test SB_GH_METADATA_CACHE_DIR="$case_dir/cache" + export CASE_DIR="$case_dir" PAYLOAD_HASH + source "$REPO_ROOT/scripts/lib/tool-lib.sh" + gh() { printf 'gh %s\n' "$*" >> "$CASE_DIR/gh.log"; return 1; } + curl() { + local output="" url="" config="" arg + for arg in "$@"; do printf '%s\n' "$arg" >> "$CASE_DIR/argv.log"; done + printf -- '--\n' >> "$CASE_DIR/argv.log" + while [ "$#" -gt 0 ]; do + case "$1" in + -K) [ "$2" = - ] && config=$(cat); shift 2 ;; + -w) shift 2 ;; + -o) output=$2; shift 2 ;; + -*o) output=$2; shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac + done + printf '%s|%s\n' "$url" "${config:-none}" >> "$CASE_DIR/requests.log" + if [[ "$url" == http*://api.test/* || "$url" == https://api.github.com/* ]]; then + if [ -n "$config" ] && [ -n "${AUTH_STATUS_WITH_TOKEN:-}" ]; then + printf '{"message":"Bad credentials"}\n%s\n' "$AUTH_STATUS_WITH_TOKEN" + else + printf '{"tag_name":"v1.0.0","assets":[{"name":"sample-1.0.0-amd64","digest":"sha256:%s"}]}\n%s\n' \ + "$PAYLOAD_HASH" "${API_STATUS:-200}" + fi + else + cp "$PAYLOAD" "$output" + fi + } + eval "$body" + ) >"$case_dir/out" 2>"$case_dir/err" +} +auth_header_line="header = \"Authorization: Bearer $AUTH_TOKEN_VALUE\"" +token_leaked() { + local case_dir=$1 + grep -rqF "$AUTH_TOKEN_VALUE" "$case_dir/argv.log" "$case_dir/cache" "$case_dir/out" "$case_dir/err" 2>/dev/null +} + +if run_auth_case gh-token "export GH_TOKEN=$AUTH_TOKEN_VALUE GITHUB_TOKEN=other_token; sb_install sample latest" \ + && grep -qxF "https://api.test/repos/example/sample/releases/latest|$auth_header_line" "$TMP/auth-gh-token/requests.log" \ + && [ -x "$TMP/auth-gh-token/home/.local/bin/sample" ]; then + ok "GH_TOKEN sends a Bearer Authorization header to the GitHub API" +else not_ok "GH_TOKEN sends a Bearer Authorization header to the GitHub API"; fi + +if ! token_leaked "$TMP/auth-gh-token" && [ -n "$(ls -A "$TMP/auth-gh-token/cache")" ]; then + ok "GitHub token is absent from curl argv, output, and the metadata cache" +else not_ok "GitHub token is absent from curl argv, output, and the metadata cache"; fi + +if grep -q '/releases/download/v1.0.0/sample-1.0.0-amd64|none$' "$TMP/auth-gh-token/requests.log" \ + && [ "$(grep -c '|none$' "$TMP/auth-gh-token/requests.log")" -eq 1 ]; then + ok "GitHub token is never sent with artifact downloads" +else not_ok "GitHub token is never sent with artifact downloads"; fi + +if run_auth_case github-token "export GITHUB_TOKEN=$AUTH_TOKEN_VALUE; sb_gh_latest_tag example/sample >/dev/null" \ + && grep -qxF "https://api.test/repos/example/sample/releases/latest|$auth_header_line" "$TMP/auth-github-token/requests.log" \ + && [ ! -e "$TMP/auth-github-token/gh.log" ]; then + ok "GITHUB_TOKEN authenticates when GH_TOKEN is unset" +else not_ok "GITHUB_TOKEN authenticates when GH_TOKEN is unset"; fi + +if run_auth_case no-token "sb_gh_latest_tag example/sample >/dev/null" \ + && grep -qxF 'https://api.test/repos/example/sample/releases/latest|none' "$TMP/auth-no-token/requests.log" \ + && ! grep -qx -- '-K' "$TMP/auth-no-token/argv.log" \ + && [ ! -e "$TMP/auth-no-token/gh.log" ]; then + ok "without a token no Authorization header is sent and gh is not consulted for a custom API base" +else not_ok "without a token no Authorization header is sent and gh is not consulted for a custom API base"; fi + +if run_auth_case plain-http "export GH_TOKEN=$AUTH_TOKEN_VALUE SB_GITHUB_API_BASE=http://api.test; sb_gh_latest_tag example/sample >/dev/null" \ + && grep -qxF 'http://api.test/repos/example/sample/releases/latest|none' "$TMP/auth-plain-http/requests.log"; then + ok "GitHub token is withheld from a non-HTTPS API base" +else not_ok "GitHub token is withheld from a non-HTTPS API base"; fi + +if run_auth_case fallback-401 "export GH_TOKEN=$AUTH_TOKEN_VALUE AUTH_STATUS_WITH_TOKEN=401 + sb_gh_latest_tag example/sample >/dev/null && sb_gh_latest_tag example/packed >/dev/null" \ + && [ "$(sed -n 1p "$TMP/auth-fallback-401/requests.log")" = "https://api.test/repos/example/sample/releases/latest|$auth_header_line" ] \ + && [ "$(sed -n 2p "$TMP/auth-fallback-401/requests.log")" = 'https://api.test/repos/example/sample/releases/latest|none' ] \ + && [ "$(sed -n 3p "$TMP/auth-fallback-401/requests.log")" = 'https://api.test/repos/example/packed/releases/latest|none' ] \ + && [ "$(wc -l < "$TMP/auth-fallback-401/requests.log")" -eq 3 ] \ + && grep -q 'rejected the configured token (HTTP 401).*retrying unauthenticated' "$TMP/auth-fallback-401/err" \ + && ! token_leaked "$TMP/auth-fallback-401"; then + ok "HTTP 401 with a token retries once unauthenticated, warns, and drops the token" +else not_ok "HTTP 401 with a token retries once unauthenticated, warns, and drops the token"; fi + +if ! run_auth_case rate-limit "export API_STATUS=403; sb_gh_latest_tag example/sample >/dev/null" \ + && grep -q "HTTP 403.*set GH_TOKEN or run 'gh auth login'" "$TMP/auth-rate-limit/err"; then + ok "unauthenticated HTTP 403 fails closed and suggests GH_TOKEN or gh auth login" +else not_ok "unauthenticated HTTP 403 fails closed and suggests GH_TOKEN or gh auth login"; fi + +if run_auth_case gh-cli "export SB_GITHUB_API_BASE=https://api.github.com + gh() { printf 'gh %s\n' \"\$*\" >> \"\$CASE_DIR/gh.log\"; [ \"\$1 \$2\" = 'auth token' ] && printf '%s\n' $AUTH_TOKEN_VALUE; } + sb_gh_latest_tag example/sample >/dev/null && sb_gh_latest_tag example/packed >/dev/null" \ + && [ "$(wc -l < "$TMP/auth-gh-cli/gh.log")" -eq 1 ] \ + && [ "$(grep -cF "|$auth_header_line" "$TMP/auth-gh-cli/requests.log")" -eq 2 ] \ + && ! token_leaked "$TMP/auth-gh-cli"; then + ok "authenticated gh CLI token is resolved lazily once for the default API base" +else not_ok "authenticated gh CLI token is resolved lazily once for the default API base"; fi + +if run_auth_case malformed-token "export GH_TOKEN='bad\"token'; sb_gh_latest_tag example/sample >/dev/null" \ + && grep -qxF 'https://api.test/repos/example/sample/releases/latest|none' "$TMP/auth-malformed-token/requests.log" \ + && grep -q 'ignoring malformed GitHub token from GH_TOKEN' "$TMP/auth-malformed-token/err" \ + && ! grep -qF 'bad"token' "$TMP/auth-malformed-token/err"; then + ok "malformed token is ignored without being printed" +else not_ok "malformed token is ignored without being printed"; fi + BAD_REGISTRY="$TMP/bad-tools.yaml" cp "$REGISTRY" "$BAD_REGISTRY" sed -i '0,/method: binary/s//method: shell-pipe/' "$BAD_REGISTRY"