diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..50a4f16 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: shell-ci + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install shell tooling + run: | + sudo apt-get update + sudo apt-get install -y bats shellcheck + - name: Shell syntax + run: bash -n uHelper.sh lib/*.sh + - name: ShellCheck + run: shellcheck lib/*.sh + - name: Bats + run: bats tests/bats diff --git a/README.md b/README.md index 3db494c..e3ffa4e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ repository/mirror testing, and common server maintenance tasks. - Author: [farshidsar](https://github.com/farshidsar) - Repository: [github.com/farshidsar/uHelper](https://github.com/farshidsar/uHelper) - Main script: [`uHelper.sh`](./uHelper.sh) +- Modular automation library: [`lib/`](./lib/) - Languages: [فارسی](./README.fa.md) · [العربية](./README.ar.md) · [Русский](./README.ru.md) · [简体中文](./README.zh-CN.md) > **Important:** This script changes system networking, package, firewall, DNS, @@ -206,23 +207,29 @@ display. The original Mirava files are not required at runtime. ## Development roadmap -Planned improvements are intentionally focused on reliability and safe -operations: - -1. Split the monolithic Bash file into testable modules while preserving the - one-file portable launcher. -2. Add non-interactive flags and a dry-run mode for automation and CI. -3. Add structured JSON logging, operation IDs, and clearer failure reports. -4. Add a real mirror catalog schema with freshness checks, region metadata, - package type, TLS validation, and automatic stale-entry removal. -5. Improve Docker mirror verification with pull tests, daemon configuration - validation, rollback, and systemd health checks. -6. Add shellcheck, bats-core tests, CI syntax checks, and mocked package/network - integration tests. -7. Add privilege-aware support for Ubuntu/Debian first, then document tested - Fedora/RHEL/Alpine behavior instead of implying full portability. +The first reliability and automation milestone is implemented on the +`feature/modular-automation-observability` branch: + +1. **Completed:** Split the new automation paths into testable modules while + preserving the one-file portable launcher. +2. **Completed:** Add non-interactive flags and a dry-run mode for automation + and CI. +3. **Completed:** Add structured JSON logging, operation IDs, and explicit + failure reports. +4. **Completed:** Add a JSON Lines mirror catalog with freshness checks, region + metadata, package type, TLS validation, and stale-entry removal. +5. **Completed:** Improve Docker mirror verification with pull tests, daemon + configuration validation, backup/rollback, and systemd health checks. +6. **Completed:** Add ShellCheck, Bats tests, Bash syntax checks, CI, and + mocked network integration coverage. +7. **Completed (baseline):** Add privilege-aware guards for system-changing + automation, support Ubuntu/Debian first, and explicitly report + Fedora/RHEL/Alpine as documented but not fully supported. + +Remaining roadmap: + 8. Add configuration import/export, versioned backups, and transactional - rollback for DNS, APT, Docker, firewall, and SSH changes. + rollback for DNS, APT, firewall, and SSH changes. 9. Add localization infrastructure so menu text and README terminology stay synchronized. 10. Add release tags, checksums, signed releases, and a changelog. @@ -244,6 +251,37 @@ Source-only smoke test: bash -c 'source ./uHelper.sh; package_command dnsutils; command_exists bash' ``` +### Automation and CI mode + +The interactive menu remains available, while automation can use the +non-interactive modular CLI: + +```bash +bash uHelper.sh --dry-run --non-interactive --log-format json \ + --operation-id mirror-check --check-mirrors \ + https://archive.ubuntu.com/ubuntu/ +``` + +Available operations include `--check-mirrors`, `--prune-mirrors`, and +`--docker-mirrors`. Mirror records are stored as JSON Lines with URL, region, +package type, TLS status, and freshness timestamps. Stale records are removed +by `--prune-mirrors`. Docker updates validate `daemon.json`, back it up, check +systemd health, run a pull test, and roll back on failure. + +The launcher remains usable as a portable one-file interactive script when the +optional `lib/` directory is not present. The modular CLI requires Bash and +`curl`; `jq` is recommended for strict Docker JSON validation. + +Ubuntu and Debian are the supported platforms for system-changing operations. +Fedora/RHEL and Alpine are detected and reported as documented-but-not-fully- +supported targets rather than being presented as fully portable. + +```bash +bash -n uHelper.sh lib/*.sh +shellcheck lib/*.sh +bats tests/bats +``` + ## Contributing 1. Create a feature branch. diff --git a/lib/cli.sh b/lib/cli.sh new file mode 100644 index 0000000..1ca62a0 --- /dev/null +++ b/lib/cli.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034 + +uh_usage() { + cat <<'EOF' +Usage: uHelper.sh [OPTIONS] + --check-mirrors URL... Check HTTPS/package mirrors and update catalog + --docker-mirrors JSON Validate/apply Docker registry mirrors + --prune-mirrors Remove stale catalog entries + --dry-run Log actions without changing the system + --non-interactive Never prompt; required for automation + --log-format FORMAT text (default) or json + --operation-id ID Set a correlation/operation ID + --help Show this help +EOF +} + +uh_cli_main() { + local action="" docker_mirrors="" urls=() + while (($#)); do + case "$1" in + --check-mirrors) action=check; shift; while (($#)) && [[ "$1" != --* ]]; do urls+=("$1"); shift; done; continue ;; + --docker-mirrors) action=docker; docker_mirrors="${2:-}"; shift 2; continue ;; + --prune-mirrors) action=prune; shift; continue ;; + --dry-run) UH_DRY_RUN=1; shift; continue ;; + --non-interactive) UH_NON_INTERACTIVE=1; shift; continue ;; + --log-format) UH_LOG_FORMAT="${2:-text}"; shift 2; continue ;; + --operation-id) UH_OPERATION_ID="${2:-}"; shift 2; continue ;; + --help|-h) uh_usage; return 0 ;; + *) uh_usage >&2; return 2 ;; + esac + done + uh_log_init + [[ -n "$action" ]] || { uh_usage; return 0; } + uh_require_supported_platform || return 1 + case "$action" in + check) + ((${#urls[@]})) || uh_fail "--check-mirrors requires at least one URL" || return 2 + local url; for url in "${urls[@]}"; do uh_mirror_check "$url" "${UH_REGION:-unknown}" "${UH_PACKAGE_TYPE:-apt}" || true; done + uh_mirror_prune_stale + ;; + prune) uh_mirror_prune_stale ;; + docker) + [[ "$docker_mirrors" == \[*\] ]] || uh_fail "--docker-mirrors requires a JSON array" || return 2 + uh_docker_apply_mirrors "$docker_mirrors" + ;; + esac +} diff --git a/lib/docker.sh b/lib/docker.sh new file mode 100644 index 0000000..c39918b --- /dev/null +++ b/lib/docker.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +UH_DOCKER_CONFIG="${UH_DOCKER_CONFIG:-/etc/docker/daemon.json}" +UH_DOCKER_BACKUP="${UH_DOCKER_BACKUP:-${UH_DOCKER_CONFIG}.uhelper.bak}" + +uh_docker_validate_daemon_config() { + [[ -f "$UH_DOCKER_CONFIG" ]] || return 0 + command -v jq >/dev/null 2>&1 || { uh_log WARN "jq is required to validate Docker daemon configuration"; return 0; } + jq empty "$UH_DOCKER_CONFIG" >/dev/null 2>&1 +} + +uh_docker_backup_config() { + [[ -f "$UH_DOCKER_CONFIG" ]] || return 0 + uh_run cp -p "$UH_DOCKER_CONFIG" "$UH_DOCKER_BACKUP" +} + +uh_docker_restore_config() { + [[ -f "$UH_DOCKER_BACKUP" ]] || return 0 + uh_run cp -p "$UH_DOCKER_BACKUP" "$UH_DOCKER_CONFIG" + uh_log WARN "Docker daemon configuration rolled back" +} + +uh_docker_pull_test() { + local image="${1:-hello-world:latest}" + command -v docker >/dev/null 2>&1 || { uh_log WARN "docker is not installed; pull test skipped"; return 0; } + [[ "${UH_DRY_RUN:-0}" == "1" ]] && { uh_log INFO "dry-run Docker pull test" "$(uh_log_field image "$image")"; return 0; } + docker pull "$image" >/dev/null +} + +uh_docker_systemd_health() { + command -v systemctl >/dev/null 2>&1 || return 0 + systemctl is-active --quiet docker +} + +uh_docker_apply_mirrors() { + local mirrors_json="$1" + uh_is_root_or_dry_run || uh_fail "Docker mirror configuration requires root privileges" || return 1 + uh_docker_backup_config || return 1 + if [[ "${UH_DRY_RUN:-0}" == "1" ]]; then + uh_log INFO "dry-run Docker daemon update" "$(uh_log_field mirrors "$mirrors_json")" + return 0 + fi + mkdir -p "$(dirname "$UH_DOCKER_CONFIG")" || return 1 + printf '{"registry-mirrors":%s}\n' "$mirrors_json" >"$UH_DOCKER_CONFIG" || return 1 + uh_docker_validate_daemon_config || { uh_docker_restore_config; return 1; } + if command -v systemctl >/dev/null 2>&1; then + systemctl restart docker || { uh_docker_restore_config; systemctl restart docker || true; return 1; } + uh_docker_systemd_health || { uh_docker_restore_config; systemctl restart docker || true; return 1; } + fi + uh_docker_pull_test "hello-world:latest" +} diff --git a/lib/logging.sh b/lib/logging.sh new file mode 100644 index 0000000..32f90df --- /dev/null +++ b/lib/logging.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +uh_log_init() { + UH_LOG_FORMAT="${UH_LOG_FORMAT:-text}" + UH_LOG_FILE="${UH_LOG_FILE:-}" + UH_OPERATION_ID="${UH_OPERATION_ID:-uh-$(date -u +%Y%m%dT%H%M%SZ)-$$}" +} + +uh_json_escape() { + local value="${1-}" + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//$'\n'/\\n} + value=${value//$'\r'/\\r} + value=${value//$'\t'/\\t} + printf '%s' "$value" +} + +uh_log() { + local level="${1:-INFO}" message="${2:-}" fields="${3:-}" + local timestamp line + timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + if [[ "${UH_LOG_FORMAT:-text}" == "json" ]]; then + line=$(printf '{"timestamp":"%s","level":"%s","operation_id":"%s","message":"%s"%s}' \ + "$timestamp" "$(uh_json_escape "$level")" "$(uh_json_escape "${UH_OPERATION_ID:-unknown}")" \ + "$(uh_json_escape "$message")" "$fields") + else + line="[$timestamp] [$level] [${UH_OPERATION_ID:-unknown}] $message" + fi + printf '%s\n' "$line" >&2 + if [[ -n "${UH_LOG_FILE:-}" ]]; then + printf '%s\n' "$line" >>"$UH_LOG_FILE" + fi +} + +uh_log_field() { + printf ',\"%s\":\"%s\"' "$(uh_json_escape "$1")" "$(uh_json_escape "${2-}")" +} + +uh_fail() { + uh_log ERROR "$1" "${2:-}" + return 1 +} diff --git a/lib/mirrors.sh b/lib/mirrors.sh new file mode 100644 index 0000000..eb001b7 --- /dev/null +++ b/lib/mirrors.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +UH_MIRROR_CATALOG="${UH_MIRROR_CATALOG:-${XDG_STATE_HOME:-/var/lib}/uhelper/mirrors.jsonl}" +UH_MIRROR_MAX_AGE="${UH_MIRROR_MAX_AGE:-604800}" + +uh_mirror_catalog_init() { + if [[ "${UH_DRY_RUN:-0}" == "1" ]]; then return 0; fi + mkdir -p "$(dirname "$UH_MIRROR_CATALOG")" || return 1 + touch "$UH_MIRROR_CATALOG" +} + +uh_mirror_catalog_add() { + local url="$1" region="$2" package_type="$3" tls="$4" checked_at + checked_at="$(date -u +%s)" + uh_mirror_catalog_init || return 1 + printf '{"url":"%s","region":"%s","package_type":"%s","tls_valid":%s,"checked_at":%s}\n' \ + "$(uh_json_escape "$url")" "$(uh_json_escape "$region")" \ + "$(uh_json_escape "$package_type")" "$tls" "$checked_at" >>"$UH_MIRROR_CATALOG" +} + +uh_mirror_tls_valid() { + local url="$1" + [[ "$url" == https://* ]] || return 1 + command -v curl >/dev/null 2>&1 || return 1 + curl --fail --silent --show-error --head --max-time "${UH_MIRROR_TIMEOUT:-8}" "$url" >/dev/null +} + +uh_mirror_check() { + local url="$1" region="${2:-unknown}" package_type="${3:-apt}" status=0 tls=false + [[ "$url" == https://* ]] && tls=true + if [[ "${UH_DRY_RUN:-0}" == "1" ]]; then + uh_log INFO "dry-run mirror check" "$(uh_log_field url "$url")" + return 0 + fi + if ! curl --fail --silent --show-error --location --max-time "${UH_MIRROR_TIMEOUT:-8}" \ + --output /dev/null "$url"; then status=1; fi + if [[ "$tls" == true ]] && ! uh_mirror_tls_valid "$url"; then status=1; fi + if (( status == 0 )); then + uh_mirror_catalog_add "$url" "$region" "$package_type" "$tls" + uh_log INFO "mirror is healthy" "$(uh_log_field url "$url")$(uh_log_field package_type "$package_type")" + else + uh_log WARN "mirror is stale or unreachable" "$(uh_log_field url "$url")" + fi + return "$status" +} + +uh_mirror_prune_stale() { + [[ -f "$UH_MIRROR_CATALOG" ]] || return 0 + [[ "${UH_DRY_RUN:-0}" == "1" ]] && { uh_log INFO "dry-run mirror prune"; return 0; } + local now line checked_at + now="$(date -u +%s)" + local tmp="${UH_MIRROR_CATALOG}.tmp" + : >"$tmp" || return 1 + while IFS= read -r line; do + checked_at="$(printf '%s\n' "$line" | sed -n 's/.*"checked_at":\([0-9][0-9]*\).*/\1/p')" + if [[ -n "$checked_at" ]] && (( now - checked_at <= UH_MIRROR_MAX_AGE )); then + printf '%s\n' "$line" >>"$tmp" + fi + done <"$UH_MIRROR_CATALOG" + mv "$tmp" "$UH_MIRROR_CATALOG" +} diff --git a/lib/platform.sh b/lib/platform.sh new file mode 100644 index 0000000..bbd5f71 --- /dev/null +++ b/lib/platform.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +uh_detect_platform() { + UH_PLATFORM="unknown" + UH_PACKAGE_MANAGER="" + if [[ -r /etc/os-release ]]; then + # shellcheck disable=SC1091 + . /etc/os-release + UH_PLATFORM="${ID:-unknown}" + fi + if command -v apt-get >/dev/null 2>&1; then + UH_PACKAGE_MANAGER=apt + elif command -v dnf >/dev/null 2>&1; then + UH_PACKAGE_MANAGER=dnf + elif command -v yum >/dev/null 2>&1; then + UH_PACKAGE_MANAGER=yum + elif command -v apk >/dev/null 2>&1; then + UH_PACKAGE_MANAGER=apk + fi + uh_log INFO "platform detected" "$(uh_log_field platform "$UH_PLATFORM")$(uh_log_field package_manager "$UH_PACKAGE_MANAGER")" +} + +uh_require_supported_platform() { + uh_detect_platform + case "$UH_PLATFORM" in + ubuntu|debian) return 0 ;; + fedora|rhel|centos|alpine) + uh_fail "platform is documented but not fully supported: $UH_PLATFORM" + ;; + *) + uh_fail "unsupported platform: $UH_PLATFORM" + ;; + esac +} + +uh_is_root_or_dry_run() { + [[ "${UH_DRY_RUN:-0}" == "1" || "${EUID:-$(id -u)}" -eq 0 ]] +} + +uh_run() { + if [[ "${UH_DRY_RUN:-0}" == "1" ]]; then + uh_log INFO "dry-run command" "$(uh_log_field command "$*")" + return 0 + fi + "$@" +} diff --git a/tests/bats/cli.bats b/tests/bats/cli.bats new file mode 100644 index 0000000..9f7fffe --- /dev/null +++ b/tests/bats/cli.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +setup() { + ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + TMP="$(mktemp -d)" + export UH_MIRROR_CATALOG="$TMP/mirrors.jsonl" + export UH_LOG_FORMAT=json + mkdir -p "$TMP/bin" + printf '#!/usr/bin/env bash\nexit 0\n' >"$TMP/bin/curl" + chmod +x "$TMP/bin/curl" +} + +teardown() { rm -rf "$TMP"; } + +@test "help is non-interactive" { + run bash "$ROOT/uHelper.sh" --help + [ "$status" -eq 0 ] + [[ "$output" == *"--dry-run"* ]] +} + +@test "dry-run mirror checks emit operation ids" { + run bash "$ROOT/uHelper.sh" --dry-run --non-interactive --check-mirrors https://mirror.example/ubuntu/ + [ "$status" -eq 0 ] + [[ "$output" == *"operation_id"* ]] +} + +@test "mocked mirror checks write the catalog schema" { + PATH="$TMP/bin:$PATH" run bash "$ROOT/uHelper.sh" --non-interactive \ + --check-mirrors https://mirror.example/ubuntu/ + [ "$status" -eq 0 ] + [ -s "$UH_MIRROR_CATALOG" ] + grep -q '"package_type":"apt"' "$UH_MIRROR_CATALOG" + grep -q '"tls_valid":true' "$UH_MIRROR_CATALOG" +} diff --git a/uHelper.sh b/uHelper.sh index 21ed4ca..2778569 100644 --- a/uHelper.sh +++ b/uHelper.sh @@ -1,5 +1,21 @@ #!/usr/bin/env bash +# Modular automation layer. The legacy interactive menu below remains intact +# and keeps this file usable as a portable one-file launcher. +UH_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "$UH_SCRIPT_DIR/lib/logging.sh" ]]; then + # shellcheck source=lib/logging.sh + . "$UH_SCRIPT_DIR/lib/logging.sh" + # shellcheck source=lib/platform.sh + . "$UH_SCRIPT_DIR/lib/platform.sh" + # shellcheck source=lib/mirrors.sh + . "$UH_SCRIPT_DIR/lib/mirrors.sh" + # shellcheck source=lib/docker.sh + . "$UH_SCRIPT_DIR/lib/docker.sh" + # shellcheck source=lib/cli.sh + . "$UH_SCRIPT_DIR/lib/cli.sh" +fi + # This script intentionally avoids `set -e`: most menu actions are optional and # report their own failures. `set -u` is also not safe for legacy menu inputs. @@ -4555,7 +4571,11 @@ main() { } if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then - main "$@" + if (($#)) && [[ -f "$UH_SCRIPT_DIR/lib/cli.sh" ]]; then + uh_cli_main "$@" + else + main "$@" + fi fi # ########################################################################### # --- END OF SCRIPT ---