From 065857ee402fbe70425afc20400c06130f1db41b Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 10:35:43 +0100 Subject: [PATCH 1/6] Build cernbox with gaia from reva's CLI and the CERNBox plugins cernbox is now reva's CLI, rebranded at build time: gaia builds it with the CERNBox CLI plugins of reva-plugins and the CERNBox brand (cli/cernbox.toml there), the same way as revad. This repository packages that build instead of its own client. packaging/prepare.sh runs gaia's prepare stage into ws/, with the versions of packaging/versions.env, or with local checkouts of reva and reva-plugins for development. gaia checks the brand against the plugins of the build there, so a bad brand fails before anything is compiled. The brand comes from the module cache at the version of reva-plugins that the build uses, so the two cannot drift apart. goreleaser cross-compiles ws/ for the same platforms as before, with the version flags that gaia writes to ws/bflags. They reach goreleaser through BFLAGS, because goreleaser reads its environment before any hook runs. The archives, the packages and their install paths keep their names. The site configuration becomes /etc/cernbox/config.toml, which the new CLI reads, and holds no values, because the brand has them all. The completions are generated from the built binary with an empty home, because the CLI copies a config.yaml it finds into config.toml. The three versions are master for now: none of gaia, reva and reva-plugins has a release with what this needs yet. Assisted-by: Claude:claude-opus-5-5 --- .gitignore | 3 ++ .goreleaser.yaml | 38 ++++++++++------ Makefile | 48 +++++++++++++-------- packaging/completions.sh | 35 +++++++++++++++ packaging/config.toml | 25 +++++++++++ packaging/config.yaml | 25 ----------- packaging/prepare.sh | 93 ++++++++++++++++++++++++++++++++++++++++ packaging/versions.env | 14 ++++++ 8 files changed, 224 insertions(+), 57 deletions(-) create mode 100755 packaging/completions.sh create mode 100644 packaging/config.toml delete mode 100644 packaging/config.yaml create mode 100755 packaging/prepare.sh create mode 100644 packaging/versions.env diff --git a/.gitignore b/.gitignore index fc770b7..86c5588 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ /cernbox /dist/ +# The gaia workspace of the build (packaging/prepare.sh) +/ws/ + # Generated shell completions /packaging/completions/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b8f7c6b..02c412f 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,19 +2,30 @@ version: 2 project_name: cernbox-cli +# cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and the +# CERNBox brand. packaging/prepare.sh runs gaia's prepare stage into ws/, and +# goreleaser cross-compiles that workspace. BFLAGS is the content of +# ws/bflags, the -X flags of gaia that set the version of reva: goreleaser +# reads the environment before any hook runs, so prepare runs before +# goreleaser (make snapshot, and the release job), not as a hook here. +# +# packaging/prepare.sh +# BFLAGS="$(cat ws/bflags)" goreleaser release --clean + +env: + # A go.work above ws/ would replace the module of the workspace. + - GOWORK=off + before: hooks: - - go mod tidy # Completions are generated rather than committed, so they can never drift # from the command tree they describe. - - mkdir -p packaging/completions - - sh -c "go run ./cmd/cernbox completion bash > packaging/completions/cernbox.bash" - - sh -c "go run ./cmd/cernbox completion zsh > packaging/completions/cernbox.zsh" - - sh -c "go run ./cmd/cernbox completion fish > packaging/completions/cernbox.fish" + - sh packaging/completions.sh builds: - id: cernbox - main: ./cmd/cernbox + dir: ws + main: . binary: cernbox # CGO off keeps the binary statically linkable, which is what lets one # artifact run across AlmaLinux versions and inside minimal containers. @@ -22,11 +33,10 @@ builds: - CGO_ENABLED=0 goos: [linux, darwin, windows] goarch: [amd64, arm64] + flags: + - -trimpath ldflags: - - -s -w - - -X main.version={{ .Version }} - - -X main.commit={{ .ShortCommit }} - - -X main.buildDate={{ .Date }} + - -s -w {{ .Env.BFLAGS }} archives: - id: default @@ -40,10 +50,10 @@ archives: files: - README.md - LICENSE - - docs/design.md + - docs/release-notes-2.0.md # RPM for AlmaLinux, which is what lxplus runs. The site configuration is -# marked as a config file so a local edit survives an upgrade. +# marked as a config file so a local edit (or puppet's) survives an upgrade. nfpms: - id: packages ids: [cernbox] @@ -58,8 +68,8 @@ nfpms: formats: [rpm, deb] bindir: /usr/bin contents: - - src: packaging/config.yaml - dst: /etc/cernbox/config.yaml + - src: packaging/config.toml + dst: /etc/cernbox/config.toml type: config|noreplace - src: packaging/completions/cernbox.bash dst: /usr/share/bash-completion/completions/cernbox diff --git a/Makefile b/Makefile index 6b28972..cc85a98 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,37 @@ BINARY := cernbox -CMD := ./cmd/cernbox -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) -BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) -LDFLAGS := -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE) +WS := ws +GORELEASER ?= goreleaser + +# The build of cernbox is gaia's: packaging/prepare.sh writes the workspace +# ws/, with the versions of packaging/versions.env, and go or goreleaser +# compile it. For a local reva or reva-plugins, set REVA_DIR or +# REVA_PLUGINS_DIR (and BRAND for another brand file); GAIA names the gaia +# binary. They go to prepare.sh only, not exported: cernbox warns about any +# REVA_* variable it sees, so the tests should not see them. +PREPARE := GAIA="$(GAIA)" BRAND="$(BRAND)" REVA_DIR="$(REVA_DIR)" REVA_PLUGINS_DIR="$(REVA_PLUGINS_DIR)" \ + REVA_VERSION="$(REVA_VERSION)" REVA_PLUGINS_VERSION="$(REVA_PLUGINS_VERSION)" packaging/prepare.sh COMPOSE := docker compose -f dev/docker-compose.yaml -.PHONY: all build install test test-race test-cover test-integration test-all \ +.PHONY: all prepare build install snapshot test test-race test-cover test-integration test-all \ lint vet fmt tidy clean completions dev-up dev-down dev-logs help all: build -build: ## Build the cernbox binary - go build -ldflags "$(LDFLAGS)" -o $(BINARY) $(CMD) +prepare: ## Prepare the gaia workspace of cernbox (ws/) + $(PREPARE) -install: ## Install the cernbox binary into GOBIN - go install -ldflags "$(LDFLAGS)" $(CMD) +$(WS)/bflags: + $(PREPARE) + +build: $(WS)/bflags ## Build cernbox for this machine from the workspace + cd $(WS) && GOWORK=off CGO_ENABLED=0 go build -trimpath -ldflags "-s -w $$(cat bflags)" -o ../$(BINARY) . + +install: build ## Install cernbox into GOBIN + install -m 0755 $(BINARY) "$$(go env GOBIN | grep . || echo "$$(go env GOPATH)/bin")/$(BINARY)" + +snapshot: $(WS)/bflags ## Build the release artefacts into dist/, without publishing + BFLAGS="$$(cat $(WS)/bflags)" $(GORELEASER) release --snapshot --clean --skip=publish test: ## Run unit tests go test ./... @@ -29,8 +44,8 @@ test-cover: ## Run unit tests and write coverage.out / coverage.html go tool cover -html=coverage.out -o coverage.html @go tool cover -func=coverage.out | tail -1 -test-integration: build ## Run integration tests (requires: make dev-up) - go test -tags integration -count=1 -timeout 20m ./integration/... +test-integration: build ## Run integration tests against ./cernbox (requires: make dev-up) + CERNBOX_BINARY=$(CURDIR)/$(BINARY) go test -tags integration -count=1 -timeout 20m ./integration/... test-all: dev-up test test-integration ## Run every test suite @@ -47,15 +62,12 @@ fmt: ## Format the code tidy: ## Tidy go.mod go mod tidy -completions: ## Generate shell completion scripts for packaging - @mkdir -p packaging/completions - go run $(CMD) completion bash > packaging/completions/cernbox.bash - go run $(CMD) completion zsh > packaging/completions/cernbox.zsh - go run $(CMD) completion fish > packaging/completions/cernbox.fish +completions: build ## Generate shell completion scripts for packaging + packaging/completions.sh ./$(BINARY) clean: ## Remove build and test artifacts rm -f $(BINARY) coverage.out coverage.html - rm -rf packaging/completions + rm -rf packaging/completions $(WS) dist dev-up: ## Start the dev environment (revad + EOS) $(COMPOSE) up -d --build diff --git a/packaging/completions.sh b/packaging/completions.sh new file mode 100755 index 0000000..21f7e25 --- /dev/null +++ b/packaging/completions.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Write the shell completion scripts of a cernbox binary into +# packaging/completions, for the packages. +# +# packaging/completions.sh [binary] +# +# Without a binary, it builds one for this machine from the prepared +# workspace (packaging/prepare.sh). The scripts are generated rather than +# committed, so they can never drift from the command tree they describe. +set -eu + +die() { printf 'completions: %s\n' "$*" >&2; exit 1; } + +root="$(cd "$(dirname "$0")/.." && pwd)" +WS="${WS:-$root/ws}" +out="$root/packaging/completions" + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT INT TERM + +binary="${1:-}" +if [ -z "$binary" ]; then + [ -f "$WS/bflags" ] || die "no workspace at $WS: run packaging/prepare.sh first" + binary="$tmp/cernbox" + (cd "$WS" && GOWORK=off CGO_ENABLED=0 go build -trimpath -ldflags "-s -w $(cat bflags)" -o "$binary" .) +fi + +# An empty home and config dir: the CLI reads the user config, and copies a +# cernbox-cli config.yaml it finds into config.toml, which a build has no +# business doing to the machine it runs on. +mkdir -p "$out" "$tmp/home" +for shell in bash zsh fish; do + HOME="$tmp/home" XDG_CONFIG_HOME="$tmp/home/.config" APPDATA="$tmp/home" \ + "$binary" completion "$shell" > "$out/cernbox.$shell" +done diff --git a/packaging/config.toml b/packaging/config.toml new file mode 100644 index 0000000..890080c --- /dev/null +++ b/packaging/config.toml @@ -0,0 +1,25 @@ +# Site configuration of the cernbox CLI, installed at /etc/cernbox/config.toml. +# +# The CERNBox defaults are built into the binary (the brand, cli/cernbox.toml +# in cernbox/reva-plugins): the endpoint, the sign-in chain with Kerberos and +# CERN SSO, the transfer settings. So on lxplus a user logs in, runs +# "cernbox ls", and it works, with this file empty. +# +# Set here what differs on this host. Per-user settings go in +# ~/.config/cernbox/config.toml, and win over this file. Flags and the +# environment win over both. An unknown key is an error, so that a typo does +# not go unseen: "cernbox config" shows what is in effect. + +[connection] +# The gRPC gateway of a CERNBox host. With a gateway, the default transport +# (auto) uses gRPC; without one, HTTPS to the endpoint. +#gateway = "localhost:9142" +# The Admin API, for operators. The local socket /run/reva/admin.sock is +# always tried first. +#admin_host = "localhost:9142" +#endpoint = "https://cernbox.cern.ch" +#transport = "auto" + +[transfer] +#chunk_size = "8M" +#archive = true diff --git a/packaging/config.yaml b/packaging/config.yaml deleted file mode 100644 index 6352528..0000000 --- a/packaging/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Site-wide configuration for the cernbox CLI, installed at /etc/cernbox/config.yaml. -# -# This file is what makes lxplus zero-configuration: a user logs in, runs -# "cernbox ls", and it works, because the endpoint and the SSO client are -# already set here. Per-user overrides go in ~/.config/cernbox/config.yaml. - -endpoint: https://cernbox.cern.ch - -auth: - kerberos: - # The ticket is presented straight to CERNBox, which verifies it against its - # own keytab. Any authenticated endpoint returns a token; this one also - # returns the user's identity, so the login costs a single request. - path: /graph/v1.0/me - - sso: - issuer: https://auth.cern.ch/auth/realms/cern - client_id: cernbox-cli - -transfer: - # Recursive downloads are served by the archiver as one stream when the - # server supports it, which is much faster for a directory of many small - # files. The client falls back to walking the tree if it cannot. - archive: true - chunk_size: 8M diff --git a/packaging/prepare.sh b/packaging/prepare.sh new file mode 100755 index 0000000..1feaf74 --- /dev/null +++ b/packaging/prepare.sh @@ -0,0 +1,93 @@ +#!/bin/sh +# Prepare the gaia workspace of the cernbox build, which goreleaser and +# "make build" then compile. +# +# cernbox is reva's CLI rebranded at build time: gaia writes a Go module with +# a main.go that imports reva's CLI and the CERNBox CLI plugins, and embeds the +# CERNBox brand. gaia also runs the brand and plugin check of reva +# (go run . __brandcheck), so a bad brand fails here and not on a user's +# machine. The workspace holds go.mod, go.sum, main.go, brand.toml, and bflags, +# the -X flags that set the version of the build. +# +# The versions come from packaging/versions.env. The environment overrides it: +# GAIA the gaia binary (default: gaia on PATH) +# WS the workspace (default: ws, at the root of the repo) +# REVA_VERSION reva, a Go module query +# REVA_PLUGINS_VERSION reva-plugins, a Go module query +# REVA_DIR a local reva checkout, instead of REVA_VERSION +# REVA_PLUGINS_DIR a local reva-plugins checkout, instead of +# REVA_PLUGINS_VERSION +# BRAND the brand file (default: cli/cernbox.toml of +# reva-plugins) +# +# A local checkout is for development. It writes its absolute path into the +# go.mod of the workspace, so a release uses published versions only. +set -eu + +die() { printf 'prepare: %s\n' "$*" >&2; exit 1; } + +root="$(cd "$(dirname "$0")/.." && pwd)" + +# versions.env sets what the environment has not set. +# shellcheck disable=SC2034 # value is read by the eval +while IFS='=' read -r key value; do + case "$key" in '' | \#*) continue ;; esac + eval "current=\${$key:-}" + [ -n "$current" ] || eval "$key=\$value" +done < "$root/packaging/versions.env" + +GAIA="${GAIA:-gaia}" +WS="${WS:-$root/ws}" +REVA_DIR="${REVA_DIR:-}" +REVA_PLUGINS_DIR="${REVA_PLUGINS_DIR:-}" +BRAND="${BRAND:-}" + +command -v "$GAIA" >/dev/null 2>&1 || + die "gaia is not installed: go install github.com/cs3org/gaia@$GAIA_VERSION, or set GAIA" + +plugins=github.com/cernbox/reva-plugins/cli +if [ -n "$REVA_PLUGINS_DIR" ]; then + with_plugins="$plugins=$REVA_PLUGINS_DIR/cli" + BRAND="${BRAND:-$REVA_PLUGINS_DIR/cli/cernbox.toml}" +else + with_plugins="$plugins@$REVA_PLUGINS_VERSION" + if [ -z "$BRAND" ]; then + # The brand is a file of the plugins module: take it from the module + # cache, at the version the build uses. Outside the repo, so that this + # does not touch its go.mod or go.sum. + dir="$(cd "$(mktemp -d)" && GOWORK=off GOFLAGS=-mod=mod \ + go mod download -json "github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION" | + sed -n 's/^[[:space:]]*"Dir": "\(.*\)",*$/\1/p')" || + die "cannot download github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION" + [ -n "$dir" ] || die "cannot find github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION in the module cache" + BRAND="$dir/cli/cernbox.toml" + fi +fi +[ -f "$BRAND" ] || die "no brand at $BRAND" + +# The arguments of gaia: a published reva is the version argument, a local +# one is a replacement. +set -- --only-prepare --workspace "$WS" --binary reva --brand "$BRAND" --with "$with_plugins" +if [ -n "$REVA_DIR" ]; then + set -- "$@" --with "github.com/cs3org/reva/v3=$REVA_DIR" +else + set -- "$REVA_VERSION" "$@" +fi + +# gaia runs "go mod init" and refuses a workspace that has a go.mod already, +# so an earlier workspace goes. Anything else in its place stays. +if [ -e "$WS" ] && [ -n "$(ls -A "$WS")" ] && ! { [ -f "$WS/go.mod" ] && [ -f "$WS/main.go" ]; }; then + die "$WS is not empty and is not a workspace: set WS to another directory" +fi +rm -rf "$WS" +mkdir -p "$WS" + +# gaia passes only some variables to go, and GOWORK is not one of them, so a +# go.work above the workspace would take over the build of the workspace. +gowork="$(cd "$WS" && env -u GOWORK go env GOWORK)" +[ -z "$gowork" ] || die "$WS is under $gowork: move the workspace (WS) out of that tree" + +"$GAIA" build "$@" + +[ -s "$WS/bflags" ] || die "gaia did not write $WS/bflags" +printf 'prepare: workspace %s\nprepare: brand %s\nprepare: bflags %s\n' "$WS" "$BRAND" "$(cat "$WS/bflags")" diff --git a/packaging/versions.env b/packaging/versions.env new file mode 100644 index 0000000..7582afc --- /dev/null +++ b/packaging/versions.env @@ -0,0 +1,14 @@ +# The versions that a release is built from. prepare.sh, the Makefile and the +# CI workflows read this file, so a release is one commit that changes it. +# +# cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and the +# CERNBox brand. Each line is a Go module query: a tag, a commit or a branch. +# +# master is a placeholder. None of the three has a release with what the +# build needs yet: gaia's --binary reva and --brand (cs3org/gaia#6), reva's +# CLI with the HTTP backend and branding (cs3org/reva#5959 to #5963), and the +# CLI plugins with cli/cernbox.toml (cernbox/reva-plugins#88 and T42). Pin +# each one to its release once it is published. +GAIA_VERSION=master +REVA_VERSION=master +REVA_PLUGINS_VERSION=master From 60e0647b34e40052bcdff5f1e3ff4b4b09f1d64d Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 10:35:43 +0100 Subject: [PATCH 2/6] CI: prepare the gaia workspace once, and build, test and release it One job prepares ws/ on Linux and uploads it. The builds on Linux, macOS and Windows, the integration tests and the release all compile that same workspace, so what is tested is what is released, and gaia's call to api.github.com, which it makes without a token and panics on, happens once per run, with one retry. The smoke test now also checks that the version flags and the brand made it into the binary. The integration tests run against the merged binary, so they fail until the suite is ported to its JSON and its exit codes. Assisted-by: Claude:claude-opus-5-5 --- .github/workflows/_build.yml | 91 ++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index c3076f5..e207b6d 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -52,10 +52,41 @@ jobs: version: latest install-mode: goinstall + # ── Prepare ────────────────────────────────────────────────────────────────── + # cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and brand. + # The workspace is prepared once, here, and every job after this one builds + # that same workspace, so what is tested is what is released. + prepare: + name: Prepare the gaia workspace + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Install gaia + run: | + . packaging/versions.env + go install "github.com/cs3org/gaia@$GAIA_VERSION" + + # gaia asks api.github.com for the commit of the reva version, without a + # token, and panics when the call fails. A shared runner can run out of + # the sixty calls an hour, so one retry after a pause. + - name: Prepare + run: packaging/prepare.sh || { sleep 60; packaging/prepare.sh; } + + - uses: actions/upload-artifact@v7 + with: + name: workspace + path: ws/ + if-no-files-found: error + # ── Build ──────────────────────────────────────────────────────────────────── build: name: Build (${{ matrix.os }}) - needs: [lint] + needs: [prepare] runs-on: ${{ matrix.os }} strategy: matrix: @@ -69,24 +100,32 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: ws/go.sum - name: Build - if: runner.os != 'Windows' - run: make build - - - name: Build (Windows) - if: runner.os == 'Windows' - run: go build -o cernbox.exe ./cmd/cernbox + working-directory: ws + env: + GOWORK: "off" + CGO_ENABLED: "0" + run: go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../cernbox${{ runner.os == 'Windows' && '.exe' || '' }} . # A binary that cannot print its own help is broken in a way no unit test # would catch: cobra panics on a duplicate flag shorthand at startup, not - # at compile time. + # at compile time. The version and the brand come from the ldflags and the + # embedded brand file, so a build that lost either shows here. - name: Smoke test run: | - ./cernbox version + ./cernbox version --plugins | tee version.txt + grep -q '^version=[^ ]' version.txt + grep -q '^brand=cernbox ' version.txt ./cernbox --help ./cernbox ls --help ./cernbox completion bash > /dev/null @@ -133,9 +172,25 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: | + go.sum + ws/go.sum + + # The suite drives the binary that the release ships. + - name: Build cernbox + working-directory: ws + env: + GOWORK: "off" + CGO_ENABLED: "0" + run: go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../cernbox . # The EOS image alone is over 6GB, and a runner has around 14GB free, so # the stack fills the disk and EOS then fails every write with "no space @@ -171,7 +226,7 @@ jobs: # as a pass. Here the realm is always present, so demand it. env: CERNBOX_REQUIRE_KERBEROS: "1" - run: go test -tags integration -count=1 -timeout 20m ./integration/... + run: CERNBOX_BINARY="$PWD/cernbox" go test -tags integration -count=1 -timeout 20m ./integration/... # The revad and EOS logs are the only way to tell a client bug from a # server one when a test fails, and the containers are gone by the time @@ -207,9 +262,19 @@ jobs: with: fetch-depth: 0 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: ws/go.sum + + # goreleaser puts these in the ldflags of every build (.goreleaser.yaml). + - name: Read the version flags of the workspace + run: echo "BFLAGS=$(cat ws/bflags)" >> "$GITHUB_ENV" - name: Move the nightly tag to this commit if: inputs.version == 'nightly' From 35d73461a74c1349b695efb61022106507614dc2 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 10:35:43 +0100 Subject: [PATCH 3/6] integration: run against the binary in CERNBOX_BINARY The binary that ships is built by gaia from reva, not from this repository. Without CERNBOX_BINARY the suite still builds cmd/cernbox, so that 1.x can be compared until its code is retired. Assisted-by: Claude:claude-opus-5-5 --- integration/helpers_test.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/integration/helpers_test.go b/integration/helpers_test.go index b6ec62c..a5d3f0c 100644 --- a/integration/helpers_test.go +++ b/integration/helpers_test.go @@ -64,11 +64,20 @@ func TestMain(m *testing.M) { } defer os.RemoveAll(dir) - binary = filepath.Join(dir, "cernbox") - build := exec.Command("go", "build", "-o", binary, "./cmd/cernbox") - build.Dir = repoRoot() - if out, err := build.CombinedOutput(); err != nil { - fmt.Fprintf(os.Stderr, "building cernbox: %v\n%s", err, out) + // CERNBOX_BINARY is the binary that the release ships: reva's CLI, built + // by gaia ("make build"). Without it, the suite builds this repository's + // own client, until that code is retired. + binary = os.Getenv("CERNBOX_BINARY") + if binary == "" { + binary = filepath.Join(dir, "cernbox") + build := exec.Command("go", "build", "-o", binary, "./cmd/cernbox") + build.Dir = repoRoot() + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building cernbox: %v\n%s", err, out) + os.Exit(1) + } + } else if !filepath.IsAbs(binary) { + fmt.Fprintf(os.Stderr, "CERNBOX_BINARY must be an absolute path, not %s\n", binary) os.Exit(1) } From c79a856730e184b9426fe7145644c30d5464b845 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 10:36:04 +0100 Subject: [PATCH 4/6] Document the packaging, and the plan to retire the client docs/packaging.md says how a release is built now (prepare, version flags, cross-compilation, packages), how to build locally against checkouts of reva and reva-plugins, how to release, what the CERNBox hosts need, and in which order the Go code of 1.x goes: port the integration suite, close the gaps of the merged CLI, release 2.0, then delete cmd/, internal/ and pkg/ in one commit. Assisted-by: Claude:claude-opus-5-5 --- docs/packaging.md | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 docs/packaging.md diff --git a/docs/packaging.md b/docs/packaging.md new file mode 100644 index 0000000..ce89c5a --- /dev/null +++ b/docs/packaging.md @@ -0,0 +1,130 @@ +# Packaging the cernbox CLI + +From 2.0, `cernbox` is reva's CLI (`cmd/reva`, `pkg/cli` in cs3org/reva), rebranded at build time. gaia builds it, +the same way as revad: + +``` +gaia build --binary reva --brand cernbox.toml --with github.com/cernbox/reva-plugins/cli -o cernbox +``` + +- `--binary reva` builds the CLI instead of the daemon; +- `--with github.com/cernbox/reva-plugins/cli` compiles in the CERNBox CLI plugins (`otg`, `user account`, CERN SSO, + `clipboard`, `inbox`, `outbox`); +- `--brand cernbox.toml` embeds the CERNBox brand: the program name, the endpoint, the sign-in chain, the + environment variables and the texts. The brand is `cli/cernbox.toml` in reva-plugins, next to the plugins it names. + +This repository no longer holds the client. It holds what turns that build into releases: the goreleaser +configuration, the packages, `install.sh`, the documentation, and the end-to-end tests with their dev environment. + +## The pipeline + +``` +packaging/versions.env ──> packaging/prepare.sh ──> ws/ ──> goreleaser ──> dist/ + GAIA_VERSION gaia build go.mod, go.sum linux, darwin, windows + REVA_VERSION --only-prepare main.go amd64, arm64 + REVA_PLUGINS_VERSION --binary reva brand.toml tar.gz, zip, rpm, deb + --brand …/cernbox.toml bflags checksums.txt + --with …/cli@version +``` + +1. **Prepare.** `packaging/prepare.sh` runs gaia's prepare stage into `ws/`. gaia writes a Go module whose `main.go` + imports reva's CLI and the plugins and embeds the brand, then runs `go run . __brandcheck` in it: reva checks the + brand strictly against the plugins of the same build (every sign-in method of the chain exists, every hidden group + exists) and checks the plugins. A bad brand fails here. The brand is read from the module cache, at the version of + reva-plugins that the build uses, so the brand and the plugins cannot come from two versions. +2. **Version.** gaia writes `ws/bflags`: the `-X` flags that set reva's `pkg/rversion` (version, commit, Go version, + build date). `cernbox version` prints them, and `cernbox version --plugins` adds the brand (with its SHA-256) and + every plugin with its module version. +3. **Build.** goreleaser cross-compiles `ws/` (`dir: ws`, `main: .`) with `CGO_ENABLED=0`, `-trimpath` and + `-ldflags "-s -w $BFLAGS"`, where `BFLAGS` is the content of `ws/bflags`. goreleaser reads its environment before + any hook runs, so prepare runs before goreleaser, not as a hook. `GOWORK=off` keeps a `go.work` above the + workspace out of the build. +4. **Package.** The same artefacts, names and install paths as 1.x: + - archives `cernbox-cli___.tar.gz` (`.zip` on Windows) with `cernbox` at the root, which + `install.sh` unpacks; + - packages `cernbox-cli` (RPM and deb): `/usr/bin/cernbox`, `/etc/cernbox/config.toml` (a config file, kept on + upgrade), and the bash, zsh and fish completions, generated from the binary by `packaging/completions.sh`; + - `checksums.txt`. + +The CI (`.github/workflows/_build.yml`) prepares the workspace once, on Linux, and hands it to the other jobs as an +artefact: the build and smoke test on Linux, macOS (arm64 and amd64) and Windows, the end-to-end tests, and the +release. What is tested is what is released. + +## Building locally + +```bash +go install github.com/cs3org/gaia@ +make prepare # ws/, from the published versions of packaging/versions.env +make build # ./cernbox for this machine +make snapshot # dist/, as a release, without publishing (needs goreleaser) +``` + +With local checkouts of reva and reva-plugins: + +```bash +make prepare REVA_DIR=~/reva REVA_PLUGINS_DIR=~/reva-plugins +make prepare REVA_DIR=~/reva REVA_PLUGINS_DIR=~/reva-plugins BRAND=/tmp/test.toml +``` + +A local checkout puts its absolute path into `ws/go.mod`, so a release uses published versions only. `GAIA` names +another gaia binary. The workspace must not be inside a tree with a `go.work`: gaia does not pass `GOWORK` to `go`, +and `prepare.sh` refuses such a workspace. + +Without goreleaser, the same build by hand: + +```bash +make prepare +cd ws && GOWORK=off CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 \ + go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../dist/darwin_arm64/cernbox . +``` + +Name the output: the module of the workspace is `reva`, so `go build` without `-o` writes `reva`. + +## Releasing + +1. Set the three versions in `packaging/versions.env` to published releases (tags, not branches). gaia looks the reva + version up on api.github.com to find its commit, and panics if that call fails: an unauthenticated runner shares + 60 calls an hour with its neighbours, so a failed prepare is worth one retry before anything else. +2. Update `docs/release-notes-.md` if the CLI changed in a way users see. +3. Tag `v`. The release workflow builds, runs the tests, and publishes. + +The version of the packages and of the archives is the tag of this repository. `cernbox version` prints the version +of reva, and `cernbox version --plugins` the versions of the plugins: together they say what a binary contains. + +## Installing on CERNBox hosts + +The package is `cernbox-cli`. It installs `/usr/bin/cernbox` and `/etc/cernbox/config.toml`. On a host, puppet +writes `/etc/cernbox/config.toml` with the local `gateway` and `admin_host`; the brand's `transport = "auto"` then +uses gRPC there. The admin socket (`/run/reva/admin.sock`) needs no configuration. + +Today the RPM is attached to the GitHub release (`https://github.com/cernbox/cernbox-cli/releases`). The CERNBox +hosts install from the koji tag `cernbox-ops` (`reva-cernbox`, `reva-cernbox-cli`), so the RPM also has to be +imported there. Note that the `ocis` hosts' manifest (`hg_box::include::ocis::reva::base`) has `cernbox-cli` set to +`ensure => absent`, for an older package of the same name: that has to go in the same change that installs this one. + +## Retiring the Go code of this repository + +The client code (`cmd/`, `internal/`, `pkg/`) is now in reva and reva-plugins. It stays here until the merged CLI +passes the end-to-end suite, so that the suite can still be run against 1.x for comparison +(`go test -tags integration ./integration/...` without `CERNBOX_BINARY` builds `cmd/cernbox`). Then, in this order: + +1. **Port the end-to-end suite** (`integration/`) to 2.0, against `CERNBOX_BINARY`: + - sign in once per account with `login --method basic` instead of passing `--method basic` to every command; + - read the CS3 JSON (release notes, section 1) in the helpers and the tests: `sweep`, the share, link, token and + trash tests, and every `json.Unmarshal` of a listing; + - check exit code 1 where the tests expect 3 to 6; + - follow the moved commands and flags (release notes, section 3); + - keep the dev environment (`dev/`): it is what the suite runs against. + The release workflow already runs the suite against the merged binary, so CI is red until this is done. +2. **Close the gaps** listed under "Not in this build yet" in the release notes, in reva and reva-plugins, or move + them into the notes as changes. +3. **Release 2.0** from this repository. +4. **Delete the client**, in one commit: `cmd/`, `internal/`, `pkg/`, `.golangci.yml`, and the unit-test and lint + jobs that cover them. `go.mod` stays, for `integration/` and `packaging/` (the tests of `install.sh`), with only + their dependencies. The `TestMain` fallback that builds `cmd/cernbox` goes with it. +5. **Rewrite the user documentation** for 2.0: `README.md`, `docs/configuration.md` (TOML, the `[plugins.*]` + sections, the environment variables of the brand) and `docs/behaviour.md` move to the CERNBox CLI docs, or are + rewritten here. `docs/design.md` is the design of 1.x, and stays as history. Until then, the unit tests of 1.x + check the YAML example in `docs/configuration.md`, so it changes together with the code. +6. **Update `CLAUDE.md`**: the repository has no client code, and its rules about the token cache, exit codes and + `pkg/client` belong to reva now. From 586e94bc8eef044cd73950a455f90fbf618a2642 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 10:36:04 +0100 Subject: [PATCH 5/6] Add the release notes of 2.0, and point the README at them 2.0 is reva's CLI under the old name, and it breaks scripts: the JSON is the CS3 JSON, the exit codes are 0, 1 and 2, some flags and commands moved, the configuration is TOML, and the token cache moved. The notes map each 1.x field, flag, command and variable to its 2.0 form, list the command names that reva and cernbox now share, and list what the build tested does not have yet. The README warns before anything else, builds from source with gaia, and shows the scripting example and exit codes of 2.0. The archives ship the release notes instead of the design of 1.x. Assisted-by: Claude:claude-opus-5-5 --- README.md | 25 ++-- docs/release-notes-2.0.md | 240 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 9 deletions(-) create mode 100644 docs/release-notes-2.0.md diff --git a/README.md b/README.md index ed67fd3..8cb894a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Work with your CERNBox files from the command line: browse them, move them between your computer and CERNBox, move them between two computers, and share them. +> **2.0 is a new program under the old name.** `cernbox` is now reva's command line client, built for CERNBox, and this repository packages it. The JSON output, the exit codes and some flags changed: read [docs/release-notes-2.0.md](docs/release-notes-2.0.md) before you upgrade a machine that runs scripts. The rest of this README, `docs/configuration.md` and `docs/behaviour.md` still describe 1.x, and are being rewritten. + ```console $ cernbox ls /eos/user/g/gdelmont $ cernbox cp ./report.pdf cb:/eos/user/g/gdelmont/Documents/ @@ -16,13 +18,13 @@ curl cli.cernbox.cern.ch | sh That fetches the release built for your system, checks it against the published checksum, and installs it where you can write — `/usr/local/bin` if that is yours, otherwise `~/.local/bin`, telling you what to add to `PATH`. It never asks for a password. `CERNBOX_VERSION` pins a version and `CERNBOX_INSTALL_DIR` chooses where it goes. -From source, or from a package: +From source, with [gaia](https://github.com/cs3org/gaia) and the versions in `packaging/versions.env`: ```bash -go install github.com/cernbox/cernbox-cli/cmd/cernbox@latest +make build # or: gaia build --binary reva --brand cernbox.toml --with github.com/cernbox/reva-plugins/cli -o cernbox ``` -RPM and deb packages are attached to each release. +RPM and deb packages (`cernbox-cli`) are attached to each release. On Windows, download `cernbox-cli__windows_amd64.zip` (or `_arm64`) from the [latest release](https://github.com/cernbox/cernbox-cli/releases/latest) and put `cernbox.exe` somewhere on your `PATH`. It runs in PowerShell, `cmd` and Windows Terminal alike. @@ -347,10 +349,10 @@ A trash listing covers a stretch of time rather than the whole bin, because that `--output json` and `--output csv` turn any listing into something a program can read, and informational messages go to standard error so a pipe sees only data. ```bash -cernbox --output json ls data | jq -r '.[] | select(.size > 1e9) | .name' +cernbox --output json ls data | jq -r '.[] | select((.size // "0" | tonumber) > 1e9) | .name' ``` -Exit codes distinguish the cases worth branching on: `0` success, `2` a mistake in the command, `3` a credentials problem, `4` not allowed, `5` not found. +The JSON is the CS3 JSON of each resource, in which 64-bit numbers are strings. Exit codes: `0` success, `1` an error, `2` a mistake in the command. ## Shell completion @@ -362,20 +364,25 @@ TAB then completes CERNBox paths as you type them, along with space names, share ## Development +The client is in [cs3org/reva](https://github.com/cs3org/reva) (`cmd/reva`, `pkg/cli`) and the CERNBox plugins and brand in [cernbox/reva-plugins](https://github.com/cernbox/reva-plugins) (`cli/`). This repository builds and ships them: see [docs/packaging.md](docs/packaging.md). + ```bash -make build # build -make test # unit tests +make prepare # the gaia workspace, ws/ (REVA_DIR, REVA_PLUGINS_DIR for local checkouts) +make build # ./cernbox for this machine +make snapshot # the release artefacts in dist/, with goreleaser make dev-up # start a local CERNBox to test against -make test-integration # run the tests that need it +make test-integration # run the end-to-end tests against ./cernbox ``` `make help` lists the rest. ## More +- [docs/release-notes-2.0.md](docs/release-notes-2.0.md): what changed in 2.0, and how to move scripts and configuration over +- [docs/packaging.md](docs/packaging.md): how a release is built, and the plan for the code of 1.x - [docs/configuration.md](docs/configuration.md) — the configuration file: where it goes, every setting, and the environment variables - [docs/behaviour.md](docs/behaviour.md) — what the client does when the server is awkward, and what each kind of transfer actually costs -- [docs/design.md](docs/design.md) — the architecture and the decisions behind it +- [docs/design.md](docs/design.md): the architecture of 1.x and the decisions behind it ## Licence diff --git a/docs/release-notes-2.0.md b/docs/release-notes-2.0.md new file mode 100644 index 0000000..e4a5370 --- /dev/null +++ b/docs/release-notes-2.0.md @@ -0,0 +1,240 @@ +# cernbox 2.0: release notes and migration guide + +`cernbox` 2.0 is a new program under the old name. It is reva's command line client, built for CERNBox: the same code +as the `reva` CLI, with the CERNBox commands compiled in and the CERNBox defaults built in. cernbox-cli 1.x, cernboxcop +and the stock `reva` CLI become one program, so users and operators run the same binary. + +What stays the same: + +- the name of the binary (`cernbox`, `cernbox.exe`), of the archives (`cernbox-cli___`) and of the + packages (`cernbox-cli`, RPM and deb); +- the install paths: `/usr/bin/cernbox`, and the shell completions in the same places; +- `curl cli.cernbox.cern.ch | sh`; +- the defaults: `https://cernbox.cern.ch`, Kerberos sign-in with a ticket, the device flow elsewhere, 8 MiB chunks, the + archiver for folders; +- the command names you know: `ls`, `cp` with `cb:`, `sync`, `share`, `link`, `trash`, `token`, `space info`, + `doctor`, `status`. + +What changes, in order of how likely it is to break a script: + +1. the JSON output is the CS3 JSON; +2. the exit codes are 0, 1 and 2; +3. some flags and commands moved; +4. the configuration file is TOML; +5. the token cache moves to the config directory. + +Check your scripts against each section before you upgrade a machine that runs them. + +## 1. JSON output is CS3 protojson + +`--output json` (`-o json`) now prints the CS3 message of each resource, in the standard protobuf JSON mapping. Field +names are lowerCamelCase, enums are their names, and 64-bit numbers are **strings**. A field with its zero value is +left out: an empty file has no `size`. + +### Files: `ls`, `stat`, `find` + +| 1.x | 2.0 | +|---|---| +| `name` | `name` | +| `path` | `path` | +| `size` (number) | `size` (string, absent when 0) | +| `is_dir` (bool) | `type`: `"RESOURCE_TYPE_CONTAINER"` or `"RESOURCE_TYPE_FILE"` | +| `modified` (RFC 3339) | `mtime.seconds` (string, Unix time) and `mtime.nanos` | +| `id` (one string) | `id.storageId`, `id.spaceId`, `id.opaqueId` | +| `etag` | `etag` | +| `mime_type` | `mimeType` | +| `permissions` (WebDAV letters) | `permissionSet` (one boolean per permission) | +| `checksums` (map) | `checksum.type` and `checksum.sum` | +| `web_url` | (none) | + +For example: + +```bash +# 1.x +cernbox -o json ls data | jq -r '.[] | select(.size > 1e9) | .name' +cernbox -o json ls data | jq -r '.[] | select(.is_dir) | .name' +# 2.0 +cernbox -o json ls data | jq -r '.[] | select((.size // "0" | tonumber) > 1e9) | .name' +cernbox -o json ls data | jq -r '.[] | select(.type == "RESOURCE_TYPE_CONTAINER") | .name' +``` + +### Shares: `share list`, `share create`, `share get` + +A share is a `cs3.sharing.collaboration.v1beta1.Share`: `id.opaqueId`, `resourceId`, `permissions`, `grantee`, +`owner`, `creator`, `ctime`, `mtime`, `expiration`, `description`. It replaces `{id, role, granted_to, link, +expires_at, path}`: + +| 1.x | 2.0 | +|---|---| +| `id` | `id.opaqueId` | +| `role` | `permissions.permissions` (the CS3 permissions; the role is no longer a field) | +| `granted_to.id`, `granted_to.type` | `grantee.userId` or `grantee.groupId`, and `grantee.type` | +| `expires_at` | `expiration.seconds` | +| `path` | `resourceId` (stat it for the path) | + +Received shares (`share received list`) are `ReceivedShare`: `share`, `state` (`SHARE_STATE_PENDING`, +`SHARE_STATE_ACCEPTED`, `SHARE_STATE_REJECTED`), `mountPoint`, `hidden`. + +Over HTTPS (laptops, and every machine without a gateway), the server does not return some fields, and they are +empty: the grantee's `idp`, the share's `ctime`, `mtime` and `creator`, and the received share's `mountPoint`. +`permissions` is derived from the role. + +### Public links: `link list`, `link create`, `link get` + +A link is a `cs3.sharing.link.v1beta1.PublicShare`: `id.opaqueId`, `token`, `resourceId`, `permissions`, +`passwordProtected`, `expiration`, `displayName`, `description`. 1.x's `link.url` and `link.has_password` become +`token` (the URL is the endpoint, `/s/`, and the token) and `passwordProtected`. + +### App tokens: `token list`, `token create` + +An app token is a `cs3.auth.applications.v1beta1.AppPassword`: `password`, `label`, `tokenScope`, `expiration`, +`ctime`, `utime`. **The id that `token revoke` takes is in `password`**: over HTTPS it is the token's client id, not +the secret. + +### Trash, spaces, users, versions + +- `trash list`: `RecycleItem`, with `key`, `ref.path`, `size`, `deletionTime.seconds` and `type`. +- `space list`, `space info`: `StorageSpace`. +- `user get`, `whoami`: `User`, with `id.opaqueId`, `id.idp`, `username`, `mail`, `displayName`. +- `versions list`: `FileVersion`. + +### Unchanged + +The statistics of `cp`, `sync` and `archive`, and the report of `doctor` (`.problems` included), keep their 1.x +shape. + +### Changed in shape, not CS3 + +- `version -o json`: `{version, commit, go_version, build_date}`. 1.x had `{version, commit, build_date, go, + platform}`. `version` is now the version of reva that the build is from; `version --plugins` adds the brand and the + CERNBox plugins with their versions. +- `du -o json`: `{path, bytes, dir}`. 1.x had `{path, size, listed, unlisted}`. + +## 2. Exit codes + +| | 1.x | 2.0 | +|---|---|---| +| success | 0 | 0 | +| an error | 1 | 1 | +| a mistake in the command (unknown flag, wrong arguments) | 2 | 2 | +| a credentials problem | 3 | 1 | +| not allowed | 4 | 1 | +| not found | 5 | 1 | +| conflict (lock, etag, quota) | 6 | 1 | + +A script that retried after `kinit` on exit code 3, or gave up on 4, cannot tell these apart by the exit code any +more. Read the message on standard error, or check before the call (`cernbox status`, `cernbox stat`). + +## 3. Commands and flags + +### Names: one set for `cernbox` and `reva` + +`cernbox` and `reva` are one program, so they have one set of names. The names are cernbox-cli's, so most of them do +not change for you. For people who used the `reva` CLI or cernboxcop, these are the renames: + +| What | `reva` (old) | `cernbox` 1.x | 2.0, both programs | +|---|---|---|---| +| public links | `public-share …` | `link …` | `link` | +| trash | `recycle list\|restore\|purge` (one key) | `trash list\|restore KEY…\|purge` | `trash`, several keys | +| app passwords | `app-password create\|list\|remove ` | `token create\|list\|revoke ID` | `token`, by id | +| transfer | `upload`, `download` | `cp SOURCE DEST` (`cb:` marks the remote side) | `cp` (there is no `get` or `put`) | +| sign-in | `login ` | `login [--method M]`, `logout`, `status` | `login [--method]`, `logout`, `status` | +| OCM | `ocm invite generate\|forward`, `ocm user find\|remove`, `ocm share …` | `ocm invite create\|list\|accept`, `ocm contacts`, `ocm providers`, `ocm received` | cernbox-cli's verbs; `ocm share …` and `ocm transfer …` stay, for operators | +| output | `-o text\|json` | `-o table\|json\|csv` | `-o table\|json\|csv` | +| server | `--host` | `--endpoint` | `--endpoint` (HTTPS), `--gateway` (gRPC); `--host` is gone | +| token input | `--token-file` | `--token`, `--app-token-file` | `--token`, `--app-token-file` | +| `--insecure` | no TLS on gRPC | plain HTTP allowed | plaintext allowed on the transport in use; `--skip-verify` for TLS without verification | +| completion | cobra's default | own `completion` | cernbox-cli's | +| quiet, debug | none | `-q`, `--debug`, `--no-progress` | `-q`, `--debug`, `--no-progress` | +| space detail | none | `space info ALIAS` | `space info ` | +| JSON | protojson | own structs | CS3 protojson (section 1) | + +### What moved for cernbox-cli users + +| 1.x | 2.0 | +|---|---| +| `--method M` before any command | `login --method M`. Commands sign in by themselves when the method needs no question (a ticket, an app token) | +| `--config FILE` | `CERNBOX_CONFIG=FILE` | +| `--kerberos-spn SPN` | `login --spn SPN`, `CERNBOX_KERBEROS_SPN`, or `cernbox config set kerberos-spn SPN` | +| `quota` | `space quota get` | +| `share received --accept ID` / `--decline ID` | `share received update ID --state accepted` / `--state rejected`; the list is `share received list` | +| `share create --expiry`, `share update --expiry` | `share update ID --expires DATE` | +| `link create --expiry`, `--name`, `--password` | `link update ID --expires DATE`, `--description`, `--password` | +| `link update --expiry`, `--name`, `--no-expiry` | `--expires DATE`, `--description`, `--expires none` | +| `link password PATH ID`, `--clear` | `link update ID --password`, `--no-password` | +| link role `upload` | `uploader` | +| `sync -r` | gone: `sync` copies folders without it | + +New in 2.0: `config view|set`, `lock get|set|remove`, `share get`, `link get`, `space update`, `space quota`, +`user get|find|groups`, `preferences get|set`, `ocm received get|update`, `elevate`, `version --plugins`. Global +flags: `--transport`, `--gateway`, `--admin-host`, `--as`, `--reason`, `--no-elevate`, `-y`/`--yes`. Commands for +operators show in `cernbox --help` only on a host with the admin socket or an admin host, or after `cernbox elevate`; +`cernbox help --all` lists them anyway. + +### Environment variables + +These keep working: `CERNBOX_CONFIG`, `CERNBOX_ENDPOINT`, `CERNBOX_TOKEN`, `CERNBOX_TOKEN_CACHE`, +`CERNBOX_APP_TOKEN`, `CERNBOX_PASSWORD`, `CERNBOX_USER` and `CERNBOX_USERNAME` (now one setting: the user of an app +token and of basic sign-in). + +New: `CERNBOX_TRANSPORT`, `CERNBOX_GATEWAY`, `CERNBOX_ADMIN_HOST`, `CERNBOX_APP_TOKEN_FILE`, +`CERNBOX_KERBEROS_SPN`, `CERNBOX_NO_ELEVATE`, `CERNBOX_OUTPUT`. + +Gone: `CERNBOX_AUTH_METHOD`, `CERNBOX_SSO_ISSUER`, `CERNBOX_SSO_CLIENT_ID`, `CERNBOX_JOBS` (use `-j`), +`CERNBOX_EDIT_FOLDER`, `CERNBOX_EDITOR` (use `VISUAL` or `EDITOR`), `CERNBOX_NO_TUI`. + +`cernbox` warns once when it sees a `REVA_*` variable: those belong to the `reva` build, and a script written for it +should not reach cernbox.cern.ch by accident. + +### Transports + +`cernbox` talks HTTPS to `https://cernbox.cern.ch`, as 1.x did. On a CERNBox host whose `/etc/cernbox/config.toml` +sets a `gateway`, it talks CS3 gRPC to that gateway instead. `--transport http|grpc|auto` chooses; `auto` is the +default. `status` shows which one is in use. + +## 4. Configuration + +The configuration files are TOML: + +| 1.x | 2.0 | +|---|---| +| `~/.config/cernbox/config.yaml` | `~/.config/cernbox/config.toml` | +| `/etc/cernbox/config.yaml` | `/etc/cernbox/config.toml` | + +- The first run of 2.0 copies your `config.yaml` into `config.toml`, and says so. It reads the YAML file only that + once. Settings that have no place in the new file are listed, and not copied. +- The site file `/etc/cernbox/config.yaml` is **not** read. The package installs `/etc/cernbox/config.toml`; on a host + where you changed the old file, move the changes over by hand. +- An unknown key is an error, not ignored, so that a typo shows. `cernbox config view` shows what is in effect. +- The outbox, inbox and editor settings live in `[plugins.]` sections, for example `[[plugins.outbox.folders]]`. + +## 5. Token cache + +1.x kept tokens in node-local `/tmp`, keyed by endpoint and principal. 2.0 keeps them in the config directory +(`~/.config/cernbox/token`), with mode 0600. The old cache is not read: sign in once after the upgrade (with a +Kerberos ticket, that happens by itself). `CERNBOX_TOKEN_CACHE` still chooses the file. On a machine whose home is +shared between nodes, set it to a node-local path if you do not want the token there. + +## 6. For maintainers of scripts and tests + +- Every consumer of `-o json` needs the changes of section 1, including cernbox-cli's own integration suite + (`integration/`), which parses the 1.x JSON throughout. +- Every branch on exit codes 3 to 6 needs section 2. +- The integration suite passes `--method basic` before every command, and 2.0 rejects that with exit code 2. Basic + sign-in is not in the CERNBox sign-in chain, so a script that uses it runs `cernbox login --method basic` first, + with the user in `CERNBOX_USERNAME` and the password in `CERNBOX_PASSWORD` or on stdin. + +## Not in this build yet + +Found on the build of reva 452dc98fd. Each item is either fixed before 2.0 is tagged, or stays in these notes as a +change: + +- `rm -r`, `rm -f`, `mkdir -p`, `mv --force`, `touch --no-create`, `logout --all` are not there; +- `ls` takes one path, and has `-l` and `--full` only: no `-a`, `-F`, `-h`, `-1`, `-R`, `-r`, `--sort`; +- `trash list --since|--from|--to`, and `--space` on the `trash` commands, are not there; `trash browse` is not + there; +- `du --all|--jobs|--versions`, `edit --editor|--force|--in|--interval|--no-watch`, `open --launch|--web`, + `ocm invite create --recipient`, `share create --with-remote` are not there; +- `apps`, `copy`, `paste`, `clipboard`, `inbox` and `outbox` are not there (the CERNBox plugins add the last four); +- a `config.yaml` with an outbox is copied into a `config.toml` that 2.0 then refuses (`unknown key + plugins.outbox.folders.after`), so every command fails until the file is fixed. From 036cb601d827e9045b720f5a064f74fabccbfa99 Mon Sep 17 00:00:00 2001 From: Gianmaria Del Monte Date: Tue, 27 Oct 2026 14:43:44 +0100 Subject: [PATCH 6/6] Trim the comments and simplify Assisted-by: Claude:claude-sonnet-5-5 --- .github/workflows/_build.yml | 17 +++++++---------- .goreleaser.yaml | 11 ++++------- Makefile | 9 +++------ integration/helpers_test.go | 5 ++--- packaging/completions.sh | 9 +++------ packaging/prepare.sh | 27 +++++++++------------------ packaging/versions.env | 15 +++++---------- 7 files changed, 33 insertions(+), 60 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index e207b6d..aacf175 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -52,10 +52,8 @@ jobs: version: latest install-mode: goinstall - # ── Prepare ────────────────────────────────────────────────────────────────── - # cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and brand. - # The workspace is prepared once, here, and every job after this one builds - # that same workspace, so what is tested is what is released. + # The workspace is prepared once, and every job after this one builds it, so + # what is tested is what is released. prepare: name: Prepare the gaia workspace runs-on: ubuntu-latest @@ -71,9 +69,8 @@ jobs: . packaging/versions.env go install "github.com/cs3org/gaia@$GAIA_VERSION" - # gaia asks api.github.com for the commit of the reva version, without a - # token, and panics when the call fails. A shared runner can run out of - # the sixty calls an hour, so one retry after a pause. + # gaia calls api.github.com without a token, and panics when the call + # fails. A shared runner can run out of its calls, so retry once. - name: Prepare run: packaging/prepare.sh || { sleep 60; packaging/prepare.sh; } @@ -119,8 +116,8 @@ jobs: # A binary that cannot print its own help is broken in a way no unit test # would catch: cobra panics on a duplicate flag shorthand at startup, not - # at compile time. The version and the brand come from the ldflags and the - # embedded brand file, so a build that lost either shows here. + # at compile time. The version and the brand check catch a build that + # lost the ldflags or the embedded brand. - name: Smoke test run: | ./cernbox version --plugins | tee version.txt @@ -184,7 +181,7 @@ jobs: go.sum ws/go.sum - # The suite drives the binary that the release ships. + # The suite tests the binary that the release ships. - name: Build cernbox working-directory: ws env: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 02c412f..c474e03 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,12 +2,9 @@ version: 2 project_name: cernbox-cli -# cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and the -# CERNBox brand. packaging/prepare.sh runs gaia's prepare stage into ws/, and -# goreleaser cross-compiles that workspace. BFLAGS is the content of -# ws/bflags, the -X flags of gaia that set the version of reva: goreleaser -# reads the environment before any hook runs, so prepare runs before -# goreleaser (make snapshot, and the release job), not as a hook here. +# goreleaser cross-compiles the gaia workspace in ws/. BFLAGS is the content +# of ws/bflags. goreleaser reads the environment before any hook runs, so +# prepare runs before goreleaser, not as a hook: # # packaging/prepare.sh # BFLAGS="$(cat ws/bflags)" goreleaser release --clean @@ -53,7 +50,7 @@ archives: - docs/release-notes-2.0.md # RPM for AlmaLinux, which is what lxplus runs. The site configuration is -# marked as a config file so a local edit (or puppet's) survives an upgrade. +# marked as a config file so a local edit survives an upgrade. nfpms: - id: packages ids: [cernbox] diff --git a/Makefile b/Makefile index cc85a98..0f1d374 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,9 @@ BINARY := cernbox WS := ws GORELEASER ?= goreleaser -# The build of cernbox is gaia's: packaging/prepare.sh writes the workspace -# ws/, with the versions of packaging/versions.env, and go or goreleaser -# compile it. For a local reva or reva-plugins, set REVA_DIR or -# REVA_PLUGINS_DIR (and BRAND for another brand file); GAIA names the gaia -# binary. They go to prepare.sh only, not exported: cernbox warns about any -# REVA_* variable it sees, so the tests should not see them. +# packaging/prepare.sh writes the gaia workspace ws/. For a local reva or +# reva-plugins, set REVA_DIR or REVA_PLUGINS_DIR. These go to prepare.sh only, +# not exported: cernbox warns about any REVA_* variable it sees. PREPARE := GAIA="$(GAIA)" BRAND="$(BRAND)" REVA_DIR="$(REVA_DIR)" REVA_PLUGINS_DIR="$(REVA_PLUGINS_DIR)" \ REVA_VERSION="$(REVA_VERSION)" REVA_PLUGINS_VERSION="$(REVA_PLUGINS_VERSION)" packaging/prepare.sh diff --git a/integration/helpers_test.go b/integration/helpers_test.go index a5d3f0c..fe97e20 100644 --- a/integration/helpers_test.go +++ b/integration/helpers_test.go @@ -64,9 +64,8 @@ func TestMain(m *testing.M) { } defer os.RemoveAll(dir) - // CERNBOX_BINARY is the binary that the release ships: reva's CLI, built - // by gaia ("make build"). Without it, the suite builds this repository's - // own client, until that code is retired. + // CERNBOX_BINARY is the binary to test ("make build"). Without it, the + // suite builds ./cmd/cernbox of this repository. binary = os.Getenv("CERNBOX_BINARY") if binary == "" { binary = filepath.Join(dir, "cernbox") diff --git a/packaging/completions.sh b/packaging/completions.sh index 21f7e25..ff53140 100755 --- a/packaging/completions.sh +++ b/packaging/completions.sh @@ -4,9 +4,7 @@ # # packaging/completions.sh [binary] # -# Without a binary, it builds one for this machine from the prepared -# workspace (packaging/prepare.sh). The scripts are generated rather than -# committed, so they can never drift from the command tree they describe. +# Without a binary, it builds one from the prepared workspace. set -eu die() { printf 'completions: %s\n' "$*" >&2; exit 1; } @@ -25,9 +23,8 @@ if [ -z "$binary" ]; then (cd "$WS" && GOWORK=off CGO_ENABLED=0 go build -trimpath -ldflags "-s -w $(cat bflags)" -o "$binary" .) fi -# An empty home and config dir: the CLI reads the user config, and copies a -# cernbox-cli config.yaml it finds into config.toml, which a build has no -# business doing to the machine it runs on. +# Use an empty home: the CLI copies a cernbox-cli config.yaml that it finds +# into config.toml, and a build must not do that to the machine. mkdir -p "$out" "$tmp/home" for shell in bash zsh fish; do HOME="$tmp/home" XDG_CONFIG_HOME="$tmp/home/.config" APPDATA="$tmp/home" \ diff --git a/packaging/prepare.sh b/packaging/prepare.sh index 1feaf74..5e8151a 100755 --- a/packaging/prepare.sh +++ b/packaging/prepare.sh @@ -1,13 +1,8 @@ #!/bin/sh # Prepare the gaia workspace of the cernbox build, which goreleaser and -# "make build" then compile. -# -# cernbox is reva's CLI rebranded at build time: gaia writes a Go module with -# a main.go that imports reva's CLI and the CERNBox CLI plugins, and embeds the -# CERNBox brand. gaia also runs the brand and plugin check of reva -# (go run . __brandcheck), so a bad brand fails here and not on a user's -# machine. The workspace holds go.mod, go.sum, main.go, brand.toml, and bflags, -# the -X flags that set the version of the build. +# "make build" then compile. cernbox is reva's CLI with the CERNBox CLI +# plugins and brand. gaia checks the brand here, so a bad brand fails in the +# build and not on a user's machine. bflags holds the -X flags of the version. # # The versions come from packaging/versions.env. The environment overrides it: # GAIA the gaia binary (default: gaia on PATH) @@ -20,15 +15,14 @@ # BRAND the brand file (default: cli/cernbox.toml of # reva-plugins) # -# A local checkout is for development. It writes its absolute path into the -# go.mod of the workspace, so a release uses published versions only. +# A local checkout writes its absolute path into the go.mod of the workspace: +# a release must use published versions only. set -eu die() { printf 'prepare: %s\n' "$*" >&2; exit 1; } root="$(cd "$(dirname "$0")/.." && pwd)" -# versions.env sets what the environment has not set. # shellcheck disable=SC2034 # value is read by the eval while IFS='=' read -r key value; do case "$key" in '' | \#*) continue ;; esac @@ -52,9 +46,8 @@ if [ -n "$REVA_PLUGINS_DIR" ]; then else with_plugins="$plugins@$REVA_PLUGINS_VERSION" if [ -z "$BRAND" ]; then - # The brand is a file of the plugins module: take it from the module - # cache, at the version the build uses. Outside the repo, so that this - # does not touch its go.mod or go.sum. + # Take the brand from the module cache, at the build version. Run + # outside the repo, so go.mod and go.sum stay as they are. dir="$(cd "$(mktemp -d)" && GOWORK=off GOFLAGS=-mod=mod \ go mod download -json "github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION" | sed -n 's/^[[:space:]]*"Dir": "\(.*\)",*$/\1/p')" || @@ -65,8 +58,6 @@ else fi [ -f "$BRAND" ] || die "no brand at $BRAND" -# The arguments of gaia: a published reva is the version argument, a local -# one is a replacement. set -- --only-prepare --workspace "$WS" --binary reva --brand "$BRAND" --with "$with_plugins" if [ -n "$REVA_DIR" ]; then set -- "$@" --with "github.com/cs3org/reva/v3=$REVA_DIR" @@ -82,8 +73,8 @@ fi rm -rf "$WS" mkdir -p "$WS" -# gaia passes only some variables to go, and GOWORK is not one of them, so a -# go.work above the workspace would take over the build of the workspace. +# gaia does not pass GOWORK to go: a go.work above the workspace would take +# over the build. gowork="$(cd "$WS" && env -u GOWORK go env GOWORK)" [ -z "$gowork" ] || die "$WS is under $gowork: move the workspace (WS) out of that tree" diff --git a/packaging/versions.env b/packaging/versions.env index 7582afc..41a1d79 100644 --- a/packaging/versions.env +++ b/packaging/versions.env @@ -1,14 +1,9 @@ -# The versions that a release is built from. prepare.sh, the Makefile and the -# CI workflows read this file, so a release is one commit that changes it. +# The versions that a release is built from. Each line is a Go module query: +# a tag, a commit or a branch. # -# cernbox is reva's CLI, built by gaia with the CERNBox CLI plugins and the -# CERNBox brand. Each line is a Go module query: a tag, a commit or a branch. -# -# master is a placeholder. None of the three has a release with what the -# build needs yet: gaia's --binary reva and --brand (cs3org/gaia#6), reva's -# CLI with the HTTP backend and branding (cs3org/reva#5959 to #5963), and the -# CLI plugins with cli/cernbox.toml (cernbox/reva-plugins#88 and T42). Pin -# each one to its release once it is published. +# master is a placeholder: none of the three has a release yet with what the +# build needs (gaia --binary reva and --brand, reva's CLI with branding, the +# CLI plugins with cli/cernbox.toml). Pin each one when it has a release. GAIA_VERSION=master REVA_VERSION=master REVA_PLUGINS_VERSION=master