diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index c3076f5..aacf175 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -52,10 +52,38 @@ jobs: version: latest install-mode: goinstall + # The workspace is prepared once, and every job after this one builds it, so + # what is tested is what is released. + prepare: + name: Prepare the gaia workspace + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Install gaia + run: | + . packaging/versions.env + go install "github.com/cs3org/gaia@$GAIA_VERSION" + + # gaia calls api.github.com without a token, and panics when the call + # fails. A shared runner can run out of its calls, so retry once. + - name: Prepare + run: packaging/prepare.sh || { sleep 60; packaging/prepare.sh; } + + - uses: actions/upload-artifact@v7 + with: + name: workspace + path: ws/ + if-no-files-found: error + # ── Build ──────────────────────────────────────────────────────────────────── build: name: Build (${{ matrix.os }}) - needs: [lint] + needs: [prepare] runs-on: ${{ matrix.os }} strategy: matrix: @@ -69,24 +97,32 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: ws/go.sum - name: Build - if: runner.os != 'Windows' - run: make build - - - name: Build (Windows) - if: runner.os == 'Windows' - run: go build -o cernbox.exe ./cmd/cernbox + working-directory: ws + env: + GOWORK: "off" + CGO_ENABLED: "0" + run: go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../cernbox${{ runner.os == 'Windows' && '.exe' || '' }} . # A binary that cannot print its own help is broken in a way no unit test # would catch: cobra panics on a duplicate flag shorthand at startup, not - # at compile time. + # at compile time. The version and the brand check catch a build that + # lost the ldflags or the embedded brand. - name: Smoke test run: | - ./cernbox version + ./cernbox version --plugins | tee version.txt + grep -q '^version=[^ ]' version.txt + grep -q '^brand=cernbox ' version.txt ./cernbox --help ./cernbox ls --help ./cernbox completion bash > /dev/null @@ -133,9 +169,25 @@ jobs: steps: - uses: actions/checkout@v6 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: | + go.sum + ws/go.sum + + # The suite tests the binary that the release ships. + - name: Build cernbox + working-directory: ws + env: + GOWORK: "off" + CGO_ENABLED: "0" + run: go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../cernbox . # The EOS image alone is over 6GB, and a runner has around 14GB free, so # the stack fills the disk and EOS then fails every write with "no space @@ -171,7 +223,7 @@ jobs: # as a pass. Here the realm is always present, so demand it. env: CERNBOX_REQUIRE_KERBEROS: "1" - run: go test -tags integration -count=1 -timeout 20m ./integration/... + run: CERNBOX_BINARY="$PWD/cernbox" go test -tags integration -count=1 -timeout 20m ./integration/... # The revad and EOS logs are the only way to tell a client bug from a # server one when a test fails, and the containers are gone by the time @@ -207,9 +259,19 @@ jobs: with: fetch-depth: 0 + - uses: actions/download-artifact@v7 + with: + name: workspace + path: ws + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + go-version-file: ws/go.mod + cache-dependency-path: ws/go.sum + + # goreleaser puts these in the ldflags of every build (.goreleaser.yaml). + - name: Read the version flags of the workspace + run: echo "BFLAGS=$(cat ws/bflags)" >> "$GITHUB_ENV" - name: Move the nightly tag to this commit if: inputs.version == 'nightly' diff --git a/.gitignore b/.gitignore index fc770b7..86c5588 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ /cernbox /dist/ +# The gaia workspace of the build (packaging/prepare.sh) +/ws/ + # Generated shell completions /packaging/completions/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b8f7c6b..c474e03 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -2,19 +2,27 @@ version: 2 project_name: cernbox-cli +# goreleaser cross-compiles the gaia workspace in ws/. BFLAGS is the content +# of ws/bflags. goreleaser reads the environment before any hook runs, so +# prepare runs before goreleaser, not as a hook: +# +# packaging/prepare.sh +# BFLAGS="$(cat ws/bflags)" goreleaser release --clean + +env: + # A go.work above ws/ would replace the module of the workspace. + - GOWORK=off + before: hooks: - - go mod tidy # Completions are generated rather than committed, so they can never drift # from the command tree they describe. - - mkdir -p packaging/completions - - sh -c "go run ./cmd/cernbox completion bash > packaging/completions/cernbox.bash" - - sh -c "go run ./cmd/cernbox completion zsh > packaging/completions/cernbox.zsh" - - sh -c "go run ./cmd/cernbox completion fish > packaging/completions/cernbox.fish" + - sh packaging/completions.sh builds: - id: cernbox - main: ./cmd/cernbox + dir: ws + main: . binary: cernbox # CGO off keeps the binary statically linkable, which is what lets one # artifact run across AlmaLinux versions and inside minimal containers. @@ -22,11 +30,10 @@ builds: - CGO_ENABLED=0 goos: [linux, darwin, windows] goarch: [amd64, arm64] + flags: + - -trimpath ldflags: - - -s -w - - -X main.version={{ .Version }} - - -X main.commit={{ .ShortCommit }} - - -X main.buildDate={{ .Date }} + - -s -w {{ .Env.BFLAGS }} archives: - id: default @@ -40,7 +47,7 @@ archives: files: - README.md - LICENSE - - docs/design.md + - docs/release-notes-2.0.md # RPM for AlmaLinux, which is what lxplus runs. The site configuration is # marked as a config file so a local edit survives an upgrade. @@ -58,8 +65,8 @@ nfpms: formats: [rpm, deb] bindir: /usr/bin contents: - - src: packaging/config.yaml - dst: /etc/cernbox/config.yaml + - src: packaging/config.toml + dst: /etc/cernbox/config.toml type: config|noreplace - src: packaging/completions/cernbox.bash dst: /usr/share/bash-completion/completions/cernbox diff --git a/Makefile b/Makefile index 6b28972..0f1d374 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,34 @@ BINARY := cernbox -CMD := ./cmd/cernbox -VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) -BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) -LDFLAGS := -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE) +WS := ws +GORELEASER ?= goreleaser + +# packaging/prepare.sh writes the gaia workspace ws/. For a local reva or +# reva-plugins, set REVA_DIR or REVA_PLUGINS_DIR. These go to prepare.sh only, +# not exported: cernbox warns about any REVA_* variable it sees. +PREPARE := GAIA="$(GAIA)" BRAND="$(BRAND)" REVA_DIR="$(REVA_DIR)" REVA_PLUGINS_DIR="$(REVA_PLUGINS_DIR)" \ + REVA_VERSION="$(REVA_VERSION)" REVA_PLUGINS_VERSION="$(REVA_PLUGINS_VERSION)" packaging/prepare.sh COMPOSE := docker compose -f dev/docker-compose.yaml -.PHONY: all build install test test-race test-cover test-integration test-all \ +.PHONY: all prepare build install snapshot test test-race test-cover test-integration test-all \ lint vet fmt tidy clean completions dev-up dev-down dev-logs help all: build -build: ## Build the cernbox binary - go build -ldflags "$(LDFLAGS)" -o $(BINARY) $(CMD) +prepare: ## Prepare the gaia workspace of cernbox (ws/) + $(PREPARE) + +$(WS)/bflags: + $(PREPARE) + +build: $(WS)/bflags ## Build cernbox for this machine from the workspace + cd $(WS) && GOWORK=off CGO_ENABLED=0 go build -trimpath -ldflags "-s -w $$(cat bflags)" -o ../$(BINARY) . + +install: build ## Install cernbox into GOBIN + install -m 0755 $(BINARY) "$$(go env GOBIN | grep . || echo "$$(go env GOPATH)/bin")/$(BINARY)" -install: ## Install the cernbox binary into GOBIN - go install -ldflags "$(LDFLAGS)" $(CMD) +snapshot: $(WS)/bflags ## Build the release artefacts into dist/, without publishing + BFLAGS="$$(cat $(WS)/bflags)" $(GORELEASER) release --snapshot --clean --skip=publish test: ## Run unit tests go test ./... @@ -29,8 +41,8 @@ test-cover: ## Run unit tests and write coverage.out / coverage.html go tool cover -html=coverage.out -o coverage.html @go tool cover -func=coverage.out | tail -1 -test-integration: build ## Run integration tests (requires: make dev-up) - go test -tags integration -count=1 -timeout 20m ./integration/... +test-integration: build ## Run integration tests against ./cernbox (requires: make dev-up) + CERNBOX_BINARY=$(CURDIR)/$(BINARY) go test -tags integration -count=1 -timeout 20m ./integration/... test-all: dev-up test test-integration ## Run every test suite @@ -47,15 +59,12 @@ fmt: ## Format the code tidy: ## Tidy go.mod go mod tidy -completions: ## Generate shell completion scripts for packaging - @mkdir -p packaging/completions - go run $(CMD) completion bash > packaging/completions/cernbox.bash - go run $(CMD) completion zsh > packaging/completions/cernbox.zsh - go run $(CMD) completion fish > packaging/completions/cernbox.fish +completions: build ## Generate shell completion scripts for packaging + packaging/completions.sh ./$(BINARY) clean: ## Remove build and test artifacts rm -f $(BINARY) coverage.out coverage.html - rm -rf packaging/completions + rm -rf packaging/completions $(WS) dist dev-up: ## Start the dev environment (revad + EOS) $(COMPOSE) up -d --build diff --git a/README.md b/README.md index ed67fd3..8cb894a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Work with your CERNBox files from the command line: browse them, move them between your computer and CERNBox, move them between two computers, and share them. +> **2.0 is a new program under the old name.** `cernbox` is now reva's command line client, built for CERNBox, and this repository packages it. The JSON output, the exit codes and some flags changed: read [docs/release-notes-2.0.md](docs/release-notes-2.0.md) before you upgrade a machine that runs scripts. The rest of this README, `docs/configuration.md` and `docs/behaviour.md` still describe 1.x, and are being rewritten. + ```console $ cernbox ls /eos/user/g/gdelmont $ cernbox cp ./report.pdf cb:/eos/user/g/gdelmont/Documents/ @@ -16,13 +18,13 @@ curl cli.cernbox.cern.ch | sh That fetches the release built for your system, checks it against the published checksum, and installs it where you can write — `/usr/local/bin` if that is yours, otherwise `~/.local/bin`, telling you what to add to `PATH`. It never asks for a password. `CERNBOX_VERSION` pins a version and `CERNBOX_INSTALL_DIR` chooses where it goes. -From source, or from a package: +From source, with [gaia](https://github.com/cs3org/gaia) and the versions in `packaging/versions.env`: ```bash -go install github.com/cernbox/cernbox-cli/cmd/cernbox@latest +make build # or: gaia build --binary reva --brand cernbox.toml --with github.com/cernbox/reva-plugins/cli -o cernbox ``` -RPM and deb packages are attached to each release. +RPM and deb packages (`cernbox-cli`) are attached to each release. On Windows, download `cernbox-cli__windows_amd64.zip` (or `_arm64`) from the [latest release](https://github.com/cernbox/cernbox-cli/releases/latest) and put `cernbox.exe` somewhere on your `PATH`. It runs in PowerShell, `cmd` and Windows Terminal alike. @@ -347,10 +349,10 @@ A trash listing covers a stretch of time rather than the whole bin, because that `--output json` and `--output csv` turn any listing into something a program can read, and informational messages go to standard error so a pipe sees only data. ```bash -cernbox --output json ls data | jq -r '.[] | select(.size > 1e9) | .name' +cernbox --output json ls data | jq -r '.[] | select((.size // "0" | tonumber) > 1e9) | .name' ``` -Exit codes distinguish the cases worth branching on: `0` success, `2` a mistake in the command, `3` a credentials problem, `4` not allowed, `5` not found. +The JSON is the CS3 JSON of each resource, in which 64-bit numbers are strings. Exit codes: `0` success, `1` an error, `2` a mistake in the command. ## Shell completion @@ -362,20 +364,25 @@ TAB then completes CERNBox paths as you type them, along with space names, share ## Development +The client is in [cs3org/reva](https://github.com/cs3org/reva) (`cmd/reva`, `pkg/cli`) and the CERNBox plugins and brand in [cernbox/reva-plugins](https://github.com/cernbox/reva-plugins) (`cli/`). This repository builds and ships them: see [docs/packaging.md](docs/packaging.md). + ```bash -make build # build -make test # unit tests +make prepare # the gaia workspace, ws/ (REVA_DIR, REVA_PLUGINS_DIR for local checkouts) +make build # ./cernbox for this machine +make snapshot # the release artefacts in dist/, with goreleaser make dev-up # start a local CERNBox to test against -make test-integration # run the tests that need it +make test-integration # run the end-to-end tests against ./cernbox ``` `make help` lists the rest. ## More +- [docs/release-notes-2.0.md](docs/release-notes-2.0.md): what changed in 2.0, and how to move scripts and configuration over +- [docs/packaging.md](docs/packaging.md): how a release is built, and the plan for the code of 1.x - [docs/configuration.md](docs/configuration.md) — the configuration file: where it goes, every setting, and the environment variables - [docs/behaviour.md](docs/behaviour.md) — what the client does when the server is awkward, and what each kind of transfer actually costs -- [docs/design.md](docs/design.md) — the architecture and the decisions behind it +- [docs/design.md](docs/design.md): the architecture of 1.x and the decisions behind it ## Licence diff --git a/docs/packaging.md b/docs/packaging.md new file mode 100644 index 0000000..ce89c5a --- /dev/null +++ b/docs/packaging.md @@ -0,0 +1,130 @@ +# Packaging the cernbox CLI + +From 2.0, `cernbox` is reva's CLI (`cmd/reva`, `pkg/cli` in cs3org/reva), rebranded at build time. gaia builds it, +the same way as revad: + +``` +gaia build --binary reva --brand cernbox.toml --with github.com/cernbox/reva-plugins/cli -o cernbox +``` + +- `--binary reva` builds the CLI instead of the daemon; +- `--with github.com/cernbox/reva-plugins/cli` compiles in the CERNBox CLI plugins (`otg`, `user account`, CERN SSO, + `clipboard`, `inbox`, `outbox`); +- `--brand cernbox.toml` embeds the CERNBox brand: the program name, the endpoint, the sign-in chain, the + environment variables and the texts. The brand is `cli/cernbox.toml` in reva-plugins, next to the plugins it names. + +This repository no longer holds the client. It holds what turns that build into releases: the goreleaser +configuration, the packages, `install.sh`, the documentation, and the end-to-end tests with their dev environment. + +## The pipeline + +``` +packaging/versions.env ──> packaging/prepare.sh ──> ws/ ──> goreleaser ──> dist/ + GAIA_VERSION gaia build go.mod, go.sum linux, darwin, windows + REVA_VERSION --only-prepare main.go amd64, arm64 + REVA_PLUGINS_VERSION --binary reva brand.toml tar.gz, zip, rpm, deb + --brand …/cernbox.toml bflags checksums.txt + --with …/cli@version +``` + +1. **Prepare.** `packaging/prepare.sh` runs gaia's prepare stage into `ws/`. gaia writes a Go module whose `main.go` + imports reva's CLI and the plugins and embeds the brand, then runs `go run . __brandcheck` in it: reva checks the + brand strictly against the plugins of the same build (every sign-in method of the chain exists, every hidden group + exists) and checks the plugins. A bad brand fails here. The brand is read from the module cache, at the version of + reva-plugins that the build uses, so the brand and the plugins cannot come from two versions. +2. **Version.** gaia writes `ws/bflags`: the `-X` flags that set reva's `pkg/rversion` (version, commit, Go version, + build date). `cernbox version` prints them, and `cernbox version --plugins` adds the brand (with its SHA-256) and + every plugin with its module version. +3. **Build.** goreleaser cross-compiles `ws/` (`dir: ws`, `main: .`) with `CGO_ENABLED=0`, `-trimpath` and + `-ldflags "-s -w $BFLAGS"`, where `BFLAGS` is the content of `ws/bflags`. goreleaser reads its environment before + any hook runs, so prepare runs before goreleaser, not as a hook. `GOWORK=off` keeps a `go.work` above the + workspace out of the build. +4. **Package.** The same artefacts, names and install paths as 1.x: + - archives `cernbox-cli___.tar.gz` (`.zip` on Windows) with `cernbox` at the root, which + `install.sh` unpacks; + - packages `cernbox-cli` (RPM and deb): `/usr/bin/cernbox`, `/etc/cernbox/config.toml` (a config file, kept on + upgrade), and the bash, zsh and fish completions, generated from the binary by `packaging/completions.sh`; + - `checksums.txt`. + +The CI (`.github/workflows/_build.yml`) prepares the workspace once, on Linux, and hands it to the other jobs as an +artefact: the build and smoke test on Linux, macOS (arm64 and amd64) and Windows, the end-to-end tests, and the +release. What is tested is what is released. + +## Building locally + +```bash +go install github.com/cs3org/gaia@ +make prepare # ws/, from the published versions of packaging/versions.env +make build # ./cernbox for this machine +make snapshot # dist/, as a release, without publishing (needs goreleaser) +``` + +With local checkouts of reva and reva-plugins: + +```bash +make prepare REVA_DIR=~/reva REVA_PLUGINS_DIR=~/reva-plugins +make prepare REVA_DIR=~/reva REVA_PLUGINS_DIR=~/reva-plugins BRAND=/tmp/test.toml +``` + +A local checkout puts its absolute path into `ws/go.mod`, so a release uses published versions only. `GAIA` names +another gaia binary. The workspace must not be inside a tree with a `go.work`: gaia does not pass `GOWORK` to `go`, +and `prepare.sh` refuses such a workspace. + +Without goreleaser, the same build by hand: + +```bash +make prepare +cd ws && GOWORK=off CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 \ + go build -trimpath -ldflags "-s -w $(cat bflags)" -o ../dist/darwin_arm64/cernbox . +``` + +Name the output: the module of the workspace is `reva`, so `go build` without `-o` writes `reva`. + +## Releasing + +1. Set the three versions in `packaging/versions.env` to published releases (tags, not branches). gaia looks the reva + version up on api.github.com to find its commit, and panics if that call fails: an unauthenticated runner shares + 60 calls an hour with its neighbours, so a failed prepare is worth one retry before anything else. +2. Update `docs/release-notes-.md` if the CLI changed in a way users see. +3. Tag `v`. The release workflow builds, runs the tests, and publishes. + +The version of the packages and of the archives is the tag of this repository. `cernbox version` prints the version +of reva, and `cernbox version --plugins` the versions of the plugins: together they say what a binary contains. + +## Installing on CERNBox hosts + +The package is `cernbox-cli`. It installs `/usr/bin/cernbox` and `/etc/cernbox/config.toml`. On a host, puppet +writes `/etc/cernbox/config.toml` with the local `gateway` and `admin_host`; the brand's `transport = "auto"` then +uses gRPC there. The admin socket (`/run/reva/admin.sock`) needs no configuration. + +Today the RPM is attached to the GitHub release (`https://github.com/cernbox/cernbox-cli/releases`). The CERNBox +hosts install from the koji tag `cernbox-ops` (`reva-cernbox`, `reva-cernbox-cli`), so the RPM also has to be +imported there. Note that the `ocis` hosts' manifest (`hg_box::include::ocis::reva::base`) has `cernbox-cli` set to +`ensure => absent`, for an older package of the same name: that has to go in the same change that installs this one. + +## Retiring the Go code of this repository + +The client code (`cmd/`, `internal/`, `pkg/`) is now in reva and reva-plugins. It stays here until the merged CLI +passes the end-to-end suite, so that the suite can still be run against 1.x for comparison +(`go test -tags integration ./integration/...` without `CERNBOX_BINARY` builds `cmd/cernbox`). Then, in this order: + +1. **Port the end-to-end suite** (`integration/`) to 2.0, against `CERNBOX_BINARY`: + - sign in once per account with `login --method basic` instead of passing `--method basic` to every command; + - read the CS3 JSON (release notes, section 1) in the helpers and the tests: `sweep`, the share, link, token and + trash tests, and every `json.Unmarshal` of a listing; + - check exit code 1 where the tests expect 3 to 6; + - follow the moved commands and flags (release notes, section 3); + - keep the dev environment (`dev/`): it is what the suite runs against. + The release workflow already runs the suite against the merged binary, so CI is red until this is done. +2. **Close the gaps** listed under "Not in this build yet" in the release notes, in reva and reva-plugins, or move + them into the notes as changes. +3. **Release 2.0** from this repository. +4. **Delete the client**, in one commit: `cmd/`, `internal/`, `pkg/`, `.golangci.yml`, and the unit-test and lint + jobs that cover them. `go.mod` stays, for `integration/` and `packaging/` (the tests of `install.sh`), with only + their dependencies. The `TestMain` fallback that builds `cmd/cernbox` goes with it. +5. **Rewrite the user documentation** for 2.0: `README.md`, `docs/configuration.md` (TOML, the `[plugins.*]` + sections, the environment variables of the brand) and `docs/behaviour.md` move to the CERNBox CLI docs, or are + rewritten here. `docs/design.md` is the design of 1.x, and stays as history. Until then, the unit tests of 1.x + check the YAML example in `docs/configuration.md`, so it changes together with the code. +6. **Update `CLAUDE.md`**: the repository has no client code, and its rules about the token cache, exit codes and + `pkg/client` belong to reva now. diff --git a/docs/release-notes-2.0.md b/docs/release-notes-2.0.md new file mode 100644 index 0000000..e4a5370 --- /dev/null +++ b/docs/release-notes-2.0.md @@ -0,0 +1,240 @@ +# cernbox 2.0: release notes and migration guide + +`cernbox` 2.0 is a new program under the old name. It is reva's command line client, built for CERNBox: the same code +as the `reva` CLI, with the CERNBox commands compiled in and the CERNBox defaults built in. cernbox-cli 1.x, cernboxcop +and the stock `reva` CLI become one program, so users and operators run the same binary. + +What stays the same: + +- the name of the binary (`cernbox`, `cernbox.exe`), of the archives (`cernbox-cli___`) and of the + packages (`cernbox-cli`, RPM and deb); +- the install paths: `/usr/bin/cernbox`, and the shell completions in the same places; +- `curl cli.cernbox.cern.ch | sh`; +- the defaults: `https://cernbox.cern.ch`, Kerberos sign-in with a ticket, the device flow elsewhere, 8 MiB chunks, the + archiver for folders; +- the command names you know: `ls`, `cp` with `cb:`, `sync`, `share`, `link`, `trash`, `token`, `space info`, + `doctor`, `status`. + +What changes, in order of how likely it is to break a script: + +1. the JSON output is the CS3 JSON; +2. the exit codes are 0, 1 and 2; +3. some flags and commands moved; +4. the configuration file is TOML; +5. the token cache moves to the config directory. + +Check your scripts against each section before you upgrade a machine that runs them. + +## 1. JSON output is CS3 protojson + +`--output json` (`-o json`) now prints the CS3 message of each resource, in the standard protobuf JSON mapping. Field +names are lowerCamelCase, enums are their names, and 64-bit numbers are **strings**. A field with its zero value is +left out: an empty file has no `size`. + +### Files: `ls`, `stat`, `find` + +| 1.x | 2.0 | +|---|---| +| `name` | `name` | +| `path` | `path` | +| `size` (number) | `size` (string, absent when 0) | +| `is_dir` (bool) | `type`: `"RESOURCE_TYPE_CONTAINER"` or `"RESOURCE_TYPE_FILE"` | +| `modified` (RFC 3339) | `mtime.seconds` (string, Unix time) and `mtime.nanos` | +| `id` (one string) | `id.storageId`, `id.spaceId`, `id.opaqueId` | +| `etag` | `etag` | +| `mime_type` | `mimeType` | +| `permissions` (WebDAV letters) | `permissionSet` (one boolean per permission) | +| `checksums` (map) | `checksum.type` and `checksum.sum` | +| `web_url` | (none) | + +For example: + +```bash +# 1.x +cernbox -o json ls data | jq -r '.[] | select(.size > 1e9) | .name' +cernbox -o json ls data | jq -r '.[] | select(.is_dir) | .name' +# 2.0 +cernbox -o json ls data | jq -r '.[] | select((.size // "0" | tonumber) > 1e9) | .name' +cernbox -o json ls data | jq -r '.[] | select(.type == "RESOURCE_TYPE_CONTAINER") | .name' +``` + +### Shares: `share list`, `share create`, `share get` + +A share is a `cs3.sharing.collaboration.v1beta1.Share`: `id.opaqueId`, `resourceId`, `permissions`, `grantee`, +`owner`, `creator`, `ctime`, `mtime`, `expiration`, `description`. It replaces `{id, role, granted_to, link, +expires_at, path}`: + +| 1.x | 2.0 | +|---|---| +| `id` | `id.opaqueId` | +| `role` | `permissions.permissions` (the CS3 permissions; the role is no longer a field) | +| `granted_to.id`, `granted_to.type` | `grantee.userId` or `grantee.groupId`, and `grantee.type` | +| `expires_at` | `expiration.seconds` | +| `path` | `resourceId` (stat it for the path) | + +Received shares (`share received list`) are `ReceivedShare`: `share`, `state` (`SHARE_STATE_PENDING`, +`SHARE_STATE_ACCEPTED`, `SHARE_STATE_REJECTED`), `mountPoint`, `hidden`. + +Over HTTPS (laptops, and every machine without a gateway), the server does not return some fields, and they are +empty: the grantee's `idp`, the share's `ctime`, `mtime` and `creator`, and the received share's `mountPoint`. +`permissions` is derived from the role. + +### Public links: `link list`, `link create`, `link get` + +A link is a `cs3.sharing.link.v1beta1.PublicShare`: `id.opaqueId`, `token`, `resourceId`, `permissions`, +`passwordProtected`, `expiration`, `displayName`, `description`. 1.x's `link.url` and `link.has_password` become +`token` (the URL is the endpoint, `/s/`, and the token) and `passwordProtected`. + +### App tokens: `token list`, `token create` + +An app token is a `cs3.auth.applications.v1beta1.AppPassword`: `password`, `label`, `tokenScope`, `expiration`, +`ctime`, `utime`. **The id that `token revoke` takes is in `password`**: over HTTPS it is the token's client id, not +the secret. + +### Trash, spaces, users, versions + +- `trash list`: `RecycleItem`, with `key`, `ref.path`, `size`, `deletionTime.seconds` and `type`. +- `space list`, `space info`: `StorageSpace`. +- `user get`, `whoami`: `User`, with `id.opaqueId`, `id.idp`, `username`, `mail`, `displayName`. +- `versions list`: `FileVersion`. + +### Unchanged + +The statistics of `cp`, `sync` and `archive`, and the report of `doctor` (`.problems` included), keep their 1.x +shape. + +### Changed in shape, not CS3 + +- `version -o json`: `{version, commit, go_version, build_date}`. 1.x had `{version, commit, build_date, go, + platform}`. `version` is now the version of reva that the build is from; `version --plugins` adds the brand and the + CERNBox plugins with their versions. +- `du -o json`: `{path, bytes, dir}`. 1.x had `{path, size, listed, unlisted}`. + +## 2. Exit codes + +| | 1.x | 2.0 | +|---|---|---| +| success | 0 | 0 | +| an error | 1 | 1 | +| a mistake in the command (unknown flag, wrong arguments) | 2 | 2 | +| a credentials problem | 3 | 1 | +| not allowed | 4 | 1 | +| not found | 5 | 1 | +| conflict (lock, etag, quota) | 6 | 1 | + +A script that retried after `kinit` on exit code 3, or gave up on 4, cannot tell these apart by the exit code any +more. Read the message on standard error, or check before the call (`cernbox status`, `cernbox stat`). + +## 3. Commands and flags + +### Names: one set for `cernbox` and `reva` + +`cernbox` and `reva` are one program, so they have one set of names. The names are cernbox-cli's, so most of them do +not change for you. For people who used the `reva` CLI or cernboxcop, these are the renames: + +| What | `reva` (old) | `cernbox` 1.x | 2.0, both programs | +|---|---|---|---| +| public links | `public-share …` | `link …` | `link` | +| trash | `recycle list\|restore\|purge` (one key) | `trash list\|restore KEY…\|purge` | `trash`, several keys | +| app passwords | `app-password create\|list\|remove ` | `token create\|list\|revoke ID` | `token`, by id | +| transfer | `upload`, `download` | `cp SOURCE DEST` (`cb:` marks the remote side) | `cp` (there is no `get` or `put`) | +| sign-in | `login ` | `login [--method M]`, `logout`, `status` | `login [--method]`, `logout`, `status` | +| OCM | `ocm invite generate\|forward`, `ocm user find\|remove`, `ocm share …` | `ocm invite create\|list\|accept`, `ocm contacts`, `ocm providers`, `ocm received` | cernbox-cli's verbs; `ocm share …` and `ocm transfer …` stay, for operators | +| output | `-o text\|json` | `-o table\|json\|csv` | `-o table\|json\|csv` | +| server | `--host` | `--endpoint` | `--endpoint` (HTTPS), `--gateway` (gRPC); `--host` is gone | +| token input | `--token-file` | `--token`, `--app-token-file` | `--token`, `--app-token-file` | +| `--insecure` | no TLS on gRPC | plain HTTP allowed | plaintext allowed on the transport in use; `--skip-verify` for TLS without verification | +| completion | cobra's default | own `completion` | cernbox-cli's | +| quiet, debug | none | `-q`, `--debug`, `--no-progress` | `-q`, `--debug`, `--no-progress` | +| space detail | none | `space info ALIAS` | `space info ` | +| JSON | protojson | own structs | CS3 protojson (section 1) | + +### What moved for cernbox-cli users + +| 1.x | 2.0 | +|---|---| +| `--method M` before any command | `login --method M`. Commands sign in by themselves when the method needs no question (a ticket, an app token) | +| `--config FILE` | `CERNBOX_CONFIG=FILE` | +| `--kerberos-spn SPN` | `login --spn SPN`, `CERNBOX_KERBEROS_SPN`, or `cernbox config set kerberos-spn SPN` | +| `quota` | `space quota get` | +| `share received --accept ID` / `--decline ID` | `share received update ID --state accepted` / `--state rejected`; the list is `share received list` | +| `share create --expiry`, `share update --expiry` | `share update ID --expires DATE` | +| `link create --expiry`, `--name`, `--password` | `link update ID --expires DATE`, `--description`, `--password` | +| `link update --expiry`, `--name`, `--no-expiry` | `--expires DATE`, `--description`, `--expires none` | +| `link password PATH ID`, `--clear` | `link update ID --password`, `--no-password` | +| link role `upload` | `uploader` | +| `sync -r` | gone: `sync` copies folders without it | + +New in 2.0: `config view|set`, `lock get|set|remove`, `share get`, `link get`, `space update`, `space quota`, +`user get|find|groups`, `preferences get|set`, `ocm received get|update`, `elevate`, `version --plugins`. Global +flags: `--transport`, `--gateway`, `--admin-host`, `--as`, `--reason`, `--no-elevate`, `-y`/`--yes`. Commands for +operators show in `cernbox --help` only on a host with the admin socket or an admin host, or after `cernbox elevate`; +`cernbox help --all` lists them anyway. + +### Environment variables + +These keep working: `CERNBOX_CONFIG`, `CERNBOX_ENDPOINT`, `CERNBOX_TOKEN`, `CERNBOX_TOKEN_CACHE`, +`CERNBOX_APP_TOKEN`, `CERNBOX_PASSWORD`, `CERNBOX_USER` and `CERNBOX_USERNAME` (now one setting: the user of an app +token and of basic sign-in). + +New: `CERNBOX_TRANSPORT`, `CERNBOX_GATEWAY`, `CERNBOX_ADMIN_HOST`, `CERNBOX_APP_TOKEN_FILE`, +`CERNBOX_KERBEROS_SPN`, `CERNBOX_NO_ELEVATE`, `CERNBOX_OUTPUT`. + +Gone: `CERNBOX_AUTH_METHOD`, `CERNBOX_SSO_ISSUER`, `CERNBOX_SSO_CLIENT_ID`, `CERNBOX_JOBS` (use `-j`), +`CERNBOX_EDIT_FOLDER`, `CERNBOX_EDITOR` (use `VISUAL` or `EDITOR`), `CERNBOX_NO_TUI`. + +`cernbox` warns once when it sees a `REVA_*` variable: those belong to the `reva` build, and a script written for it +should not reach cernbox.cern.ch by accident. + +### Transports + +`cernbox` talks HTTPS to `https://cernbox.cern.ch`, as 1.x did. On a CERNBox host whose `/etc/cernbox/config.toml` +sets a `gateway`, it talks CS3 gRPC to that gateway instead. `--transport http|grpc|auto` chooses; `auto` is the +default. `status` shows which one is in use. + +## 4. Configuration + +The configuration files are TOML: + +| 1.x | 2.0 | +|---|---| +| `~/.config/cernbox/config.yaml` | `~/.config/cernbox/config.toml` | +| `/etc/cernbox/config.yaml` | `/etc/cernbox/config.toml` | + +- The first run of 2.0 copies your `config.yaml` into `config.toml`, and says so. It reads the YAML file only that + once. Settings that have no place in the new file are listed, and not copied. +- The site file `/etc/cernbox/config.yaml` is **not** read. The package installs `/etc/cernbox/config.toml`; on a host + where you changed the old file, move the changes over by hand. +- An unknown key is an error, not ignored, so that a typo shows. `cernbox config view` shows what is in effect. +- The outbox, inbox and editor settings live in `[plugins.]` sections, for example `[[plugins.outbox.folders]]`. + +## 5. Token cache + +1.x kept tokens in node-local `/tmp`, keyed by endpoint and principal. 2.0 keeps them in the config directory +(`~/.config/cernbox/token`), with mode 0600. The old cache is not read: sign in once after the upgrade (with a +Kerberos ticket, that happens by itself). `CERNBOX_TOKEN_CACHE` still chooses the file. On a machine whose home is +shared between nodes, set it to a node-local path if you do not want the token there. + +## 6. For maintainers of scripts and tests + +- Every consumer of `-o json` needs the changes of section 1, including cernbox-cli's own integration suite + (`integration/`), which parses the 1.x JSON throughout. +- Every branch on exit codes 3 to 6 needs section 2. +- The integration suite passes `--method basic` before every command, and 2.0 rejects that with exit code 2. Basic + sign-in is not in the CERNBox sign-in chain, so a script that uses it runs `cernbox login --method basic` first, + with the user in `CERNBOX_USERNAME` and the password in `CERNBOX_PASSWORD` or on stdin. + +## Not in this build yet + +Found on the build of reva 452dc98fd. Each item is either fixed before 2.0 is tagged, or stays in these notes as a +change: + +- `rm -r`, `rm -f`, `mkdir -p`, `mv --force`, `touch --no-create`, `logout --all` are not there; +- `ls` takes one path, and has `-l` and `--full` only: no `-a`, `-F`, `-h`, `-1`, `-R`, `-r`, `--sort`; +- `trash list --since|--from|--to`, and `--space` on the `trash` commands, are not there; `trash browse` is not + there; +- `du --all|--jobs|--versions`, `edit --editor|--force|--in|--interval|--no-watch`, `open --launch|--web`, + `ocm invite create --recipient`, `share create --with-remote` are not there; +- `apps`, `copy`, `paste`, `clipboard`, `inbox` and `outbox` are not there (the CERNBox plugins add the last four); +- a `config.yaml` with an outbox is copied into a `config.toml` that 2.0 then refuses (`unknown key + plugins.outbox.folders.after`), so every command fails until the file is fixed. diff --git a/integration/helpers_test.go b/integration/helpers_test.go index b6ec62c..fe97e20 100644 --- a/integration/helpers_test.go +++ b/integration/helpers_test.go @@ -64,11 +64,19 @@ func TestMain(m *testing.M) { } defer os.RemoveAll(dir) - binary = filepath.Join(dir, "cernbox") - build := exec.Command("go", "build", "-o", binary, "./cmd/cernbox") - build.Dir = repoRoot() - if out, err := build.CombinedOutput(); err != nil { - fmt.Fprintf(os.Stderr, "building cernbox: %v\n%s", err, out) + // CERNBOX_BINARY is the binary to test ("make build"). Without it, the + // suite builds ./cmd/cernbox of this repository. + binary = os.Getenv("CERNBOX_BINARY") + if binary == "" { + binary = filepath.Join(dir, "cernbox") + build := exec.Command("go", "build", "-o", binary, "./cmd/cernbox") + build.Dir = repoRoot() + if out, err := build.CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "building cernbox: %v\n%s", err, out) + os.Exit(1) + } + } else if !filepath.IsAbs(binary) { + fmt.Fprintf(os.Stderr, "CERNBOX_BINARY must be an absolute path, not %s\n", binary) os.Exit(1) } diff --git a/packaging/completions.sh b/packaging/completions.sh new file mode 100755 index 0000000..ff53140 --- /dev/null +++ b/packaging/completions.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Write the shell completion scripts of a cernbox binary into +# packaging/completions, for the packages. +# +# packaging/completions.sh [binary] +# +# Without a binary, it builds one from the prepared workspace. +set -eu + +die() { printf 'completions: %s\n' "$*" >&2; exit 1; } + +root="$(cd "$(dirname "$0")/.." && pwd)" +WS="${WS:-$root/ws}" +out="$root/packaging/completions" + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT INT TERM + +binary="${1:-}" +if [ -z "$binary" ]; then + [ -f "$WS/bflags" ] || die "no workspace at $WS: run packaging/prepare.sh first" + binary="$tmp/cernbox" + (cd "$WS" && GOWORK=off CGO_ENABLED=0 go build -trimpath -ldflags "-s -w $(cat bflags)" -o "$binary" .) +fi + +# Use an empty home: the CLI copies a cernbox-cli config.yaml that it finds +# into config.toml, and a build must not do that to the machine. +mkdir -p "$out" "$tmp/home" +for shell in bash zsh fish; do + HOME="$tmp/home" XDG_CONFIG_HOME="$tmp/home/.config" APPDATA="$tmp/home" \ + "$binary" completion "$shell" > "$out/cernbox.$shell" +done diff --git a/packaging/config.toml b/packaging/config.toml new file mode 100644 index 0000000..890080c --- /dev/null +++ b/packaging/config.toml @@ -0,0 +1,25 @@ +# Site configuration of the cernbox CLI, installed at /etc/cernbox/config.toml. +# +# The CERNBox defaults are built into the binary (the brand, cli/cernbox.toml +# in cernbox/reva-plugins): the endpoint, the sign-in chain with Kerberos and +# CERN SSO, the transfer settings. So on lxplus a user logs in, runs +# "cernbox ls", and it works, with this file empty. +# +# Set here what differs on this host. Per-user settings go in +# ~/.config/cernbox/config.toml, and win over this file. Flags and the +# environment win over both. An unknown key is an error, so that a typo does +# not go unseen: "cernbox config" shows what is in effect. + +[connection] +# The gRPC gateway of a CERNBox host. With a gateway, the default transport +# (auto) uses gRPC; without one, HTTPS to the endpoint. +#gateway = "localhost:9142" +# The Admin API, for operators. The local socket /run/reva/admin.sock is +# always tried first. +#admin_host = "localhost:9142" +#endpoint = "https://cernbox.cern.ch" +#transport = "auto" + +[transfer] +#chunk_size = "8M" +#archive = true diff --git a/packaging/config.yaml b/packaging/config.yaml deleted file mode 100644 index 6352528..0000000 --- a/packaging/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Site-wide configuration for the cernbox CLI, installed at /etc/cernbox/config.yaml. -# -# This file is what makes lxplus zero-configuration: a user logs in, runs -# "cernbox ls", and it works, because the endpoint and the SSO client are -# already set here. Per-user overrides go in ~/.config/cernbox/config.yaml. - -endpoint: https://cernbox.cern.ch - -auth: - kerberos: - # The ticket is presented straight to CERNBox, which verifies it against its - # own keytab. Any authenticated endpoint returns a token; this one also - # returns the user's identity, so the login costs a single request. - path: /graph/v1.0/me - - sso: - issuer: https://auth.cern.ch/auth/realms/cern - client_id: cernbox-cli - -transfer: - # Recursive downloads are served by the archiver as one stream when the - # server supports it, which is much faster for a directory of many small - # files. The client falls back to walking the tree if it cannot. - archive: true - chunk_size: 8M diff --git a/packaging/prepare.sh b/packaging/prepare.sh new file mode 100755 index 0000000..5e8151a --- /dev/null +++ b/packaging/prepare.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# Prepare the gaia workspace of the cernbox build, which goreleaser and +# "make build" then compile. cernbox is reva's CLI with the CERNBox CLI +# plugins and brand. gaia checks the brand here, so a bad brand fails in the +# build and not on a user's machine. bflags holds the -X flags of the version. +# +# The versions come from packaging/versions.env. The environment overrides it: +# GAIA the gaia binary (default: gaia on PATH) +# WS the workspace (default: ws, at the root of the repo) +# REVA_VERSION reva, a Go module query +# REVA_PLUGINS_VERSION reva-plugins, a Go module query +# REVA_DIR a local reva checkout, instead of REVA_VERSION +# REVA_PLUGINS_DIR a local reva-plugins checkout, instead of +# REVA_PLUGINS_VERSION +# BRAND the brand file (default: cli/cernbox.toml of +# reva-plugins) +# +# A local checkout writes its absolute path into the go.mod of the workspace: +# a release must use published versions only. +set -eu + +die() { printf 'prepare: %s\n' "$*" >&2; exit 1; } + +root="$(cd "$(dirname "$0")/.." && pwd)" + +# shellcheck disable=SC2034 # value is read by the eval +while IFS='=' read -r key value; do + case "$key" in '' | \#*) continue ;; esac + eval "current=\${$key:-}" + [ -n "$current" ] || eval "$key=\$value" +done < "$root/packaging/versions.env" + +GAIA="${GAIA:-gaia}" +WS="${WS:-$root/ws}" +REVA_DIR="${REVA_DIR:-}" +REVA_PLUGINS_DIR="${REVA_PLUGINS_DIR:-}" +BRAND="${BRAND:-}" + +command -v "$GAIA" >/dev/null 2>&1 || + die "gaia is not installed: go install github.com/cs3org/gaia@$GAIA_VERSION, or set GAIA" + +plugins=github.com/cernbox/reva-plugins/cli +if [ -n "$REVA_PLUGINS_DIR" ]; then + with_plugins="$plugins=$REVA_PLUGINS_DIR/cli" + BRAND="${BRAND:-$REVA_PLUGINS_DIR/cli/cernbox.toml}" +else + with_plugins="$plugins@$REVA_PLUGINS_VERSION" + if [ -z "$BRAND" ]; then + # Take the brand from the module cache, at the build version. Run + # outside the repo, so go.mod and go.sum stay as they are. + dir="$(cd "$(mktemp -d)" && GOWORK=off GOFLAGS=-mod=mod \ + go mod download -json "github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION" | + sed -n 's/^[[:space:]]*"Dir": "\(.*\)",*$/\1/p')" || + die "cannot download github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION" + [ -n "$dir" ] || die "cannot find github.com/cernbox/reva-plugins@$REVA_PLUGINS_VERSION in the module cache" + BRAND="$dir/cli/cernbox.toml" + fi +fi +[ -f "$BRAND" ] || die "no brand at $BRAND" + +set -- --only-prepare --workspace "$WS" --binary reva --brand "$BRAND" --with "$with_plugins" +if [ -n "$REVA_DIR" ]; then + set -- "$@" --with "github.com/cs3org/reva/v3=$REVA_DIR" +else + set -- "$REVA_VERSION" "$@" +fi + +# gaia runs "go mod init" and refuses a workspace that has a go.mod already, +# so an earlier workspace goes. Anything else in its place stays. +if [ -e "$WS" ] && [ -n "$(ls -A "$WS")" ] && ! { [ -f "$WS/go.mod" ] && [ -f "$WS/main.go" ]; }; then + die "$WS is not empty and is not a workspace: set WS to another directory" +fi +rm -rf "$WS" +mkdir -p "$WS" + +# gaia does not pass GOWORK to go: a go.work above the workspace would take +# over the build. +gowork="$(cd "$WS" && env -u GOWORK go env GOWORK)" +[ -z "$gowork" ] || die "$WS is under $gowork: move the workspace (WS) out of that tree" + +"$GAIA" build "$@" + +[ -s "$WS/bflags" ] || die "gaia did not write $WS/bflags" +printf 'prepare: workspace %s\nprepare: brand %s\nprepare: bflags %s\n' "$WS" "$BRAND" "$(cat "$WS/bflags")" diff --git a/packaging/versions.env b/packaging/versions.env new file mode 100644 index 0000000..41a1d79 --- /dev/null +++ b/packaging/versions.env @@ -0,0 +1,9 @@ +# The versions that a release is built from. Each line is a Go module query: +# a tag, a commit or a branch. +# +# master is a placeholder: none of the three has a release yet with what the +# build needs (gaia --binary reva and --brand, reva's CLI with branding, the +# CLI plugins with cli/cernbox.toml). Pin each one when it has a release. +GAIA_VERSION=master +REVA_VERSION=master +REVA_PLUGINS_VERSION=master