From 2111a27926bf3b7b186001f28a1778c2cf3d631e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 12 Oct 2026 11:52:49 +0900 Subject: [PATCH 1/3] tufclient: Support bare signed targets files as repositories A build publishes its own signed targets_.json next to its files, but a client could only consume full TUF repositories with snapshot and timestamp metadata. Reading one build's artifacts, as the CVE scanner does, therefore required a whole repository per build. A repository can now be configured with targets_url pointing at such a signed targets file, which is verified against the targets keys of the trusted root. It has no freshness guarantees, which is acceptable for a build's own listing. A Repo type covers both kinds, and quarry-client uses it for refreshing, expiry and target lookups. --- cmd/quarry-client/http.go | 20 +-- cmd/quarry-client/refresh.go | 2 +- cmd/quarry-client/utils_tuf.go | 23 +--- internal/tufclient/client.go | 159 +++++++++++++++++++---- internal/tufclient/client_bare_test.go | 140 ++++++++++++++++++++ internal/tufclient/config/config.go | 53 +++++++- internal/tufclient/config/config_test.go | 46 +++++++ 7 files changed, 381 insertions(+), 62 deletions(-) create mode 100644 internal/tufclient/client_bare_test.go diff --git a/cmd/quarry-client/http.go b/cmd/quarry-client/http.go index 291e889..629c980 100644 --- a/cmd/quarry-client/http.go +++ b/cmd/quarry-client/http.go @@ -86,24 +86,14 @@ func serveSHA256SUMS(rw http.ResponseWriter, req *http.Request) (Err error) { errs []error expiry time.Time ) - for repoName, updater := range client.IterRepos(ctx) { + for _, updater := range client.IterRepos(ctx) { // Construct BEST-BEFORE-YYYY-MM-DD based on the earliest timestamp // expiry in any of the enabled repositories. - meta := updater.GetTrustedMetadataSet() - if meta.Timestamp == nil { - // FIXME: The local client TrustedMetadata state does not get - // filled until we do a refresh but go-tuf's client does not - // allow Refresh on the same updater more than once(?!). So we - // do a refresh here opportunistically. - // TODO: Add a (*Client).Refresh helper to make this much less - // fragile. - if err := updater.Refresh(); err != nil { - err := fmt.Errorf("refresh repo %s: %w", repoName, err) - errs = append(errs, err) - } - meta = updater.GetTrustedMetadataSet() + timestampExpiry, err := updater.Expiry(ctx) + if err != nil { + errs = append(errs, err) + continue } - timestampExpiry := meta.Timestamp.Signed.Expires if expiry.IsZero() || expiry.After(timestampExpiry) { // The BEST-BEFORE-* format only has day-resolution, so round to // the next day so that we never indicate that a repository is diff --git a/cmd/quarry-client/refresh.go b/cmd/quarry-client/refresh.go index 32d6412..b9da641 100644 --- a/cmd/quarry-client/refresh.go +++ b/cmd/quarry-client/refresh.go @@ -40,7 +40,7 @@ var refreshCommand = &cli.Command{ mustFprintf(wtr, "Refreshing %s ...", repoName) _ = wtr.Sync() - if err := updater.Refresh(); err != nil { + if err := updater.Refresh(ctx); err != nil { mustFprintf(wtr, " FAILED: %v\n", err) errs = append(errs, err) } else { diff --git a/cmd/quarry-client/utils_tuf.go b/cmd/quarry-client/utils_tuf.go index 40fce1c..a168e5d 100644 --- a/cmd/quarry-client/utils_tuf.go +++ b/cmd/quarry-client/utils_tuf.go @@ -50,28 +50,19 @@ func getClient(ctx context.Context, repoNames ...string) (_ *tufclient.Client, E // earliestTimestampExpiry returns the earliest timestamp.json expiry of any of // the repositories in the client, which bounds how long any listing generated // from those repositories can be trusted for. The zero time is returned if the -// client has no repositories. +// client has no repositories. For bare targets repositories, the targets.json +// expiry is used instead. func earliestTimestampExpiry(ctx context.Context, client *tufclient.Client) (time.Time, error) { var ( errs []error expiry time.Time ) - for repoName, updater := range client.IterRepos(ctx) { - meta := updater.GetTrustedMetadataSet() - if meta.Timestamp == nil { - // FIXME: The local client TrustedMetadata state does not get filled - // until we do a refresh but go-tuf's client does not allow Refresh - // on the same updater more than once(?!). So we do a refresh here - // opportunistically. - // TODO: Add a (*Client).Refresh helper to make this much less - // fragile. - if err := updater.Refresh(); err != nil { - errs = append(errs, fmt.Errorf("refresh repo %s: %w", repoName, err)) - continue - } - meta = updater.GetTrustedMetadataSet() + for _, updater := range client.IterRepos(ctx) { + timestampExpiry, err := updater.Expiry(ctx) + if err != nil { + errs = append(errs, err) + continue } - timestampExpiry := meta.Timestamp.Signed.Expires if expiry.IsZero() || expiry.After(timestampExpiry) { expiry = timestampExpiry } diff --git a/internal/tufclient/client.go b/internal/tufclient/client.go index e4fd5e7..d2d06d0 100644 --- a/internal/tufclient/client.go +++ b/internal/tufclient/client.go @@ -144,6 +144,125 @@ func RepoClient(ctx context.Context, cacheDir *pathrs.Root, repo *config.Reposit return tufupdater.New(tufConfig) } +// Repo is one repository loaded into a [Client]. It is either a full TUF +// repository driven by a go-tuf updater or a bare signed targets file (see +// [config.Repository.TargetsURL]). A bare targets repository only uses the +// updater for its trusted root and reference time. +type Repo struct { + repo *config.Repository + cacheDir *pathrs.Root + updater *tufupdater.Updater + + // targets is the verified targets.json of a bare targets repository. It is + // nil until the repository has been refreshed. + targets *tufext.SignedTargets +} + +// Refresh loads the repository's trusted metadata from its source. A go-tuf +// backed repository can only be refreshed once. +func (r *Repo) Refresh(ctx context.Context) error { + var err error + if r.repo.IsBareTargets() { + err = r.refreshBareTargets(ctx) + } else { + err = r.updater.Refresh() + } + if err != nil { + return fmt.Errorf("refresh repo %s: %w", r.repo.Name, err) + } + return nil +} + +// refreshBareTargets fetches the targets.json of a bare targets repository and +// verifies it against the targets keys of the trusted root. Without snapshot +// and timestamp there is no protection against freeze, rollback or +// mix-and-match attacks, which is acceptable for a build's own listing of its +// artifacts. +func (r *Repo) refreshBareTargets(ctx context.Context) error { + targetsURL := r.repo.TargetsURL.String() + data, err := r.repo.FetchTargets(ctx) + if err != nil { + return err + } + targets, err := jsonutils.Parse[*tufext.SignedTargets](data) + if err != nil { + return fmt.Errorf("%s is invalid JSON: %w", targetsURL, err) + } + if err := tufext.CheckMetadataType(tufmetadata.TARGETS, targets); err != nil { + return fmt.Errorf("%s is invalid targets.json: %w", targetsURL, err) + } + meta := r.updater.GetTrustedMetadataSet() + if err := meta.Root.VerifyDelegate(tufmetadata.TARGETS, targets); err != nil { + return fmt.Errorf("%s is not signed by the trusted root's targets keys: %w", targetsURL, err) + } + if targets.Signed.Delegations != nil { + return fmt.Errorf("%s has delegations, which bare targets repositories do not support", targetsURL) + } + if targets.Signed.IsExpired(meta.RefTime) { + return fmt.Errorf("%s expired at %s", targetsURL, targets.Signed.Expires.Format(time.RFC3339)) + } + r.targets = targets + return nil +} + +// ensure refreshes the repository if it has not been loaded yet. +func (r *Repo) ensure(ctx context.Context) error { + // FIXME: The local client TrustedMetadata state does not get filled until + // we do a refresh but go-tuf's client does not allow Refresh on the same + // updater more than once(?!). So we do a refresh here opportunistically. + // TODO: Add a (*Client).Refresh helper to make this much less fragile. + // Check targets like go-tuf's GetTargetInfo, so that a partially failed + // refresh is retried instead of leaving Snapshot nil. Only one of the two + // is ever set, depending on the kind of repository. + if r.targets != nil || r.updater.GetTrustedMetadataSet().Targets[tufmetadata.TARGETS] != nil { + return nil + } + return r.Refresh(ctx) +} + +// Expiry returns when the loaded metadata expires and loads it first if +// needed. This is the timestamp.json expiry of a full repository or the +// targets.json expiry of a bare one. +func (r *Repo) Expiry(ctx context.Context) (time.Time, error) { + if err := r.ensure(ctx); err != nil { + return time.Time{}, err + } + if r.repo.IsBareTargets() { + return r.targets.Signed.Expires, nil + } + return r.updater.GetTrustedMetadataSet().Timestamp.Signed.Expires, nil +} + +// targetsFetcher returns the fetcher [tufext.IterTargetFiles] walks the +// repository with. +func (r *Repo) targetsFetcher(ctx context.Context) (tufext.TargetMetadataFetchFunc, error) { + if err := r.ensure(ctx); err != nil { + return nil, err + } + if r.repo.IsBareTargets() { + return tufext.TargetsMapFetcher(map[string]*tufext.SignedTargets{tufmetadata.TARGETS: r.targets}), nil + } + meta := r.updater.GetTrustedMetadataSet() + return trustedMetadataTargetsFetcher(r.cacheDir, r.repo, &meta), nil +} + +// getTargetInfo returns the metadata of one target, or go-tuf's "target +// not found" error. +func (r *Repo) getTargetInfo(ctx context.Context, targetPath string) (*tufmetadata.TargetFiles, error) { + if !r.repo.IsBareTargets() { + return r.updater.GetTargetInfo(targetPath) + } + if err := r.ensure(ctx); err != nil { + return nil, err + } + target, ok := r.targets.Signed.Targets[targetPath] + if !ok { + // This is the error go-tuf returns, which GetTargetInfo checks for. + return nil, fmt.Errorf("target %s not found", targetPath) + } + return target, nil +} + // Client represents a Quarry client. type Client struct { // Config is a copy of the configuration file used to instantiate this @@ -159,7 +278,7 @@ type Client struct { // updaters contains the TUF updaters for every repository that was // successfully loaded by [NewClient], regardless of the subset currently // selected with [Client.WithRepos] (activeRepos). - updaters map[string]*tufupdater.Updater + updaters map[string]*Repo // activeRepos contains the TUF updaters for the active set of repositories // that all operations act on. By default this is every loaded repository @@ -209,10 +328,8 @@ func (client *Client) WithRepos(repoNames ...string) error { // repository order index (and name as a tie-breaker). Note that use of this // operation directly is very rarely necessary, most of the time // [GetTargetInfo] and [FetchTargetFile] are more ergonomic. -// -// TODO: Return some custom type? -func (client *Client) IterRepos(_ context.Context) iter.Seq2[string, *tufupdater.Updater] { - return func(yield func(string, *tufupdater.Updater) bool) { +func (client *Client) IterRepos(_ context.Context) iter.Seq2[string, *Repo] { + return func(yield func(string, *Repo) bool) { repoIndex := func(name string) int64 { return client.Config.Repos[name].OrderIndex() } @@ -242,8 +359,8 @@ func (client *Client) Close() error { // opens you up to freeze attacks, so this should only ever be used for testing // purposes. func (client *Client) SetRefTime(ctx context.Context, refTime time.Time) { - for _, updater := range client.IterRepos(ctx) { - updater.UnsafeSetRefTime(refTime) + for _, repo := range client.IterRepos(ctx) { + repo.updater.UnsafeSetRefTime(refTime) } } @@ -302,7 +419,7 @@ func (info *TargetInfo) Fetch(ctx context.Context) (io.ReadCloser, error) { // found. func (client *Client) GetTargetInfo(ctx context.Context, targetPath string) (*TargetInfo, error) { for repoName, updater := range client.IterRepos(ctx) { - info, err := updater.GetTargetInfo(targetPath) + info, err := updater.getTargetInfo(ctx, targetPath) if err != nil { // FIXME: Grrr, why don't they use wrapped errors for this?! if err.Error() == fmt.Sprintf("target %s not found", targetPath) { @@ -336,8 +453,8 @@ func (client *Client) FetchTargetFile(ctx context.Context, targetPath string) (i } // trustedMetadataTargetsFetcher returns a [tufext.TargetMetadataFetchFunc] for -// the given repository in the client. -func (client *Client) trustedMetadataTargetsFetcher(repo *config.Repository, metadata *tuftrustedmetadata.TrustedMetadata) tufext.TargetMetadataFetchFunc { +// the given repository. +func trustedMetadataTargetsFetcher(cacheDir *pathrs.Root, repo *config.Repository, metadata *tuftrustedmetadata.TrustedMetadata) tufext.TargetMetadataFetchFunc { var mu sync.RWMutex // to serialise access to TrustedMetadata return func(ctx context.Context, roleName, delegatorName string) (_ *tufext.SignedTargets, Err error) { @@ -375,7 +492,7 @@ func (client *Client) trustedMetadataTargetsFetcher(repo *config.Repository, met // Make sure to make a local copy of the file. // TODO: Use a pathrs-backed mktemp to allocate and swap over the file. filePath := filepath.Join(repo.Name, roleName+".json") //nolint:forbidigo // lexical pathname - localFile, err := client.CacheDir.Create(filePath, unix.O_TRUNC|unix.O_CREAT|unix.O_WRONLY|unix.O_NOFOLLOW, 0o644) + localFile, err := cacheDir.Create(filePath, unix.O_TRUNC|unix.O_CREAT|unix.O_WRONLY|unix.O_NOFOLLOW, 0o644) if err != nil { return nil, fmt.Errorf("write role %s cached json: %w", roleName, err) } @@ -399,20 +516,10 @@ func (client *Client) IterTargetFiles(ctx context.Context) iter.Seq2[*TargetInfo } repo := client.Config.Repos[repoName] - meta := updater.GetTrustedMetadataSet() - if meta.Timestamp == nil { - // FIXME: The local client TrustedMetadata state does not get - // filled until we do a refresh but go-tuf's client does not - // allow Refresh on the same updater more than once(?!). So we - // do a refresh here opportunistically. - // TODO: Add a (*Client).Refresh helper to make this much less - // fragile. - if err := updater.Refresh(); err != nil { - return fmt.Errorf("refresh repo %s: %w", repo.Name, err) - } - meta = updater.GetTrustedMetadataSet() + fetchFn, err := updater.targetsFetcher(ctx) + if err != nil { + return err } - fetchFn := client.trustedMetadataTargetsFetcher(repo, &meta) for target, err := range tufext.IterTargetFiles(ctx, fetchFn) { if err != nil { @@ -461,7 +568,7 @@ func NewClient(ctx context.Context, config *config.Config) (_ *Client, Err error defer funchelpers.CloseOnError(&Err, cacheDir) var ( - updaters = make(map[string]*tufupdater.Updater, len(config.Repos)) + updaters = make(map[string]*Repo, len(config.Repos)) skippedRepos = make(map[string]error) ) for _, repo := range config.Repos { @@ -481,7 +588,7 @@ func NewClient(ctx context.Context, config *config.Config) (_ *Client, Err error if err != nil { return nil, fmt.Errorf("bad repo %s: %w", repo.Name, err) } - updaters[repo.Name] = updater + updaters[repo.Name] = &Repo{repo: repo, cacheDir: cacheDir, updater: updater} } if len(updaters) < 1 { slog.Warn("No repositories defined -- all operations are a no-op!") diff --git a/internal/tufclient/client_bare_test.go b/internal/tufclient/client_bare_test.go new file mode 100644 index 0000000..4ecd01c --- /dev/null +++ b/internal/tufclient/client_bare_test.go @@ -0,0 +1,140 @@ +//go:build insecure + +// Copyright (C) 2026 Amutable GmbH + +package tufclient_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tufmetadata "github.com/theupdateframework/go-tuf/v2/metadata" + + "go.amutable.dev/quarry/internal/testrepo" +) + +// httpGet fetches a URL of the test server. +func httpGet(t *testing.T, url string) []byte { + t.Helper() + res, err := http.Get(url) //nolint:noctx // test code + require.NoError(t, err) + defer func() { _ = res.Body.Close() }() + require.Equal(t, http.StatusOK, res.StatusCode, url) + data, err := io.ReadAll(res.Body) + require.NoError(t, err) + return data +} + +// publishedTargetsURL returns the URL of the server's current signed +// targets.json. +func publishedTargetsURL(t *testing.T, srv *testrepo.Server) string { + t.Helper() + targets, err := srv.TxnStart(t).TargetsRoleData(t.Context(), tufmetadata.TARGETS) + require.NoError(t, err) + return fmt.Sprintf("%s/%d.targets.json", srv.MetaRootURL(), targets.Signed.Version) +} + +// bareConfigBlock configures the server's repository as a bare targets file +// (the server's own signed targets.json, trusted through its root). +func bareConfigBlock(t *testing.T, srv *testrepo.Server, name, targetsURL string, opts ...testrepo.ConfigBlockOption) string { + t.Helper() + return srv.ConfigBlock(name, append([]testrepo.ConfigBlockOption{ + testrepo.ConfigWithRootTrust(srv.InlineTrust(t)), + testrepo.ConfigWithExtraLines(fmt.Sprintf("targets_url = %q", targetsURL)), + }, opts...)...) +} + +// A bare targets repository lists and serves exactly what its signed +// targets.json says, without any snapshot or timestamp. +func TestBareTargets_ListAndFetch(t *testing.T) { + targetData := []byte("hello bare targets") + srv := testrepo.New(t) + srv.Publish(t, srv.AddTarget(t, "foo/hello.txt", bytes.NewReader(targetData))) + + cfg := testrepo.Config(t, bareConfigBlock(t, srv, "bare", publishedTargetsURL(t, srv))) + require.True(t, cfg.Repos["bare"].IsBareTargets()) + client := newClient(t, cfg) + assert.Equal(t, []string{"bare"}, clientRepoNames(t, client)) + + var paths []string + for info, err := range client.IterTargetFiles(t.Context()) { + require.NoError(t, err) + assert.Equal(t, "bare", info.Repo.Name) + paths = append(paths, info.Path) + } + assert.Equal(t, []string{"foo/hello.txt"}, paths) + + info, err := client.GetTargetInfo(t.Context(), "foo/hello.txt") + require.NoError(t, err) + assert.Equal(t, "foo/hello.txt", info.Path) + assert.Equal(t, targetData, fetchAll(t, info)) + + _, err = client.GetTargetInfo(t.Context(), "missing.txt") + require.ErrorIs(t, err, fs.ErrNotExist) + + for _, repo := range client.IterRepos(t.Context()) { + expiry, err := repo.Expiry(t.Context()) + require.NoError(t, err) + assert.True(t, expiry.After(time.Now()), "targets.json expiry %s should lie ahead", expiry) + } +} + +// A targets file whose signature does not verify against the trusted root is +// rejected, as is one that has expired or cannot be found. The error is not +// masked by a later repository that has the target. +func TestBareTargets_Rejected(t *testing.T) { + srv := testrepo.New(t) + srv.Publish(t, srv.AddTarget(t, "foo/hello.txt", bytes.NewReader([]byte("x")))) + targetsURL := publishedTargetsURL(t, srv) + + // In the tampered file the signed body changes but the signature does not. + var tampered map[string]any + require.NoError(t, json.Unmarshal(httpGet(t, targetsURL), &tampered)) + signed, ok := tampered["signed"].(map[string]any) + require.True(t, ok, "targets.json has no signed object") + signed["expires"] = "2999-01-01T00:00:00Z" + tamperedJSON, err := json.Marshal(tampered) + require.NoError(t, err) + srv.Handle("/tampered/targets.json", http.HandlerFunc(func(wtr http.ResponseWriter, _ *http.Request) { + _, _ = wtr.Write(tamperedJSON) + })) + + for _, tc := range []struct { + name, url, wantErr string + notExist bool + }{ + {name: "Tampered", url: srv.URL + "/tampered/targets.json", wantErr: "not signed by the trusted root"}, + {name: "Missing", url: srv.URL + "/nowhere/targets.json", wantErr: "status code 404", notExist: true}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := testrepo.Config(t, + bareConfigBlock(t, srv, "bare", tc.url), + srv.ConfigBlock("full", configWithOrderIndex(200)), + ) + client := newClient(t, cfg) + assert.Equal(t, []string{"bare", "full"}, clientRepoNames(t, client)) + _, err := client.GetTargetInfo(t.Context(), "foo/hello.txt") + require.ErrorContains(t, err, "refresh repo bare") + require.ErrorContains(t, err, tc.wantErr) + if tc.notExist { + require.ErrorIs(t, err, fs.ErrNotExist) + } + }) + } + + t.Run("Expired", func(t *testing.T) { + cfg := testrepo.Config(t, bareConfigBlock(t, srv, "bare", targetsURL)) + client := newClient(t, cfg) + client.SetRefTime(t.Context(), time.Date(2999, 1, 1, 0, 0, 0, 0, time.UTC)) + _, err := client.GetTargetInfo(t.Context(), "foo/hello.txt") + require.ErrorContains(t, err, "expired") + }) +} diff --git a/internal/tufclient/config/config.go b/internal/tufclient/config/config.go index 2ba3109..cc92cfb 100644 --- a/internal/tufclient/config/config.go +++ b/internal/tufclient/config/config.go @@ -15,8 +15,10 @@ import ( "net/http" "net/url" "os" + "path" "path/filepath" "slices" + "strings" "github.com/BurntSushi/toml" @@ -27,6 +29,10 @@ import ( // TODO: Make these more configurable. const ( MaxRootBytes = 512_000 // 512k + + // maxTargetsBytes is the maximum size of the targets.json of a bare + // targets repository (see [Repository.TargetsURL]). + maxTargetsBytes = 16 << 20 // 16 MiB ) // RootTrustSource represents a source of trust for the initial state of a @@ -199,8 +205,13 @@ func (tofuRootTrust) IsRemote() bool { return true } func (tofuRootTrust) FetchRoot(ctx context.Context, repo *Repository) (_ []byte, Err error) { // The updater will bump the root.json to the latest version afterwards. rootURL := repo.MetaRootURL.JoinPath("1.root.json") + return fetchJSON(ctx, rootURL, MaxRootBytes) // use same max as client +} - req, err := http.NewRequestWithContext(ctx, "GET", rootURL.String(), nil) +// fetchJSON fetches the JSON document at the given URL and reads at most +// maxBytes of it. A 404 response results in a wrapped [fs.ErrNotExist]. +func fetchJSON(ctx context.Context, docURL *url.URL, maxBytes int64) (_ []byte, Err error) { + req, err := http.NewRequestWithContext(ctx, "GET", docURL.String(), nil) if err != nil { return nil, fmt.Errorf("create http request: %w", err) } @@ -209,20 +220,20 @@ func (tofuRootTrust) FetchRoot(ctx context.Context, repo *Repository) (_ []byte, client := http.DefaultClient res, err := client.Do(req) if err != nil { - return nil, fmt.Errorf("fetch %s: %w", rootURL, err) + return nil, fmt.Errorf("fetch %s: %w", docURL, err) } if res.StatusCode >= 300 { if res.Body != nil { _ = res.Body.Close() } - err := fmt.Errorf("fetch %s failed with status code %.3d", rootURL, res.StatusCode) + err := fmt.Errorf("fetch %s failed with status code %.3d", docURL, res.StatusCode) if res.StatusCode == http.StatusNotFound { // Emulate ENOENT for 404. err = fmt.Errorf("%w: %w", err, fs.ErrNotExist) } return nil, err } - rdr := http.MaxBytesReader(nil, res.Body, MaxRootBytes) // use same max as client + rdr := http.MaxBytesReader(nil, res.Body, maxBytes) defer funchelpers.VerifyClose(&Err, rdr) return io.ReadAll(rdr) @@ -342,6 +353,24 @@ type Repository struct { // DataRootURL is the base URL for the directory containing target data // files. DataRootURL *tomlURL `toml:"data_root_url"` + + // TargetsURL, if set, makes this a "bare targets" repository. Its only + // metadata is the signed targets.json at this URL, verified against the + // targets keys of the trusted root.json. Without snapshot and timestamp + // there are no freshness guarantees. DataRootURL defaults to the + // directory of the targets file, and MetaRootURL is only used to fetch + // 1.root.json for insecure-tofu root_trust. + TargetsURL *tomlURL `toml:"targets_url"` +} + +// IsBareTargets returns whether the repository is a bare signed targets file +// rather than a full TUF repository (see [Repository.TargetsURL]). +func (repo Repository) IsBareTargets() bool { return repo.TargetsURL != nil } + +// FetchTargets fetches the unverified targets.json of a bare targets +// repository from [Repository.TargetsURL]. +func (repo Repository) FetchTargets(ctx context.Context) ([]byte, error) { + return fetchJSON(ctx, &repo.TargetsURL.URL, maxTargetsBytes) } const defaultOrderIndex = 100 @@ -491,6 +520,21 @@ func (cfg *Config) expandAndValidate() error { return fmt.Errorf("repository %s has invalid meta_root_url value: %w", repo.Name, err) } + if repo.TargetsURL != nil { + if err := repo.TargetsURL.Expand(subExpander); err != nil { + return fmt.Errorf("repository %s has invalid targets_url value: %w", repo.Name, err) + } + if repo.TargetsURL.Path == "" || strings.HasSuffix(repo.TargetsURL.Path, "/") { + return fmt.Errorf("repository %s has invalid targets_url value: %q does not name a file", repo.Name, repo.TargetsURL.String()) + } + if repo.DataRootURL == nil { + // A bare targets file lists the files beside it. + dirURL := repo.TargetsURL.URL + dirURL.Path = path.Dir(dirURL.Path) //nolint:forbidigo // a URL path, not a file path + repo.DataRootURL = &tomlURL{rawString: dirURL.String()} + } + } + if repo.DataRootURL == nil { // If unspecified, assume that the targets URL is a subdirectory of // the metadata URL (this matches the stock go-tuf client @@ -544,6 +588,7 @@ func (cfg *Config) merge(fragment *Config) error { } old.MetaRootURL = mergeURL(old.MetaRootURL, repo.MetaRootURL) old.DataRootURL = mergeURL(old.DataRootURL, repo.DataRootURL) + old.TargetsURL = mergeURL(old.TargetsURL, repo.TargetsURL) } return nil } diff --git a/internal/tufclient/config/config_test.go b/internal/tufclient/config/config_test.go index 12099b5..ddc373e 100644 --- a/internal/tufclient/config/config_test.go +++ b/internal/tufclient/config/config_test.go @@ -1110,3 +1110,49 @@ func TestRootTrustSource_String(t *testing.T) { }) } } + +// A targets_url makes the repository a bare targets file. The data root +// defaults to the file's directory, the URL has to name a file, and a drop-in +// can turn a repository into a bare targets one and back. +func TestParseConfig_TargetsURL(t *testing.T) { + const ( + head = "config_version = 1\n[repo.\"example.com/build\"]\n" + tofu = head + "root_trust = \"insecure-tofu\"\n" + nightly = "targets_url = \"https://example.com/images/nightly-1/targets_x86-64.json\"\n" + setBare = head + "targets_url = \"https://example.com/build/targets.json\"\n" + clearBare = head + "targets_url = \"\"\n" + ) + for _, tc := range []struct { + name string + fragments []string + wantErr string + wantBare bool + wantDataRoot string + }{ + {name: "Bare", fragments: []string{tofu + nightly}, wantBare: true, wantDataRoot: "https://example.com/images/nightly-1"}, + {name: "DataRootURL", fragments: []string{tofu + nightly + "data_root_url = \"https://cdn.example.com/nightly-1\"\n"}, wantBare: true, wantDataRoot: "https://cdn.example.com/nightly-1"}, + {name: "HostRoot", fragments: []string{tofu + "targets_url = \"https://example.com/targets.json\"\n"}, wantBare: true, wantDataRoot: "https://example.com/"}, + {name: "Directory", fragments: []string{tofu + "targets_url = \"https://example.com/images/nightly-1/\"\n"}, wantErr: "does not name a file"}, + {name: "Full", fragments: []string{tofu + "meta_root_url = \"https://example.com/repo\"\n"}, wantDataRoot: "https://example.com/repo/targets"}, + {name: "MergeSet", fragments: []string{tofu, setBare}, wantBare: true, wantDataRoot: "https://example.com/build"}, + {name: "MergeClear", fragments: []string{tofu, setBare, clearBare}, wantDataRoot: "https://example.com/build/targets"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := &Config{Version: ConfigVersion, Repos: make(map[string]*Repository)} + for _, fragment := range tc.fragments { + parsed, err := parseToml(strings.NewReader(fragment)) + require.NoError(t, err) + require.NoError(t, cfg.merge(parsed)) + } + err := cfg.expandAndValidate() + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + repo := cfg.Repos["example.com/build"] + assert.Equal(t, tc.wantBare, repo.IsBareTargets()) + assert.Equal(t, tc.wantDataRoot, repo.DataRootURL.String()) + }) + } +} From 35b1b1a2af39cf12e11d0a362503f44fea3ac01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 12 Oct 2026 11:54:37 +0900 Subject: [PATCH 2/3] hardhat: Publish sysupdate definitions inline with --quarry-transfer-dir quarry-sysupdate takes the sysupdate definitions of a component from special targets under .quarry/sysupdate[.].d/, but every producer had to lay out these paths and inline the files itself. That duplicates the layout detail and can drift from what the client expects. hardhat targets gains --quarry-transfer-dir =, which publishes the files of a definitions directory as inline targets in that layout. A source may also be confext:, for which hardhat generates the transfer installing the confext DDI, and several sources merge into one directory. A named component without its own component file gets a generated one, which is enabled like sysupdate's default. The layout rules live in internal/transferlayout so that quarry-sysupdate can share them. --- cmd/hardhat/targets.go | 32 +++- cmd/hardhat/targets_quarry.go | 209 +++++++++++++++++++++++++ cmd/hardhat/targets_quarry_test.go | 179 +++++++++++++++++++++ internal/transferlayout/layout.go | 78 +++++++++ internal/transferlayout/layout_test.go | 48 ++++++ 5 files changed, 544 insertions(+), 2 deletions(-) create mode 100644 cmd/hardhat/targets_quarry.go create mode 100644 cmd/hardhat/targets_quarry_test.go create mode 100644 internal/transferlayout/layout.go create mode 100644 internal/transferlayout/layout_test.go diff --git a/cmd/hardhat/targets.go b/cmd/hardhat/targets.go index 723205a..f4e2d38 100644 --- a/cmd/hardhat/targets.go +++ b/cmd/hardhat/targets.go @@ -101,6 +101,23 @@ func hashToTargets(ctx context.Context, builder *tufext.TargetsBuilder, logicalP return nil } +// maxInlineSize bounds the contents the --quarry-* options embed into +// targets.json. Inline data is meant for tiny metadata files, not payloads. +const maxInlineSize = 64 << 10 + +// addInlineBytes adds data as a target file with its contents inlined. +func addInlineBytes(builder *tufext.TargetsBuilder, logicalPath string, data []byte) (*tufmetadata.TargetFiles, error) { + if int64(len(data)) > maxInlineSize { + return nil, fmt.Errorf("target %s is too large (%d bytes) to be inlined (limit %d bytes)", logicalPath, len(data), maxInlineSize) + } + target, err := builder.AddTargetFile(logicalPath, int64(len(data)), hashAlgorithm.FromBytes(data)) + if err != nil { + return nil, fmt.Errorf("add file %s to targets data: %w", logicalPath, err) + } + tufext.TargetFilesExt(target).WithInlineData(data) + return target, nil +} + const dataRootURLCtxKey ctxKey = "--data-root-url" func ctxDataRoolURL(ctx context.Context) *url.URL { @@ -255,7 +272,7 @@ func addPrehashedToTargets(ctx context.Context, builder *tufext.TargetsBuilder, var targetsCommand = withKeystoreFlag(&cli.Command{ Name: "targets", Usage: "generate a TUF targets.json (or delegated target) file", - Flags: []cli.Flag{ + Flags: append([]cli.Flag{ &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, @@ -289,7 +306,7 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Name: "expire-after", Usage: "configure the expiry of the targets file (duration relative to --ref-time)", }, - }, + }, quarryTargetsFlags()...), Arguments: []cli.Argument{ // TODO: This doesn't work if you have pre-hashed values. // TODO: This won't work if you have enough files to hit the @@ -344,6 +361,13 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Action: func(ctx context.Context, cmd *cli.Command) error { store := ctxKeystore(ctx) + // This happens before creating --output so that bad options leave it + // alone. + quarry, err := quarrySpecFromFlags(cmd) + if err != nil { + return err + } + var output io.Writer if outPath := cmd.String("output"); outPath != "-" { outFile, err := os.Create(outPath) //nolint:forbidigo // user-controlled host path @@ -438,6 +462,10 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ } } + if err := addQuarryTargets(builder, quarry); err != nil { + return err + } + signed, err := builder.SignWith(ctx, store, keys...) if err != nil { return fmt.Errorf("failed to sign targets data: %w", err) diff --git a/cmd/hardhat/targets_quarry.go b/cmd/hardhat/targets_quarry.go new file mode 100644 index 0000000..7e39d18 --- /dev/null +++ b/cmd/hardhat/targets_quarry.go @@ -0,0 +1,209 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "unicode" + + "github.com/urfave/cli/v3" + + "go.amutable.dev/quarry/internal/transferlayout" + "go.amutable.dev/quarry/internal/tufext" +) + +// quarryTargetsFlags returns the --quarry-* options of "hardhat targets". They +// add the transfer definition directories of +// internal/transferlayout that quarry-sysupdate reads. hardhat lays them out +// from plain definition files, so producers need not know the layout. +func quarryTargetsFlags() []cli.Flag { + return []cli.Flag{ + &cli.StringSliceFlag{ + Name: "quarry-transfer-dir", + Usage: "ship sysupdate definitions for one component, given as =. is the component's sysupdate definitions directory. It is \"sysupdate.d\" for the default component or \"sysupdate..d\". is a directory of definitions (*.transfer, *.feature, .component and drop-in directories) or \"" + confextSourcePrefix + "\" for a generated transfer installing the confext DDI _.confext.raw. Repeat the option for the same to merge sources. The files are published under " + transferlayout.Prefix + "/. For a named component, .component is generated (without Enabled=, so it is an enabled-by-default component) if no source ships it", + }, + } +} + +// quarryTransferDir is one of --quarry-transfer-dir with its +// sources. +type quarryTransferDir struct { + dir string + sources []string +} + +// quarrySpecFromFlags parses and checks the --quarry-* options. +func quarrySpecFromFlags(cmd *cli.Command) ([]*quarryTransferDir, error) { + var dirs []*quarryTransferDir + byDir := map[string]*quarryTransferDir{} + for _, arg := range cmd.StringSlice("quarry-transfer-dir") { + dir, source, ok := strings.Cut(arg, "=") + if !ok || source == "" { + return nil, fmt.Errorf("--quarry-transfer-dir=%q must be =", arg) + } + if _, err := transferlayout.ParseComponentDir(dir); err != nil { + return nil, fmt.Errorf("--quarry-transfer-dir=%q: %w", arg, err) + } + entry, ok := byDir[dir] + if !ok { + entry = &quarryTransferDir{dir: dir} + byDir[dir] = entry + dirs = append(dirs, entry) + } + if slices.Contains(entry.sources, source) { + return nil, fmt.Errorf("--quarry-transfer-dir=%q given twice", arg) + } + entry.sources = append(entry.sources, source) + } + return dirs, nil +} + +// confextSourcePrefix marks a --quarry-transfer-dir source that names a +// confext to generate the definitions for rather than a directory. +const confextSourcePrefix = "confext:" + +// hasSpaceOrControl returns whether s would break a definition line. +func hasSpaceOrControl(s string) bool { + return strings.ContainsFunc(s, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) +} + +// confextTransfer returns the transfer definition installing the confext DDI +// "_.confext.raw" into the /var/lib/confexts/.raw.v/ +// directory, from which systemd picks the newest instance. The source pattern +// matches the image at any depth (**/), so a repository may serve it from a +// subdirectory as well as flat. The source path is a placeholder that +// quarry-sysupdate replaces with its proxy URL when staging. Verify=no is set +// because TUF already authenticated the bytes. +func confextTransfer(name string) []byte { + return []byte(`[Transfer] +Verify=no + +[Source] +Type=url-file +Path=__quarry_will_replace_this__ +MatchPattern=**/` + name + `_@v.confext.raw + +[Target] +Type=regular-file +Path=/var/lib/confexts/` + name + `.raw.v +PathRelativeTo=root +MatchPattern=` + name + `_@v.raw +Mode=0644 +InstancesMax=2 +`) +} + +// readDefinitionSource adds the definition files of one source directory to +// files, keyed by their path relative to it, or generates them for a confext +// source. A file name that another source already added is an error. +func readDefinitionSource(files map[string][]byte, source string) error { + add := func(rel string, data []byte) error { + if _, dup := files[rel]; dup { + return fmt.Errorf("%s: definition file %s is also in another source", source, rel) + } + files[rel] = data + return nil + } + if name, ok := strings.CutPrefix(source, confextSourcePrefix); ok { + // systemd reads the confext name up to the first "_" of the file name. + if !transferlayout.ValidPathComponent(name) || strings.Contains(name, "_") || hasSpaceOrControl(name) { + return fmt.Errorf("invalid confext name %q in %s", name, source) + } + return add("50-"+name+".transfer", confextTransfer(name)) + } + found := false + err := filepath.WalkDir(source, func(path string, entry fs.DirEntry, err error) error { //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + rel, err := filepath.Rel(source, path) //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + info, err := entry.Info() + if err != nil { + return err + } + if info.Size() > maxInlineSize { + return fmt.Errorf("definition file %s is too large (%d bytes) to be inlined (limit %d bytes)", path, info.Size(), maxInlineSize) + } + data, err := os.ReadFile(path) //nolint:forbidigo // user-controlled host path + if err != nil { + return err + } + found = true + return add(rel, data) + }) + if err != nil { + return err + } + if !found { + return fmt.Errorf("%s holds no definition files", source) + } + return nil +} + +// layoutTransferDir turns the sources of one transfer directory into its +// files. For a named component, these include the generated component file +// if no source ships one. +func layoutTransferDir(entry *quarryTransferDir) (map[string][]byte, error) { + files := map[string][]byte{} + for _, source := range entry.sources { + if err := readDefinitionSource(files, source); err != nil { + return nil, err + } + } + // quarrySpecFromFlags already checked the directory name. + component, _ := transferlayout.ParseComponentDir(entry.dir) + if component == "" { + return files, nil + } + componentFile := component + ".component" + if _, shipped := files[componentFile]; !shipped { + files[componentFile] = transferlayout.ComponentFile(component) + } + return files, nil +} + +// addQuarryTargets adds what the --quarry-* options ask for to the targets. +func addQuarryTargets(builder *tufext.TargetsBuilder, dirs []*quarryTransferDir) error { + for _, entry := range dirs { + files, err := layoutTransferDir(entry) + if err != nil { + return err + } + for _, rel := range slices.Sorted(maps.Keys(files)) { + path, err := transferlayout.TargetPath(entry.dir, rel) + if err != nil { + return fmt.Errorf("%s: %s: %w", entry.dir, rel, err) + } + if err := rejectExisting(builder, path); err != nil { + return err + } + if _, err := addInlineBytes(builder, path, files[rel]); err != nil { + return err + } + } + } + return nil +} + +// rejectExisting returns an error if the targets already hold path. The +// builder would otherwise silently replace the entry (a generated entry +// colliding with a hashed or merged one). +func rejectExisting(builder *tufext.TargetsBuilder, path string) error { + if _, dup := builder.TargetsType().Targets[path]; dup { + return fmt.Errorf("target %s is given twice", path) + } + return nil +} diff --git a/cmd/hardhat/targets_quarry_test.go b/cmd/hardhat/targets_quarry_test.go new file mode 100644 index 0000000..99abddf --- /dev/null +++ b/cmd/hardhat/targets_quarry_test.go @@ -0,0 +1,179 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" + + "go.amutable.dev/quarry/internal/tufext" +) + +func writeDefinitions(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for rel, content := range files { + path := filepath.Join(dir, rel) //nolint:forbidigo // test code + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint:forbidigo // test code + } + return dir +} + +func inlineString(t *testing.T, builder *tufext.TargetsBuilder, path string) string { + t.Helper() + target := builder.TargetsType().Targets[path] + require.NotNil(t, target, "target %s", path) + data, err := tufext.TargetFilesExt(target).InlineData() + require.NoError(t, err) + return string(data) +} + +func TestAddQuarryTargets(t *testing.T) { + base := writeDefinitions(t, t.TempDir(), map[string]string{ + "10-usr.transfer": "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\n", + "10-usr.transfer.d/x.conf": "[Transfer]\nVerify=no\n", + "docker.feature": "[Feature]\nSuggestOnMachineTag=amutable.ext.docker\n", + }) + k8s := writeDefinitions(t, t.TempDir(), map[string]string{ + "k8s.transfer": "[Transfer]\n\n[Source]\nType=url-file\n", + }) + shipped := writeDefinitions(t, t.TempDir(), map[string]string{ + "own.transfer": "[Transfer]\n", + "own.component": "[Component]\nDescription=Own\n", + }) + spec := []*quarryTransferDir{ + {dir: "sysupdate.d", sources: []string{base}}, + {dir: "sysupdate.k8s.d", sources: []string{k8s}}, + {dir: "sysupdate.own.d", sources: []string{shipped}}, + } + builder := tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + + // The files are published as they are. + const baseDir = ".quarry/sysupdate.d/" + assert.Equal(t, "[Transfer]\nProtectVersion=%A\n\n[Source]\nType=url-file\n", inlineString(t, builder, baseDir+"10-usr.transfer")) + assert.Equal(t, "[Transfer]\nVerify=no\n", inlineString(t, builder, baseDir+"10-usr.transfer.d/x.conf")) + assert.Equal(t, "[Feature]\nSuggestOnMachineTag=amutable.ext.docker\n", inlineString(t, builder, baseDir+"docker.feature")) + + const k8sDir = ".quarry/sysupdate.k8s.d/" + assert.Equal(t, "[Component]\nDescription=k8s\n", + inlineString(t, builder, k8sDir+"k8s.component"), "a missing component file is generated") + assert.Equal(t, "[Component]\nDescription=Own\n", inlineString(t, builder, ".quarry/sysupdate.own.d/own.component")) + + targets := builder.TargetsType().Targets + assert.Len(t, targets, 7) + for path, target := range targets { + assert.Nil(t, target.Custom, "%s: quarry metadata belongs to no version", path) + } +} + +func TestAddQuarryTargetsRejects(t *testing.T) { + dir := writeDefinitions(t, t.TempDir(), map[string]string{}) + spec := []*quarryTransferDir{{dir: "sysupdate.d", sources: []string{dir}}} + require.ErrorContains(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "holds no definition files") + + // A path already in the targets is an error rather than a silent + // replacement. + dir = writeDefinitions(t, t.TempDir(), map[string]string{"x.transfer": "[Transfer]\n"}) + builder := tufext.NewTargetsBuilder() + _, err := addInlineBytes(builder, ".quarry/sysupdate.d/x.transfer", nil) + require.NoError(t, err) + spec = []*quarryTransferDir{{dir: "sysupdate.d", sources: []string{dir}}} + require.ErrorContains(t, addQuarryTargets(builder, spec), "given twice") +} + +// A confext source generates the definitions. The transfer comes from +// hardhat's template, and the component file is generated as for any component +// directory without one. +func TestAddQuarryTargets_Confext(t *testing.T) { + spec := []*quarryTransferDir{ + {dir: "sysupdate.edge-gw.d", sources: []string{"confext:edge-gw"}}, + } + builder := tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + + const dir = ".quarry/sysupdate.edge-gw.d/" + transfer := inlineString(t, builder, dir+"50-edge-gw.transfer") + assert.True(t, strings.HasPrefix(transfer, "[Transfer]\nVerify=no\n"), transfer) + for _, want := range []string{ + "MatchPattern=**/edge-gw_@v.confext.raw\n", + "Path=/var/lib/confexts/edge-gw.raw.v\n", + "MatchPattern=edge-gw_@v.raw\n", + } { + assert.Contains(t, transfer, want) + } + assert.Equal(t, "[Component]\nDescription=edge-gw\n", inlineString(t, builder, dir+"edge-gw.component")) + + // Several sources merge into one directory. A file in two of them is an error. + extra := writeDefinitions(t, t.TempDir(), map[string]string{"60-extra.transfer": "[Transfer]\n"}) + spec = []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", "confext:gw-certs", extra}}, + } + builder = tufext.NewTargetsBuilder() + require.NoError(t, addQuarryTargets(builder, spec)) + assert.Contains(t, inlineString(t, builder, ".quarry/sysupdate.gw.d/50-gw.transfer"), "MatchPattern=**/gw_@v.confext.raw\n") + assert.Contains(t, inlineString(t, builder, ".quarry/sysupdate.gw.d/50-gw-certs.transfer"), "MatchPattern=**/gw-certs_@v.confext.raw\n") + assert.Equal(t, "[Transfer]\n", inlineString(t, builder, ".quarry/sysupdate.gw.d/60-extra.transfer")) + spec = []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", extra, extra}}, + } + require.ErrorContains(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "also in another source") + + for _, name := range []string{"", "a_b", "a/b", "..", "a b", "a\nb"} { + spec := []*quarryTransferDir{ + {dir: "sysupdate.x.d", sources: []string{"confext:" + name}}, + } + require.Error(t, addQuarryTargets(tufext.NewTargetsBuilder(), spec), "confext name %q", name) + } +} + +// quarrySpecFor runs quarrySpecFromFlags on a dummy command with the args. +func quarrySpecFor(t *testing.T, args ...string) ([]*quarryTransferDir, error) { + t.Helper() + var ( + spec []*quarryTransferDir + specErr error + ) + cmd := &cli.Command{ + Name: "targets", + Flags: quarryTargetsFlags(), + Action: func(_ context.Context, cmd *cli.Command) error { + spec, specErr = quarrySpecFromFlags(cmd) + return nil + }, + } + require.NoError(t, cmd.Run(t.Context(), append([]string{"targets"}, args...))) + return spec, specErr +} + +func TestQuarrySpecFromFlags(t *testing.T) { + spec, err := quarrySpecFor(t, + "--quarry-transfer-dir=sysupdate.gw.d=confext:gw", + "--quarry-transfer-dir=sysupdate.d=/defs", + "--quarry-transfer-dir=sysupdate.gw.d=confext:gw-certs", + "--quarry-transfer-dir=sysupdate.k8s.d=/k8s", + ) + require.NoError(t, err) + assert.Equal(t, []*quarryTransferDir{ + {dir: "sysupdate.gw.d", sources: []string{"confext:gw", "confext:gw-certs"}}, + {dir: "sysupdate.d", sources: []string{"/defs"}}, + {dir: "sysupdate.k8s.d", sources: []string{"/k8s"}}, + }, spec) + + const dir = "--quarry-transfer-dir=sysupdate.x.d=/defs" + for name, args := range map[string][]string{ + "invalid component dir": {"--quarry-transfer-dir=other.d=/defs"}, + "no source": {"--quarry-transfer-dir=sysupdate.d="}, + "repeated source": {dir, dir}, + } { + _, err := quarrySpecFor(t, args...) + require.Error(t, err, name) + } +} diff --git a/internal/transferlayout/layout.go b/internal/transferlayout/layout.go new file mode 100644 index 0000000..d4463e1 --- /dev/null +++ b/internal/transferlayout/layout.go @@ -0,0 +1,78 @@ +// Copyright (C) 2026 Amutable GmbH + +// Package transferlayout describes the repository layout of what quarry +// ships as special TUF targets for systemd-sysupdate. This covers the +// transfer definition directories under ".quarry/". +// +// hardhat writes this layout ("hardhat targets --quarry-*") and +// quarry-sysupdate (internal/xsysupdate) reads it, so both sides share the +// names and rules here. +package transferlayout + +import ( + "fmt" + "strings" +) + +const ( + // Prefix is the target path prefix of everything quarry-specific in a + // repository. Such a target is never a file that a client installs + // directly. + Prefix = ".quarry/" + + // defaultDir is the definitions directory of the default (component-less) + // sysupdate component. The directory of a named component is + // "sysupdate..d". + defaultDir = "sysupdate.d" +) + +// ValidPathComponent returns whether name is usable as one path element. +func ValidPathComponent(name string) bool { + return name != "" && name != "." && name != ".." && !strings.Contains(name, "/") +} + +// ComponentDir returns the definitions directory name of a component. The +// component "" names the default component. +func ComponentDir(component string) string { + if component == "" { + return defaultDir + } + return "sysupdate." + component + ".d" +} + +// ParseComponentDir returns the component a definitions directory name is +// for. It returns "" for the default component's directory "sysupdate.d". +func ParseComponentDir(dir string) (string, error) { + if dir == defaultDir { + return "", nil + } + if !strings.HasPrefix(dir, "sysupdate.") || !strings.HasSuffix(dir, ".d") { + return "", fmt.Errorf("invalid definitions directory name %q", dir) + } + component := strings.TrimSuffix(strings.TrimPrefix(dir, "sysupdate."), ".d") + if !ValidPathComponent(component) || strings.Contains(component, ".") { + return "", fmt.Errorf("invalid component name %q in directory %q", component, dir) + } + return component, nil +} + +// TargetPath returns the target path of one file of a transfer definition +// directory, which is "/". +func TargetPath(dir, file string) (string, error) { + if _, err := ParseComponentDir(dir); err != nil { + return "", err + } + for part := range strings.SplitSeq(file, "/") { + if !ValidPathComponent(part) { + return "", fmt.Errorf("invalid transfer file path %q", file) + } + } + return Prefix + dir + "/" + file, nil +} + +// ComponentFile returns the generated ".component" definition for +// a component directory that ships none. It sets no Enabled=, so it is an +// enabled-by-default component as in sysupdate. +func ComponentFile(component string) []byte { + return []byte("[Component]\nDescription=" + component + "\n") +} diff --git a/internal/transferlayout/layout_test.go b/internal/transferlayout/layout_test.go new file mode 100644 index 0000000..656fb59 --- /dev/null +++ b/internal/transferlayout/layout_test.go @@ -0,0 +1,48 @@ +// Copyright (C) 2026 Amutable GmbH + +package transferlayout + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTargetPath(t *testing.T) { + path, err := TargetPath("sysupdate.d", "12-usr.transfer") + require.NoError(t, err) + assert.Equal(t, ".quarry/sysupdate.d/12-usr.transfer", path) + + path, err = TargetPath("sysupdate.k8s.d", "k8s.transfer.d/10-x.conf") + require.NoError(t, err) + assert.Equal(t, ".quarry/sysupdate.k8s.d/k8s.transfer.d/10-x.conf", path) + + for _, tc := range []struct{ dir, file string }{ + {"other.d", "x.transfer"}, + {"sysupdate.a.b.d", "x.transfer"}, + {"sysupdate.d", ""}, + {"sysupdate.d", "a/../b"}, + } { + _, err := TargetPath(tc.dir, tc.file) + require.Error(t, err, "%v", tc) + } +} + +func TestComponentDir(t *testing.T) { + assert.Equal(t, "sysupdate.d", ComponentDir("")) + assert.Equal(t, "sysupdate.k8s.d", ComponentDir("k8s")) + for _, dir := range []string{"sysupdate.a.b.d", "sysupdate..d"} { + _, err := ParseComponentDir(dir) + require.Error(t, err, dir) + } + for _, dir := range []string{"sysupdate.d", "sysupdate.k8s.d"} { + component, err := ParseComponentDir(dir) + require.NoError(t, err) + assert.Equal(t, dir, ComponentDir(component)) + } +} + +func TestComponentFile(t *testing.T) { + assert.Equal(t, "[Component]\nDescription=k8s\n", string(ComponentFile("k8s"))) +} From 54f64bbd6f9d61b077df1e768d58f4bee549a306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 12 Oct 2026 11:57:04 +0900 Subject: [PATCH 3/3] hardhat: Add --custom-from and --custom-version A build has to mark its targets for consumers, for example which version they belong to. hardhat could hash the files but offered no way to attach such metadata, so a producer would have to post-process the signed targets file and resign. Add hardhat targets --custom-from , which merges per-target "custom" objects from a JSON file and fails on a path that is not a target, so that a typo cannot drop metadata silently. --custom-version sets custom.quarry.version on every file target. --- cmd/hardhat/targets.go | 84 ++++++++++++++++++++++++++++++ cmd/hardhat/targets_custom_test.go | 72 +++++++++++++++++++++++++ cmd/hardhat/targets_quarry.go | 38 +++++++++++++- cmd/hardhat/targets_quarry_test.go | 17 ++++++ internal/tufext/quarrycustom.go | 8 +++ 5 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 cmd/hardhat/targets_custom_test.go create mode 100644 internal/tufext/quarrycustom.go diff --git a/cmd/hardhat/targets.go b/cmd/hardhat/targets.go index f4e2d38..f4f5a6c 100644 --- a/cmd/hardhat/targets.go +++ b/cmd/hardhat/targets.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "io" "maps" @@ -31,6 +32,7 @@ import ( "go.amutable.dev/quarry/internal/keystore" "go.amutable.dev/quarry/internal/linux" "go.amutable.dev/quarry/internal/pathrsext" + "go.amutable.dev/quarry/internal/transferlayout" "go.amutable.dev/quarry/internal/tufext" ) @@ -118,6 +120,64 @@ func addInlineBytes(builder *tufext.TargetsBuilder, logicalPath string, data []b return target, nil } +// applyCustomFrom merges the per-target "custom" objects from the given JSON +// file ({"": {}}) into the targets. Every path +// must name a target that was already added, so that a typo cannot silently +// drop metadata. No path may be under .quarry/ because the +// --quarry-* options own that metadata. Existing custom fields are kept unless +// overridden by key. +func applyCustomFrom(builder *tufext.TargetsBuilder, path string) error { + data, err := os.ReadFile(path) //nolint:forbidigo // user-controlled host path + if err != nil { + return fmt.Errorf("--custom-from=%q file is invalid: %w", path, err) + } + customs, err := jsonutils.Parse[map[string]map[string]json.RawMessage](data) + if err != nil { + return fmt.Errorf("--custom-from=%q file is invalid json: %w", path, err) + } + targets := builder.TargetsType().Targets + for targetPath, custom := range customs { + if strings.HasPrefix(targetPath, transferlayout.Prefix) { + return fmt.Errorf("--custom-from=%q: target %q is quarry metadata, which --custom-from must not change", path, targetPath) + } + target, ok := targets[targetPath] + if !ok { + return fmt.Errorf("--custom-from=%q: target %q does not exist", path, targetPath) + } + err := updateCustom(target, func(fields map[string]json.RawMessage) error { + maps.Copy(fields, custom) + return nil + }) + if err != nil { + return fmt.Errorf("--custom-from=%q: target %q: %w", path, targetPath, err) + } + } + return nil +} + +// updateCustom lets update change the fields of a target's custom object and +// stores the result. A missing or null custom object is an empty one. +func updateCustom(target *tufmetadata.TargetFiles, update func(fields map[string]json.RawMessage) error) error { + fields := map[string]json.RawMessage{} + if target.Custom != nil { + if err := json.Unmarshal(*target.Custom, &fields); err != nil { + return fmt.Errorf("existing custom data is not an object: %w", err) + } + } + if fields == nil { // "custom": null + fields = map[string]json.RawMessage{} + } + if err := update(fields); err != nil { + return err + } + encoded, err := jsonutils.MarshalNoEscapeHTML(fields) + if err != nil { + return fmt.Errorf("encode custom data: %w", err) + } + target.Custom = (*json.RawMessage)(&encoded) + return nil +} + const dataRootURLCtxKey ctxKey = "--data-root-url" func ctxDataRoolURL(ctx context.Context) *url.URL { @@ -288,6 +348,11 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ Name: "include-from", Usage: "include targets defined in the given targets.json files (later files take precedence, with arguments taking highest precedence)", }, + &cli.StringSliceFlag{ + Name: "custom-from", + Usage: "merge per-target \"custom\" objects from the given JSON file ({\"\": {...}}), where every path must name a target", + TakesFile: true, + }, &cli.BoolFlag{ Name: "ext-override-url", Usage: "when using --data-root-url --pre-hashed, add an override URL to each target file to make clients fetch from --data-root-url", @@ -466,6 +531,25 @@ var targetsCommand = withKeystoreFlag(&cli.Command{ return err } + for _, customPath := range cmd.StringSlice("custom-from") { + if err := applyCustomFrom(builder, customPath); err != nil { + return err + } + } + // This comes after --custom-from because that replaces a target's + // whole "quarry" object. The .quarry/ targets are skipped + // because they are quarry metadata and belong to no version. + if version := cmd.String("custom-version"); version != "" { + for path, target := range builder.TargetsType().Targets { + if strings.HasPrefix(path, transferlayout.Prefix) { + continue + } + if err := setQuarryCustomVersion(target, version); err != nil { + return fmt.Errorf("--custom-version: %s: %w", path, err) + } + } + } + signed, err := builder.SignWith(ctx, store, keys...) if err != nil { return fmt.Errorf("failed to sign targets data: %w", err) diff --git a/cmd/hardhat/targets_custom_test.go b/cmd/hardhat/targets_custom_test.go new file mode 100644 index 0000000..b65174b --- /dev/null +++ b/cmd/hardhat/targets_custom_test.go @@ -0,0 +1,72 @@ +// Copyright (C) 2026 Amutable GmbH + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.amutable.dev/quarry/internal/tufext" +) + +func customFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "custom.json") //nolint:forbidigo // test code + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint:forbidigo // test code + return path +} + +func TestApplyCustomFrom(t *testing.T) { + builder := tufext.NewTargetsBuilder() + _, err := builder.AddTargetFile("a.raw", 1, hashAlgorithm.FromBytes([]byte("a"))) + require.NoError(t, err) + existing, err := builder.AddTargetFile("b.raw", 1, hashAlgorithm.FromBytes([]byte("b"))) + require.NoError(t, err) + old := json.RawMessage(`{"quarry":{"old":true},"keep":1}`) + existing.Custom = &old + + null := json.RawMessage(`null`) + _, err = builder.AddTargetFile("c.raw", 1, hashAlgorithm.FromBytes([]byte("c"))) + require.NoError(t, err) + builder.TargetsType().Targets["c.raw"].Custom = &null + + path := customFile(t, `{ + "a.raw": {"quarry": {"version": "1", "update": true}}, + "b.raw": {"quarry": {"version": "1"}}, + "c.raw": {"quarry": {"version": "1"}} + }`) + require.NoError(t, applyCustomFrom(builder, path)) + + targets := builder.TargetsType().Targets + var a map[string]any + require.NoError(t, json.Unmarshal(*targets["a.raw"].Custom, &a)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1", "update": true}}, a) + + // Keys are merged at the top level. "quarry" is replaced and "keep" survives. + var b map[string]any + require.NoError(t, json.Unmarshal(*targets["b.raw"].Custom, &b)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1"}, "keep": float64(1)}, b) + + // An existing "custom": null is replaced. + var c map[string]any + require.NoError(t, json.Unmarshal(*targets["c.raw"].Custom, &c)) + assert.Equal(t, map[string]any{"quarry": map[string]any{"version": "1"}}, c) +} + +func TestApplyCustomFromRejects(t *testing.T) { + builder := tufext.NewTargetsBuilder() + _, err := builder.AddTargetFile(".quarry/sysupdate.d/x.transfer", 1, hashAlgorithm.FromBytes([]byte("a"))) + require.NoError(t, err) + for content, want := range map[string]string{ + `{"missing.raw": {"quarry": {}}}`: `target "missing.raw" does not exist`, + `["not", "an", "object"]`: "invalid json", + `{".quarry/sysupdate.d/x.transfer": {"quarry": {}}}`: "is quarry metadata", + } { + require.ErrorContains(t, applyCustomFrom(builder, customFile(t, content)), want, content) + } +} diff --git a/cmd/hardhat/targets_quarry.go b/cmd/hardhat/targets_quarry.go index 7e39d18..2f5c3fe 100644 --- a/cmd/hardhat/targets_quarry.go +++ b/cmd/hardhat/targets_quarry.go @@ -3,6 +3,7 @@ package main import ( + "encoding/json" "fmt" "io/fs" "maps" @@ -12,14 +13,16 @@ import ( "strings" "unicode" + tufmetadata "github.com/theupdateframework/go-tuf/v2/metadata" "github.com/urfave/cli/v3" + "go.amutable.dev/quarry/internal/jsonutils" "go.amutable.dev/quarry/internal/transferlayout" "go.amutable.dev/quarry/internal/tufext" ) -// quarryTargetsFlags returns the --quarry-* options of "hardhat targets". They -// add the transfer definition directories of +// quarryTargetsFlags returns the --quarry-* options of "hardhat targets" (and +// --custom-version). They add the transfer definition directories of // internal/transferlayout that quarry-sysupdate reads. hardhat lays them out // from plain definition files, so producers need not know the layout. func quarryTargetsFlags() []cli.Flag { @@ -28,6 +31,10 @@ func quarryTargetsFlags() []cli.Flag { Name: "quarry-transfer-dir", Usage: "ship sysupdate definitions for one component, given as =. is the component's sysupdate definitions directory. It is \"sysupdate.d\" for the default component or \"sysupdate..d\". is a directory of definitions (*.transfer, *.feature, .component and drop-in directories) or \"" + confextSourcePrefix + "\" for a generated transfer installing the confext DDI _.confext.raw. Repeat the option for the same to merge sources. The files are published under " + transferlayout.Prefix + "/. For a named component, .component is generated (without Enabled=, so it is an enabled-by-default component) if no source ships it", }, + &cli.StringFlag{ + Name: "custom-version", + Usage: "set custom.quarry.version on every target outside " + transferlayout.Prefix, + }, } } @@ -207,3 +214,30 @@ func rejectExisting(builder *tufext.TargetsBuilder, path string) error { } return nil } + +// setQuarryCustomVersion sets custom.quarry.version on a target, keeping its +// other custom fields. +func setQuarryCustomVersion(target *tufmetadata.TargetFiles, version string) error { + return updateCustom(target, func(custom map[string]json.RawMessage) error { + quarry := map[string]json.RawMessage{} + if raw, ok := custom[tufext.QuarryCustomField]; ok { + if err := json.Unmarshal(raw, &quarry); err != nil { + return fmt.Errorf("existing custom.quarry data is not an object: %w", err) + } + } + if quarry == nil { // "quarry": null + quarry = map[string]json.RawMessage{} + } + encodedVersion, err := json.Marshal(version) + if err != nil { + return err + } + quarry["version"] = encodedVersion + encodedQuarry, err := jsonutils.MarshalNoEscapeHTML(quarry) + if err != nil { + return err + } + custom[tufext.QuarryCustomField] = encodedQuarry + return nil + }) +} diff --git a/cmd/hardhat/targets_quarry_test.go b/cmd/hardhat/targets_quarry_test.go index 99abddf..d9843df 100644 --- a/cmd/hardhat/targets_quarry_test.go +++ b/cmd/hardhat/targets_quarry_test.go @@ -4,6 +4,7 @@ package main import ( "context" + "encoding/json" "os" "path/filepath" "strings" @@ -134,6 +135,22 @@ func TestAddQuarryTargets_Confext(t *testing.T) { } } +func TestSetQuarryCustomVersion(t *testing.T) { + builder := tufext.NewTargetsBuilder() + target, err := addInlineBytes(builder, "x", []byte("x")) + require.NoError(t, err) + // Existing custom fields, inside and outside quarry, survive. + existing := json.RawMessage(`{"quarry": {"update": true}, "other": 1}`) + target.Custom = &existing + require.NoError(t, setQuarryCustomVersion(target, "2")) + assert.JSONEq(t, `{"quarry": {"update": true, "version": "2"}, "other": 1}`, string(*target.Custom)) + + null := json.RawMessage(`{"quarry": null}`) + target.Custom = &null + require.NoError(t, setQuarryCustomVersion(target, "3")) + assert.JSONEq(t, `{"quarry": {"version": "3"}}`, string(*target.Custom)) +} + // quarrySpecFor runs quarrySpecFromFlags on a dummy command with the args. func quarrySpecFor(t *testing.T, args ...string) ([]*quarryTransferDir, error) { t.Helper() diff --git a/internal/tufext/quarrycustom.go b/internal/tufext/quarrycustom.go new file mode 100644 index 0000000..a8365fe --- /dev/null +++ b/internal/tufext/quarrycustom.go @@ -0,0 +1,8 @@ +// Copyright (C) 2026 Amutable GmbH + +package tufext + +// QuarryCustomField is the sub-field of the TUF "custom" target field that +// holds the Quarry-specific target metadata, as written by the build ("hardhat +// targets" --custom-from, --custom-version). +const QuarryCustomField = "quarry"