Add change feeds for npm and Packagist

README.md

@@ -1,6 +1,6 @@
 # registries
 
-Go library for fetching package metadata from registry APIs. Supports 29 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`) and streaming artifact downloads (`fetch/`).
+Go library for fetching package metadata from registry APIs. Supports 29 ecosystems with a unified interface. Also provides sub-packages for HTTP client usage (`client/`), streaming artifact downloads (`fetch/`) and registry change feeds (`changes/`).
 
 ## Installation
 
@@ -568,6 +568,30 @@ resolver := fetch.NewResolver(fetch.WithAddressPolicy(opts))
 client := safehttp.New(nil, opts)
 ```
 
+## Change Feeds (`changes/`)
+
+Some registries publish a feed of the packages that changed since a given point. `changes` reads them for npm (`replicate.npmjs.com`) and Packagist (`metadata/changes.json`), so a cache or mirror can refresh just those packages.
+
+Feeds are stateless: the caller keeps the cursor, and moves on to the one `Changes` returns only after acting on the changes, so a failure leaves them to be read again. A `Cursor` is a string and can be stored across restarts.
+
+```go
+feed := changes.NewNPM() // or changes.NewPackagist(changes.WithURL(...))
+
+cursor, err := feed.Start(ctx)
+// later, repeatedly:
+c, next, err := feed.Changes(ctx, cursor)
+if err != nil {
+    return err // cursor stays where it was
+}
+if c.All {
+    // the feed lost track; treat every package as changed
+}
+refresh(c.Names)
+cursor = next
+```
+
+Both take `changes.WithClient` for a configured `*client.Client` and `changes.WithURL` for a registry other than the public one. Packagist names the metadata files that changed (`vendor/pkg`, `vendor/pkg~dev`); a deleted package is reported as both.
+
 ## Private Registries
 
 PURLs with a `repository_url` qualifier automatically use that URL:

changes/changes.go

@@ -0,0 +1,80 @@
+// Package changes reads the feeds in which registries list the packages that
+// changed since a given point, so a cache or mirror can refresh just those.
+//
+// Feeds are stateless: the caller keeps the Cursor, starting from Start and
+// passing the one Changes returns to the next call. A caller that acts on the
+// changes moves on to the new cursor only once it has, so a failure leaves
+// the changes to be read again. A Cursor is a plain string and can be stored.
+package changes
+
+import (
+	"context"
+	"net/url"
+	"strings"
+
+	"github.com/git-pkgs/registries/client"
+)
+
+// Cursor is a position in a feed, as the feed's registry gives it. Treat it
+// as opaque.
+type Cursor string
+
+// Changes is what one read of a feed found.
+type Changes struct {
+	// Names are the packages that changed, as the registry names them.
+	Names []string
+	// All is set when the feed can't say what changed since the cursor, so
+	// every package has to be treated as changed.
+	All bool
+}
+
+// Feed is one registry's change feed.
+type Feed interface {
+	// Ecosystem is the ecosystem the changed names belong to.
+	Ecosystem() string
+	// Start returns the feed's current position, from which Changes reports
+	// what changes next.
+	Start(ctx context.Context) (Cursor, error)
+	// Changes returns what changed after cursor, and the cursor to read on
+	// from. On error the returned cursor is the one passed in.
+	Changes(ctx context.Context, cursor Cursor) (Changes, Cursor, error)
+}
+
+// Option configures a feed.
+type Option func(*options)
+
+type options struct {
+	client *client.Client
+	url    string
+}
+
+// WithClient sets the client a feed makes its requests with. The default is
+// client.DefaultClient().
+func WithClient(c *client.Client) Option {
+	return func(o *options) { o.client = c }
+}
+
+// WithURL sets the URL of the feed, for a registry other than the public one.
+func WithURL(u string) Option {
+	return func(o *options) { o.url = u }
+}
+
+func newOptions(defaultURL string, opts []Option) options {
+	o := options{url: defaultURL}
+	for _, opt := range opts {
+		opt(&o)
+	}
+	if o.client == nil {
+		o.client = client.DefaultClient()
+	}
+	return o
+}
+
+// withQuery returns base with the query parameters in q added.
+func withQuery(base string, q url.Values) string {
+	sep := "?"
+	if strings.Contains(base, "?") {
+		sep = "&"
+	}
+	return base + sep + q.Encode()
+}