feat(fleet): saved server lists, with CRUD

README.md

@@ -14,8 +14,6 @@ diskpush ./data/ prod:/data/ -- --checksum # your own rsync flags
 ```
 
 - **Local → server**, **server → local**, and **server → server** directly.
-- **Sync a whole directory, or just what you picked.** Tick files and folders
-  in the pane, or `--only NAME` from the CLI.
 - **Archive metadata by default.** Permissions, timestamps, symlinks.
 - **Resumable by default.** An interrupted transfer keeps its partial data.
 - **Skips unchanged files.** Re-running a job moves almost nothing.

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diskpush/cli",
-  "version": "0.2.11",
+  "version": "0.2.10",
   "type": "module",
   "bin": {
     "diskpush": "./dist/bin.js"

apps/cli/src/commands/fleet.ts

@@ -10,12 +10,15 @@ import {
   describeHazards,
   inspectScript,
   needsAttention,
+  parseSelector,
   runFleet,
   selectConnections,
   SelectionError,
   type SudoMode,
 } from '@diskpush/fleet-core'
 import {
+  isListTerm,
+  listTermName,
   FLEET_DEFAULT_CONCURRENCY,
   FLEET_DEFAULT_TIMEOUT_SECONDS,
   FleetInterpreterSchema,
@@ -72,6 +75,9 @@ async function dispatch(parsed: ParsedArgv, store: DiskPushStore, output: Output
       return fleetServers(parsed, store, output)
     case 'commands':
       return fleetCommands(parsed, store, output)
+    case 'lists':
+    case 'list':
+      return fleetLists(parsed, store, output)
     case 'runs':
       return fleetRuns(parsed, store, output)
     case 'show':
@@ -80,7 +86,7 @@ async function dispatch(parsed: ParsedArgv, store: DiskPushStore, output: Output
       return failure(
         output,
         `Unknown subcommand ${JSON.stringify(subcommand)}. ` +
-          'Try: run, script, upgrade, check, servers, commands, runs, show.',
+          'Try: run, script, upgrade, check, servers, lists, commands, runs, show.',
         EXIT.usage,
       )
   }
@@ -116,7 +122,7 @@ async function resolveTargets(parsed: ParsedArgv, store: DiskPushStore, fallback
   }
 
   const available = await availableConnections(store)
-  const selection = selectConnections(available, terms)
+  const selection = await resolveSelector(terms, store, available)
 
   if (selection.unmatched.length > 0) {
     // A typo'd host is not a smaller fleet. Refusing here is the difference
@@ -133,6 +139,70 @@ async function resolveTargets(parsed: ParsedArgv, store: DiskPushStore, fallback
   return { connections: selection.matched, selector: terms }
 }
 
+/**
+ * Parse, expand any `list:` terms, then select.
+ *
+ * The order matters and got it wrong once: `parseSelector` is what splits
+ * `--on 'all,!list:web'` into terms, so expanding before that leaves
+ * `all,!list:web` as one unrecognised string and the exclusion silently does
+ * nothing. Everything that resolves a selector goes through here so there is
+ * one order rather than one per caller.
+ */
+async function resolveSelector(
+  terms: readonly string[],
+  store: DiskPushStore,
+  available: readonly Connection[],
+): Promise<ReturnType<typeof selectConnections>> {
+  return selectConnections(available, await expandLists(parseSelector(terms), store, available))
+}
+
+/**
+ * Turns `list:production` into the servers that list holds.
+ *
+ * Expanded here rather than inside `selectConnections`, which is a pure
+ * function over connections and has no business reaching for a database.
+ *
+ * A member whose connection has since been deleted is named and refused
+ * rather than skipped: a list that quietly shrinks is how a command misses
+ * the one server it most needed to reach.
+ */
+async function expandLists(
+  terms: readonly string[],
+  store: DiskPushStore,
+  available: readonly Connection[],
+): Promise<string[]> {
+  const byId = new Map(available.map((connection) => [connection.id, connection]))
+  const expandedTerms: string[] = []
+
+  for (const term of terms) {
+    const negated = term.startsWith('!')
+    const bare = negated ? term.slice(1) : term
+    if (!isListTerm(bare)) {
+      expandedTerms.push(term)
+      continue
+    }
+
+    const name = listTermName(bare)
+    const list = await store.findFleetList(name)
+    if (!list) throw new SelectionError(`No saved list named ${JSON.stringify(name)}. Run \`diskpush fleet lists\`.`)
+    if (list.members.length === 0) throw new SelectionError(`The list ${JSON.stringify(name)} has no servers in it.`)
+
+    const missing = list.members.filter((member) => !byId.has(member.connectionId))
+    if (missing.length > 0) {
+      throw new SelectionError(
+        `The list ${JSON.stringify(name)} names ${missing.length} server(s) that no longer exist: ` +
+          `${missing.map((member) => member.connectionName).join(', ')}. ` +
+          'Save the list again to drop them.',
+      )
+    }
+    // Ids, not names: a list resolves to exactly the servers it was saved with,
+    // even if one has been renamed since.
+    for (const member of list.members) expandedTerms.push(`${negated ? '!' : ''}${member.connectionId}`)
+  }
+
+  return expandedTerms
+}
+
 // --- prompts ---------------------------------------------------------------
 
 /**
@@ -680,7 +750,7 @@ function formatUptime(seconds: number): string {
 async function fleetServers(parsed: ParsedArgv, store: DiskPushStore, output: Output): Promise<number> {
   const available = await availableConnections(store)
   const selector = flagValues(parsed, '--on')
-  const shown = selector.length > 0 ? selectConnections(available, selector).matched : available
+  const shown = selector.length > 0 ? (await resolveSelector(selector, store, available)).matched : available
 
   if (output.isJson) {
     output.json({ status: 'ok', servers: shown })
@@ -814,6 +884,125 @@ async function fleetCommands(parsed: ParsedArgv, store: DiskPushStore, output: O
   return failure(output, `Unknown action ${JSON.stringify(action)}. Try: list, show, save, copy, remove.`, EXIT.usage)
 }
 
+/**
+ * Saved sets of servers.
+ *
+ * Tags say what a server *is*; a list is a set someone assembled by hand and
+ * wants back. Used as `--on list:NAME`, prefixed so a list and a server may
+ * share a name without either shadowing the other.
+ */
+async function fleetLists(parsed: ParsedArgv, store: DiskPushStore, output: Output): Promise<number> {
+  const action = parsed.positionals[1] ?? 'list'
+
+  if (action === 'list') {
+    const lists = await store.listFleetLists()
+    if (output.isJson) {
+      output.json({ status: 'ok', lists })
+      return EXIT.ok
+    }
+    if (lists.length === 0) {
+      output.line('No saved lists. Make one with: diskpush fleet lists save NAME --on tag:production')
+      return EXIT.ok
+    }
+    output.line(
+      table(
+        lists.map((list) => [
+          list.name,
+          String(list.members.length),
+          list.members.slice(0, 4).map((member) => member.connectionName).join(', ') +
+            (list.members.length > 4 ? ', ...' : ''),
+          list.description.slice(0, 40),
+        ]),
+        ['LIST', 'SERVERS', 'MEMBERS', 'DESCRIPTION'],
+      ),
+    )
+    return EXIT.ok
+  }
+
+  if (action === 'show') {
+    const name = parsed.positionals[2]
+    if (!name) return failure(output, 'Usage: diskpush fleet lists show NAME', EXIT.usage)
+    const list = await store.findFleetList(name)
+    if (!list) return failure(output, `No saved list named ${JSON.stringify(name)}.`, EXIT.configuration)
+
+    if (output.isJson) {
+      output.json({ status: 'ok', list })
+      return EXIT.ok
+    }
+
+    // A member whose connection has gone is shown as missing rather than
+    // dropped: that is the difference between a list you can trust and one
+    // that quietly got smaller.
+    const available = await availableConnections(store)
+    const byId = new Map(available.map((connection) => [connection.id, connection]))
+    output.line(`${list.name}${list.description ? `  ${list.description}` : ''}`)
+    output.line()
+    output.line(
+      table(
+        list.members.map((member) => {
+          const live = byId.get(member.connectionId)
+          return [
+            member.connectionName,
+            live ? `${live.username}@${live.host}:${live.port}` : '-',
+            live ? 'ok' : 'MISSING',
+          ]
+        }),
+        ['SERVER', 'TARGET', 'STATE'],
+      ),
+    )
+    return EXIT.ok
+  }
+
+  if (action === 'save') {
+    const name = parsed.positionals[2]
+    if (!name) return failure(output, 'Usage: diskpush fleet lists save NAME --on SELECTOR', EXIT.usage)
+    if (isListTerm(name)) {
+      return failure(output, `A list is named without the ${JSON.stringify('list:')} prefix.`, EXIT.usage)
+    }
+
+    // Resolved now, and stored as members. A list is a set someone chose, not
+    // a query that might mean something different next week.
+    const targets = await resolveTargets(parsed, store)
+    const saved = await store.saveFleetList({
+      name,
+      description: flagValue(parsed, '--description') ?? '',
+      members: targets.connections.map((connection) => ({
+        connectionId: connection.id,
+        connectionName: connection.name,
+      })),
+    })
+
+    if (output.isJson) output.json({ status: 'ok', list: saved })
+    else {
+      output.line(`Saved list ${saved.name} with ${saved.members.length} server(s).`)
+      output.line(`Use it with: diskpush fleet run "uptime" --on list:${saved.name}`)
+    }
+    return EXIT.ok
+  }
+
+  if (action === 'rename') {
+    const [, , from, to] = parsed.positionals
+    if (!from || !to) return failure(output, 'Usage: diskpush fleet lists rename NAME NEW-NAME', EXIT.usage)
+    const renamed = await store.renameFleetList(from, to)
+    if (!renamed) return failure(output, `No saved list named ${JSON.stringify(from)}.`, EXIT.configuration)
+    if (output.isJson) output.json({ status: 'ok', list: renamed })
+    else output.line(`Renamed list ${from} to ${renamed.name}.`)
+    return EXIT.ok
+  }
+
+  if (action === 'remove' || action === 'rm') {
+    const name = parsed.positionals[2]
+    if (!name) return failure(output, 'Usage: diskpush fleet lists remove NAME', EXIT.usage)
+    const removed = await store.deleteFleetList(name)
+    if (!removed) return failure(output, `No saved list named ${JSON.stringify(name)}.`, EXIT.configuration)
+    if (output.isJson) output.json({ status: 'ok', removed: name })
+    else output.line(`Removed list ${name}. The servers themselves are untouched.`)
+    return EXIT.ok
+  }
+
+  return failure(output, `Unknown action ${JSON.stringify(action)}. Try: list, show, save, rename, remove.`, EXIT.usage)
+}
+
 async function fleetRuns(parsed: ParsedArgv, store: DiskPushStore, output: Output): Promise<number> {
   const runs = await store.listFleetRuns(numberFlag(parsed, '--limit') ?? 25)
   if (output.isJson) {

apps/cli/src/commands/transfer-helpers.ts

@@ -8,10 +8,7 @@ export {
   planTransfer,
   runPlan,
   runToCompletion,
-  writeSelectionList,
-  describeSelection,
   type ExecutionPlan,
-  type SelectionList,
 } from '@diskpush/rsync-core'
 
 import { runToCompletion, type ExecutionPlan } from '@diskpush/rsync-core'

apps/cli/src/commands/transfer.ts

@@ -7,18 +7,15 @@ import {
   intersectCapabilities,
   planTransfer,
   runPlan,
-  writeSelectionList,
-  describeSelection,
   summarizeChangesFrom,
   type ExecutionPlan,
-  type SelectionList,
 } from './transfer-helpers.js'
 import type { Change, RsyncOptions } from '@diskpush/schemas'
 import { summarizeChanges, topologyOf } from '@diskpush/schemas'
 import { EXIT } from '../exit-codes.js'
 import { estimateRemaining, formatBytes, formatDuration, formatRate, pluralize, table } from '../format.js'
 import { failure, type Output } from '../output.js'
-import { flagValue, flagValues, hasFlag, type ParsedArgv } from '../parse-argv.js'
+import { flagValue, hasFlag, type ParsedArgv } from '../parse-argv.js'
 import { detectLocalCapabilities, optionsFromFlags, resolveEndpoint } from '../resolve.js'
 import type { RsyncCapabilities } from '@diskpush/rsync-core'
 
@@ -64,61 +61,6 @@ export async function runTransfer(
 
   if (alias.deleteMode !== 'off') options.deleteMode = alias.deleteMode
 
-  /*
-   * `--only NAME` transfers just the entries named, rather than everything in
-   * the source directory — the thing an SFTP client makes trivial and a bare
-   * rsync does not.
-   *
-   * The names go to rsync as a NUL-separated `--files-from` list, which is not
-   * bounded by the command-line length limit and can express any name a
-   * filesystem allows. `writeSelectionList` refuses `..` and absolute paths:
-   * a selection is a choice among what the source directory holds, so a name
-   * is the only thing it can be.
-   */
-  const only = flagValues(parsed, '--only')
-  let selection: SelectionList | null = null
-  if (only.length > 0) {
-    if (options.filesFrom) {
-      return failure(output, '--only and --files-from both choose what to send; use one.', EXIT.usage)
-    }
-    try {
-      selection = writeSelectionList(only)
-    } catch (error) {
-      return failure(output, (error as Error).message, EXIT.usage)
-    }
-    options.filesFrom = selection.path
-    options.from0 = true
-  }
-
-  try {
-    return await runResolvedTransfer(
-      command,
-      parsed,
-      store,
-      output,
-      alias,
-      sourceInput,
-      destinationInput,
-      options,
-      only,
-    )
-  } finally {
-    // rsync reads the list at startup, but it is not gone until the run is.
-    selection?.cleanup()
-  }
-}
-
-async function runResolvedTransfer(
-  command: string,
-  parsed: ParsedArgv,
-  store: DiskPushStore,
-  output: Output,
-  alias: (typeof TRANSFER_ALIASES)[string],
-  sourceInput: string,
-  destinationInput: string,
-  options: RsyncOptions,
-  only: readonly string[],
-): Promise<number> {
   const source = await resolveEndpoint(store, sourceInput)
   const destination = await resolveEndpoint(store, destinationInput)
   const topology = topologyOf(source.endpoint, destination.endpoint)
@@ -238,9 +180,6 @@ async function runResolvedTransfer(
   output.line(`DiskPush: ${alias.label} ${describe(sourceInput)} -> ${describe(destinationInput)}`)
   output.line(`Source:      ${sourceInput}`)
   output.line(`Destination: ${destinationInput}`)
-  // Named before the transfer runs, for the same reason a mirror shows its
-  // delete list: what is about to move is worth stating.
-  if (only.length > 0) output.line(`Only:        ${describeSelection(only)}`)
   if (topology === 'remote-to-remote') {
     output.line('')
     output.line(`Direct path:    ${source.endpoint.type === 'ssh' ? source.endpoint.host : '?'} -> ${destination.endpoint.type === 'ssh' ? destination.endpoint.host : '?'}`)

apps/cli/src/parse-argv.ts

@@ -33,7 +33,6 @@ export const VALUE_FLAGS = new Set([
   '--exclude-from',
   '--include-from',
   '--files-from',
-  '--only',
   '--bwlimit',
   '--max-size',
   '--min-size',