From a8bfab107be9371be079524af57a1debbe24c7fe Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Wed, 21 Oct 2026 21:23:10 +0200 Subject: [PATCH 1/2] feat(cli): a git-ignored .labels file labels the namespaces Kates creates Shared clusters often admit a namespace only with the labels their policies require. A .labels file (one key=value per line, git-ignored, with a commented .labels.example) now feeds the same label set as KATES_NAMESPACE_LABELS and --namespace-label, merged per key in that order. Every source is held to the Kubernetes label syntax, and keys under kubernetes.io, k8s.io, helm.sh, strimzi.io and kates.io are refused, except pod-security.kubernetes.io. kates deploy checks the labels before its pre-flight, and --dry-run lists each label with its source. A namespace Kates creates gets the labels, managed-by and a kates.io/namespace-labels annotation in one create. An existing namespace is patched only when a label differs, managed-by is never added to it, and a key the annotation lists is removed once it leaves the labels. When RBAC refuses the create or the patch, the error keeps kubectl's message and adds the manifest or the kubectl label command for a cluster admin. An admission denial keeps only its own message. --verbose no longer hides that message. The labels also reach cert-manager and kyverno when deploy installs them, kates auto, kates kyverno apply, kates detect, migrate lab namespaces, the detect probe namespaces (without Pod Security labels; --no-write-probes skips those probes) and scripts/common.sh ensure_namespace. --labels-file reads another file on each of these commands. --- .gitignore | 1 + .labels.example | 27 ++ cli/cmd/auto.go | 14 + cli/cmd/deploy.go | 30 +- cli/cmd/deploy_components.go | 168 +++++++---- cli/cmd/deploy_components_test.go | 128 -------- cli/cmd/deploy_labels.go | 155 ++++++++++ cli/cmd/deploy_labels_test.go | 471 ++++++++++++++++++++++++++++++ cli/cmd/deploy_plan.go | 4 +- cli/cmd/deploy_preview.go | 17 +- cli/cmd/deploy_ui_test.go | 6 +- cli/cmd/detect.go | 6 + cli/cmd/kyverno_apply.go | 7 + cli/cmd/main_test.go | 11 + cli/cmd/migrate_blocks_test.go | 3 +- cli/cmd/migrate_source.go | 1 + cli/cmd/migrate_up.go | 36 +-- cli/cmd/migrate_up_test.go | 81 ++++- cli/cmd/migrate_verify.go | 1 + cli/pkg/detect/collector.go | 98 +++++-- cli/pkg/detect/collector_test.go | 97 +++++- cli/pkg/detect/renderer_test.go | 8 +- docs/book/10-cli-reference.md | 12 + docs/book/12-deployment.md | 109 +++++++ scripts/common.sh | 19 ++ scripts/deploy-kafka-generic.sh | 3 +- 26 files changed, 1231 insertions(+), 282 deletions(-) create mode 100644 .labels.example create mode 100644 cli/cmd/deploy_labels.go create mode 100644 cli/cmd/deploy_labels_test.go diff --git a/.gitignore b/.gitignore index 55a600b3..a8808710 100644 --- a/.gitignore +++ b/.gitignore @@ -213,6 +213,7 @@ secrets/ .env .env.* !.env.example +.labels credentials.json service-account.json diff --git a/.labels.example b/.labels.example new file mode 100644 index 00000000..2537a014 --- /dev/null +++ b/.labels.example @@ -0,0 +1,27 @@ +# Labels for the namespaces Kates creates or deploys into. +# +# Copy this file to .labels (git-ignored) in the directory you run kates in, +# and uncomment or add lines. kates deploy, kates auto, kates detect, kates +# kyverno apply and the kates migrate commands that create lab namespaces read +# ./.labels, or the file --labels-file names; scripts/common.sh's +# ensure_namespace reads ./.labels. +# +# Format: one key=value per line. Spaces around the key and the value are +# trimmed; blank lines and lines whose first non-space character is # are +# skipped. There are no inline comments and no quotes: a # or a quote inside +# a value is invalid. A repeated key is an error. +# +# Precedence, per key, later wins: .labels, then KATES_NAMESPACE_LABELS +# (comma-separated key=value), then each --namespace-label flag. +# +# Keys: an optional DNS-subdomain prefix and a /, then a name of 1-63 +# characters [A-Za-z0-9-_.] that starts and ends alphanumeric. Values: empty, +# or 1-63 characters under the same rule. +# Reserved prefixes are refused: kubernetes.io, k8s.io, helm.sh, strimzi.io, +# kates.io and their subdomains (app.kubernetes.io/... too), except +# pod-security.kubernetes.io exactly. + +# team=payments +# segment=payments +# cost-center=4711 +# pod-security.kubernetes.io/enforce=restricted diff --git a/cli/cmd/auto.go b/cli/cmd/auto.go index 9ffef057..fa1c0295 100644 --- a/cli/cmd/auto.go +++ b/cli/cmd/auto.go @@ -42,17 +42,24 @@ func init() { autoCmd.Flags().BoolVar(&autoDryRun, "dry-run", false, "Preview the Helm installation without executing it") autoCmd.Flags().StringVarP(&autoBaseValues, "values", "f", "", "Path to base values.yaml to merge with detected values") autoCmd.Flags().BoolVar(&autoSkipMonitoring, "skip-monitoring", false, "Skip deploying the monitoring stack (Prometheus + Grafana)") + addLabelsFileFlag(autoCmd) rootCmd.AddCommand(autoCmd) } func runAuto(cmd *cobra.Command, args []string) error { output.Header("Kates Auto-Deploy") + namespaceLabels, _, err := namespaceUserLabels() + if err != nil { + return err + } + executor := defaultExecutor collector := detect.NewCollector(executor) // Introspection creates a namespace, a Secret and prober pods. A dry run // collects without them, as kates deploy --dry-run does. collector.ReadOnly = autoDryRun + collector.NamespaceLabels = namespaceLabels if err := collector.Preflight(); err != nil { return cmdErr(fmt.Sprintf("Preflight failed: %v", err)) @@ -118,6 +125,13 @@ func runAuto(cmd *cobra.Command, args []string) error { // 4. Helm Deploy Phase output.Hint(fmt.Sprintf("๐Ÿ“Œ Target namespace: %s", autoNamespace)) + // Created with the namespace labels before Helm's --create-namespace + // could make it bare; the monitoring release shares it. + if !autoDryRun { + if err := ensureConfiguredDeployNamespace(context.Background(), autoNamespace); err != nil { + return err + } + } helmArgs := []string{ "upgrade", "--install", autoReleaseName, autoChartDir, "--namespace", autoNamespace, "--create-namespace", diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index 1e3f5e4e..42c4d43d 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "io" "os" "os/exec" "strings" @@ -70,6 +71,7 @@ var ( deployWithMirrorMaker2 bool deployMM2NS string deployNamespaceLabels []string + deployNoWriteProbes bool deployYes bool // Versions and scope (multi-version plan ยง3.1โ€“3.3). @@ -104,7 +106,9 @@ func init() { deployCmd.Flags().BoolVar(&deployWithKafkaUI, "with-kafka-ui", true, "Deploy Kafka UI for browser-based cluster monitoring") deployCmd.Flags().BoolVar(&deployWithMirrorMaker2, "with-mirror-maker2", false, "Deploy MirrorMaker 2 as a loopback mirror of the primary (replication lab; kates migrate for real sources)") deployCmd.Flags().StringVar(&deployMM2NS, "mm2-ns", "kafka", "Namespace for MirrorMaker 2 when topology is 'isolated'") - deployCmd.Flags().StringArrayVar(&deployNamespaceLabels, "namespace-label", nil, "Label to apply to namespaces created by deploy (key=value, repeatable; KATES_NAMESPACE_LABELS also accepts comma-separated key=value entries)") + deployCmd.Flags().StringArrayVar(&deployNamespaceLabels, "namespace-label", nil, "Label to apply to the namespaces deploy ensures (key=value, repeatable; overrides KATES_NAMESPACE_LABELS, which overrides the labels file)") + addLabelsFileFlag(deployCmd) + deployCmd.Flags().BoolVar(&deployNoWriteProbes, "no-write-probes", false, "Skip the introspection probes that create a namespace, a Secret or pods (as --dry-run does)") deployCmd.Flags().BoolVarP(&deployInteractive, "interactive", "i", false, "Use interactive UI to configure deployment") deployCmd.Flags().BoolVar(&deployVerbose, "verbose", false, "Show every kubectl/helm command as it runs") deployCmd.Flags().BoolVarP(&deployPortForward, "port-forward", "P", false, "After deploy, run kates ports: forward every service in the background and make the ports context current") @@ -125,6 +129,12 @@ func runDeploy(cmd *cobra.Command, args []string) error { dl = &DashboardController{} + // A bad namespace label fails before anything asks the cluster. + namespaceLabels, namespaceLabelSources, err := namespaceUserLabels() + if err != nil { + return err + } + // โ”€โ”€ Target cluster โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ // This runs FIRST, before the banner and the wizard. There is no point // asking eight questions about a deployment and only then discovering @@ -170,8 +180,9 @@ func runDeploy(cmd *cobra.Command, args []string) error { // Secret to audit Secret creation, prober pods to measure inter-AZ // latency, and a dns-detect pod when no running pod can be read. The // preview uses none of that, so a dry run collects without it. - collector.ReadOnly = deployDryRun + collector.ReadOnly = deployDryRun || deployNoWriteProbes collector.KubeContext = target + collector.NamespaceLabels = namespaceLabels if err := collector.Preflight(); err != nil { output.Error(fmt.Sprintf("Preflight failed: %v", err)) return err @@ -253,7 +264,7 @@ func runDeploy(cmd *cobra.Command, args []string) error { // โ”€โ”€ Dry-run: show preview and exit โ”€โ”€ if deployDryRun { - return runDeployDryRun(sharedEntries) + return runDeployDryRun(sharedEntries, namespaceLabels, namespaceLabelSources) } // An operator upgrade or a scope change rolls things; say so and ask. @@ -477,9 +488,18 @@ func runExecStdinDefault(ctx context.Context, name string, args []string, stdinD if deployVerbose { dl.Printf(" %s\n", output.DimStyle.Render("$ "+name+" "+strings.Join(args, " "))) + // stderr is also kept for the error, so a caller can still tell why + // kubectl failed (ensureDeployNamespace looks for an RBAC refusal). + var errBuf bytes.Buffer cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf) + if runErr := cmd.Run(); runErr != nil { + if errMsg := strings.TrimSpace(errBuf.String()); errMsg != "" { + return fmt.Errorf("%w: %s", runErr, errMsg) + } + return runErr + } + return nil } var outBuf, errBuf bytes.Buffer diff --git a/cli/cmd/deploy_components.go b/cli/cmd/deploy_components.go index 563b67e1..3c9dfeaa 100644 --- a/cli/cmd/deploy_components.go +++ b/cli/cmd/deploy_components.go @@ -6,16 +6,18 @@ import ( "encoding/json" "fmt" "io" + "maps" "net" "net/url" "os" "path/filepath" - "sort" + "slices" "strconv" "strings" "time" "github.com/bmscomp/kates/cli/output" + "github.com/bmscomp/kates/cli/pkg/detect" "github.com/bmscomp/kates/cli/pkg/strimzi" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" @@ -198,20 +200,15 @@ func ensureKubernetesNoProxy(noProxy, clusterDomain string) string { ) } +// deployNamespaceLabelMap is the user's namespace labels plus the built-in +// managed-by label, which ensureDeployNamespace sets only on a namespace it +// creates. func deployNamespaceLabelMap() (map[string]string, error) { - labels := map[string]string{ - "app.kubernetes.io/managed-by": "kates", - } - if raw := strings.TrimSpace(os.Getenv("KATES_NAMESPACE_LABELS")); raw != "" { - if err := addDeployNamespaceLabelSpecs(labels, raw); err != nil { - return nil, err - } - } - for _, raw := range deployNamespaceLabels { - if err := addDeployNamespaceLabelSpecs(labels, raw); err != nil { - return nil, err - } + labels, _, err := namespaceUserLabels() + if err != nil { + return nil, err } + labels[managedByLabel] = "kates" return labels, nil } @@ -250,81 +247,120 @@ func monitoringNodeExporterDaemonSetScheduled(ctx context.Context, namespace str return err == nil && desired > 0 } -func addDeployNamespaceLabelSpecs(labels map[string]string, raw string) error { - for _, spec := range strings.Split(raw, ",") { - spec = strings.TrimSpace(spec) - if spec == "" { - continue - } - key, value, ok := strings.Cut(spec, "=") - key = strings.TrimSpace(key) - value = strings.TrimSpace(value) - if !ok || key == "" || value == "" { - return fmt.Errorf("invalid namespace label %q: expected key=value", spec) - } - labels[key] = value +// deployNamespaceManifest is a Namespace carrying labels, and the annotation +// that records which of them are the user's (every key but managed-by). +func deployNamespaceManifest(name string, labels map[string]string) string { + var annotations map[string]string + if user := userNamespaceLabels(labels); len(user) > 0 { + annotations = map[string]string{namespaceLabelsAnnotation: strings.Join(slices.Sorted(maps.Keys(user)), ",")} } - return nil + return detect.NamespaceManifest(name, labels, annotations) } -func deployNamespaceManifest(name string, labels map[string]string) string { - var b strings.Builder - fmt.Fprintf(&b, "apiVersion: v1\nkind: Namespace\nmetadata:\n name: %s\n", name) - if len(labels) > 0 { - b.WriteString(" labels:\n") - keys := make([]string, 0, len(labels)) - for key := range labels { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - fmt.Fprintf(&b, " %s: %s\n", key, strconv.Quote(labels[key])) - } - } - return b.String() +func userNamespaceLabels(labels map[string]string) map[string]string { + user := maps.Clone(labels) + delete(user, managedByLabel) + return user } +// ensureDeployNamespace creates a missing namespace with labels, or brings an +// existing one's user labels in line with them. On an existing namespace it +// never touches managed-by (the namespace is not Kates'), and removes only +// keys its annotation says Kates set. func ensureDeployNamespace(ctx context.Context, namespace string, labels map[string]string) error { namespace = strings.TrimSpace(namespace) if namespace == "" { return nil } - exists, err := deployNamespaceExists(ctx, namespace) + out, err := runExecOutputFn(ctx, "kubectl", "get", "namespace", namespace, "--ignore-not-found", "-o", "json") if err != nil { return fmt.Errorf("check namespace %s: %w", namespace, err) } - if !exists { - if err := runExecStdinFn(ctx, "kubectl", []string{"create", "-f", "-"}, deployNamespaceManifest(namespace, labels)); err != nil { + if strings.TrimSpace(string(out)) == "" { + manifest := deployNamespaceManifest(namespace, labels) + if err := runExecStdinFn(ctx, "kubectl", []string{"create", "-f", "-"}, manifest); err != nil { + if rbacRefused(err, "create") { + return fmt.Errorf("create namespace %s: %w\n\nThis account may not create namespaces. Ask a cluster admin to apply this manifest (kubectl create -f ), then run again:\n\n%s", namespace, err, manifest) + } return fmt.Errorf("create namespace %s: %w", namespace, err) } return nil } - if len(labels) == 0 { + + var current struct { + Metadata struct{ Labels, Annotations map[string]string } + } + if err := json.Unmarshal(out, ¤t); err != nil { + return fmt.Errorf("check namespace %s: %w", namespace, err) + } + user, have := userNamespaceLabels(labels), current.Metadata.Labels + set := map[string]string{} + for key, value := range user { + if old, ok := have[key]; !ok || old != value { + set[key] = value + } + } + var remove, tracked []string + for _, key := range strings.Split(current.Metadata.Annotations[namespaceLabelsAnnotation], ",") { + key = strings.TrimSpace(key) + if _, wanted := user[key]; wanted { + tracked = append(tracked, key) + } else if _, present := have[key]; key != "" && present { + remove = append(remove, key) + } + } + // No call when nothing changes, so an account that may not patch + // namespaces still passes once the labels are right. + if len(set) == 0 && len(remove) == 0 { return nil } - labelArgs := []string{"label", "namespace", namespace} - keys := make([]string, 0, len(labels)) - for key := range labels { - keys = append(keys, key) + labelPatch := map[string]any{} + for key, value := range set { + labelPatch[key] = value + if !slices.Contains(tracked, key) { + tracked = append(tracked, key) + } } - sort.Strings(keys) - for _, key := range keys { - labelArgs = append(labelArgs, key+"="+labels[key]) + for _, key := range remove { + labelPatch[key] = nil } - labelArgs = append(labelArgs, "--overwrite") - if out, err := runExecCombinedFn(ctx, "kubectl", labelArgs...); err != nil { - return commandErrorWithOutput("label namespace "+namespace, err, out) + var annotation any // null removes it + if len(tracked) > 0 { + slices.Sort(tracked) + annotation = strings.Join(tracked, ",") + } + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{ + "labels": labelPatch, + "annotations": map[string]any{namespaceLabelsAnnotation: annotation}, + }}) + if err != nil { + return err + } + if out, err := runExecCombinedFn(ctx, "kubectl", "patch", "namespace", namespace, "--type", "merge", "-p", string(patch)); err != nil { + err = commandErrorWithOutput("patch namespace "+namespace, err, out) + if !rbacRefused(err, "patch") { + return err + } + args := []string{"kubectl", "label", "namespace", namespace} + for _, key := range slices.Sorted(maps.Keys(set)) { + args = append(args, key+"="+set[key]) + } + slices.Sort(remove) + for _, key := range remove { + args = append(args, key+"-") + } + return fmt.Errorf("%w\n\nThis account may not change the labels of namespace %s. Ask a cluster admin to run this, then run again:\n\n %s", err, namespace, strings.Join(append(args, "--overwrite"), " ")) } return nil } -func deployNamespaceExists(ctx context.Context, namespace string) (bool, error) { - out, err := runExecOutputFn(ctx, "kubectl", "get", "namespace", namespace, "--ignore-not-found", "-o", "jsonpath={.metadata.name}") - if err != nil { - return false, err - } - return strings.TrimSpace(string(out)) == namespace, nil +// rbacRefused says whether RBAC refused verb on a resource, as the API +// server words it ("User ... cannot create resource ..."). An admission +// webhook or policy denial also says Forbidden, but an admin running the same +// request is refused the same way, so it gets no admin instruction. +func rbacRefused(err error, verb string) bool { + return strings.Contains(err.Error(), "cannot "+verb+" resource") } func ensureConfiguredDeployNamespace(ctx context.Context, namespace string) error { @@ -772,6 +808,11 @@ func deployGroupA(dc *deployContext) error { dl.Printf("\n๐Ÿ“ฆ Deploying Cert-Manager (Namespace: %s)...\n", "cert-manager") runHelmFn(gCtx, "repo", "add", "jetstack", "https://charts.jetstack.io") runHelmFn(gCtx, "repo", "update", "jetstack") + // Created here with the namespace labels; --create-namespace + // would make it bare. + if err := ensureConfiguredDeployNamespace(gCtx, "cert-manager"); err != nil { + return err + } // global.clusterDomain ensures cert-manager generates webhook TLS certificates // and service references using the actual cluster DNS domain (not always cluster.local). // report.Network.ClusterDomain is detected from the live cluster by kates detect. @@ -914,6 +955,9 @@ spec: dl.Println("\n๐Ÿ“ฆ Deploying Kyverno (Namespace: kyverno)...") runHelmFn(gCtx, "repo", "add", "kyverno", "https://kyverno.github.io/kyverno/") runHelmFn(gCtx, "repo", "update", "kyverno") + if err := ensureConfiguredDeployNamespace(gCtx, "kyverno"); err != nil { + return err + } // Kyverno v3.x splits into 4 controllers; replicaCount=1 is a v2.x flag. // global.clusterDomain ensures Kyverno webhook certificates use the diff --git a/cli/cmd/deploy_components_test.go b/cli/cmd/deploy_components_test.go index 3894c8db..d1e94f6d 100644 --- a/cli/cmd/deploy_components_test.go +++ b/cli/cmd/deploy_components_test.go @@ -229,134 +229,6 @@ func TestDetectDeployProxySettingsLoopbackHandling(t *testing.T) { } } -func TestDeployNamespaceLabelMap(t *testing.T) { - orig := deployNamespaceLabels - t.Cleanup(func() { deployNamespaceLabels = orig }) - - t.Setenv("KATES_NAMESPACE_LABELS", "gov.example.com/app-name=kates,gov.example.com/product-id=KATES1") - deployNamespaceLabels = []string{ - "cyber.example.com/sensitivity=2", - "gov.example.com/product-id=OVR123", - } - - labels, err := deployNamespaceLabelMap() - if err != nil { - t.Fatalf("deployNamespaceLabelMap: %v", err) - } - want := map[string]string{ - "app.kubernetes.io/managed-by": "kates", - "gov.example.com/app-name": "kates", - "gov.example.com/product-id": "OVR123", - "cyber.example.com/sensitivity": "2", - } - if !reflect.DeepEqual(labels, want) { - t.Fatalf("labels = %v, want %v", labels, want) - } - - deployNamespaceLabels = []string{"bad"} - if _, err := deployNamespaceLabelMap(); err == nil || !strings.Contains(err.Error(), "expected key=value") { - t.Fatalf("invalid label error = %v, want key=value", err) - } -} - -func TestDeployNamespaceManifestIncludesLabels(t *testing.T) { - got := deployNamespaceManifest("monitoring", map[string]string{ - "gov.example.com/product-id": "KATES1", - "cyber.example.com/sensitivity": "2", - "app.kubernetes.io/managed-by": "kates", - }) - for _, want := range []string{ - "kind: Namespace", - "name: monitoring", - " app.kubernetes.io/managed-by: \"kates\"", - " cyber.example.com/sensitivity: \"2\"", - " gov.example.com/product-id: \"KATES1\"", - } { - if !strings.Contains(got, want) { - t.Fatalf("namespace manifest lacks %q:\n%s", want, got) - } - } -} - -func TestEnsureDeployNamespaceLabelsExistingNamespaceWithoutApply(t *testing.T) { - origOutput, origStdin, origCombined := runExecOutputFn, runExecStdinFn, runExecCombinedFn - t.Cleanup(func() { - runExecOutputFn, runExecStdinFn, runExecCombinedFn = origOutput, origStdin, origCombined - }) - - var calls []string - runExecOutputFn = func(_ context.Context, name string, args ...string) ([]byte, error) { - calls = append(calls, name+" "+strings.Join(args, " ")) - return []byte("kafka"), nil - } - runExecStdinFn = func(_ context.Context, name string, args []string, _ string) error { - t.Fatalf("existing namespace must not use stdin apply/create path: %s %s", name, strings.Join(args, " ")) - return nil - } - runExecCombinedFn = func(_ context.Context, name string, args ...string) ([]byte, error) { - calls = append(calls, name+" "+strings.Join(args, " ")) - return nil, nil - } - - err := ensureDeployNamespace(context.Background(), "kafka", map[string]string{ - "gov.example.com/product-id": "KATES1", - "app.kubernetes.io/managed-by": "kates", - }) - if err != nil { - t.Fatalf("ensureDeployNamespace returned error: %v", err) - } - - want := []string{ - "kubectl get namespace kafka --ignore-not-found -o jsonpath={.metadata.name}", - "kubectl label namespace kafka app.kubernetes.io/managed-by=kates gov.example.com/product-id=KATES1 --overwrite", - } - if !reflect.DeepEqual(calls, want) { - t.Fatalf("calls = %v, want %v", calls, want) - } -} - -func TestEnsureDeployNamespaceCreatesMissingNamespaceWithLabels(t *testing.T) { - origOutput, origStdin, origCombined := runExecOutputFn, runExecStdinFn, runExecCombinedFn - t.Cleanup(func() { - runExecOutputFn, runExecStdinFn, runExecCombinedFn = origOutput, origStdin, origCombined - }) - - var calls []string - var createdManifest string - runExecOutputFn = func(_ context.Context, name string, args ...string) ([]byte, error) { - calls = append(calls, name+" "+strings.Join(args, " ")) - return nil, nil - } - runExecStdinFn = func(_ context.Context, name string, args []string, stdinData string) error { - calls = append(calls, name+" "+strings.Join(args, " ")) - createdManifest = stdinData - return nil - } - runExecCombinedFn = func(_ context.Context, name string, args ...string) ([]byte, error) { - t.Fatalf("missing namespace must be created from a labelled manifest, not labelled afterwards: %s %s", name, strings.Join(args, " ")) - return nil, nil - } - - err := ensureDeployNamespace(context.Background(), "connect", map[string]string{ - "gov.example.com/product-id": "KATES1", - "app.kubernetes.io/managed-by": "kates", - }) - if err != nil { - t.Fatalf("ensureDeployNamespace returned error: %v", err) - } - - want := []string{ - "kubectl get namespace connect --ignore-not-found -o jsonpath={.metadata.name}", - "kubectl create -f -", - } - if !reflect.DeepEqual(calls, want) { - t.Fatalf("calls = %v, want %v", calls, want) - } - if !strings.Contains(createdManifest, "name: connect") || !strings.Contains(createdManifest, "gov.example.com/product-id: \"KATES1\"") { - t.Fatalf("create manifest missing namespace labels:\n%s", createdManifest) - } -} - func TestRunKatesHelmUpgradeRecoversFromPostgresImmutableStatefulSet(t *testing.T) { origHelm, origCombined := runHelmFn, runExecCombinedFn t.Cleanup(func() { diff --git a/cli/cmd/deploy_labels.go b/cli/cmd/deploy_labels.go new file mode 100644 index 00000000..ee00e3d2 --- /dev/null +++ b/cli/cmd/deploy_labels.go @@ -0,0 +1,155 @@ +package cmd + +import ( + "errors" + "fmt" + "io/fs" + "os" + "regexp" + "strings" + + "github.com/spf13/cobra" +) + +const ( + managedByLabel = "app.kubernetes.io/managed-by" + // namespaceLabelsAnnotation lists the label keys Kates set on a namespace + // from the user's labels: the only keys it may later remove. + namespaceLabelsAnnotation = "kates.io/namespace-labels" + defaultLabelsFile = ".labels" + namespaceLabelsEnv = "KATES_NAMESPACE_LABELS" + namespaceLabelFlag = "--namespace-label" +) + +// namespaceLabelsFile is --labels-file. Empty reads ./.labels when it exists. +var namespaceLabelsFile string + +var ( + labelNamePattern = regexp.MustCompile(`^[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$`) + labelPrefixPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`) + // Kubernetes, Helm, Strimzi and Kates own these prefixes; a user label + // there could impersonate or overwrite theirs. + reservedLabelDomains = []string{"kubernetes.io", "k8s.io", "helm.sh", "strimzi.io", "kates.io"} +) + +// addLabelsFileFlag registers --labels-file on a command that creates +// namespaces; every such command shares namespaceLabelsFile. +func addLabelsFileFlag(cmd *cobra.Command) { + cmd.Flags().StringVar(&namespaceLabelsFile, "labels-file", "", "File of key=value labels for the namespaces Kates creates (default: ./.labels, when it exists)") +} + +// namespaceUserLabels merges the user's namespace labels, later sources +// winning per key: the labels file, then KATES_NAMESPACE_LABELS, then +// --namespace-label. sources says where each key's value came from. +func namespaceUserLabels() (labels, sources map[string]string, err error) { + labels, sources = map[string]string{}, map[string]string{} + path, required := namespaceLabelsFile, true + if path == "" { + path, required = defaultLabelsFile, false + } + data, err := os.ReadFile(path) + switch { + case err == nil: + if err := parseNamespaceLabelsFile(path, string(data), labels, sources); err != nil { + return nil, nil, err + } + case required || !errors.Is(err, fs.ErrNotExist): + return nil, nil, fmt.Errorf("labels file: %w", err) + } + if raw := strings.TrimSpace(os.Getenv(namespaceLabelsEnv)); raw != "" { + if err := addDeployNamespaceLabelSpecs(labels, sources, raw, namespaceLabelsEnv); err != nil { + return nil, nil, err + } + } + for _, raw := range deployNamespaceLabels { + if err := addDeployNamespaceLabelSpecs(labels, sources, raw, namespaceLabelFlag); err != nil { + return nil, nil, err + } + } + return labels, sources, nil +} + +// parseNamespaceLabelsFile reads key=value lines. Blank lines and # comment +// lines are skipped; nothing is unquoted, so a quote or # in a value fails +// validation instead of being silently kept or dropped. +func parseNamespaceLabelsFile(path, data string, labels, sources map[string]string) error { + firstLine := map[string]int{} + for i, line := range strings.Split(data, "\n") { + n := i + 1 + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + where := fmt.Sprintf("%s:%d", path, n) + key, value, ok := strings.Cut(line, "=") + if !ok { + return fmt.Errorf("%s: invalid namespace label %q: expected key=value", where, line) + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if first, seen := firstLine[key]; seen { + return fmt.Errorf("%s: label %q is already set on line %d", where, key, first) + } + if err := validateNamespaceLabel(key, value); err != nil { + return fmt.Errorf("%s: %w", where, err) + } + firstLine[key] = n + labels[key], sources[key] = value, where + } + return nil +} + +func addDeployNamespaceLabelSpecs(labels, sources map[string]string, raw, source string) error { + for _, spec := range strings.Split(raw, ",") { + spec = strings.TrimSpace(spec) + if spec == "" { + continue + } + key, value, ok := strings.Cut(spec, "=") + if !ok { + return fmt.Errorf("%s: invalid namespace label %q: expected key=value", source, spec) + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if err := validateNamespaceLabel(key, value); err != nil { + return fmt.Errorf("%s: %w", source, err) + } + labels[key], sources[key] = value, source + } + return nil +} + +// validateNamespaceLabel applies the API server's label syntax, so a bad +// label fails here rather than halfway through a deploy, and refuses the +// prefixes Kubernetes and the charts Kates installs own. +func validateNamespaceLabel(key, value string) error { + name := key + if prefix, rest, ok := strings.Cut(key, "/"); ok { + name = rest + switch { + case prefix == "": + return fmt.Errorf("label key %q: the prefix before '/' is empty", key) + case len(prefix) > 253 || !labelPrefixPattern.MatchString(prefix): + return fmt.Errorf("label key %q: prefix %q must be a lowercase DNS subdomain of at most 253 characters", key, prefix) + case reservedLabelPrefix(prefix): + return fmt.Errorf("label key %q: prefix %q is reserved (kubernetes.io, k8s.io, helm.sh, strimzi.io, kates.io; only pod-security.kubernetes.io is allowed)", key, prefix) + } + } + if len(name) > 63 || !labelNamePattern.MatchString(name) { + return fmt.Errorf("label key %q: name %q must be 1-63 characters of letters, digits, '-', '_' or '.', starting and ending with a letter or digit", key, name) + } + if len(value) > 63 || value != "" && !labelNamePattern.MatchString(value) { + return fmt.Errorf("label %q: value %q must be empty or 1-63 characters of letters, digits, '-', '_' or '.', starting and ending with a letter or digit", key, value) + } + return nil +} + +func reservedLabelPrefix(prefix string) bool { + if prefix == "pod-security.kubernetes.io" { + return false + } + for _, domain := range reservedLabelDomains { + if prefix == domain || strings.HasSuffix(prefix, "."+domain) { + return true + } + } + return false +} diff --git a/cli/cmd/deploy_labels_test.go b/cli/cmd/deploy_labels_test.go new file mode 100644 index 00000000..60299339 --- /dev/null +++ b/cli/cmd/deploy_labels_test.go @@ -0,0 +1,471 @@ +package cmd + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// useLabelsFile points --labels-file at path ("" is the optional ./.labels) +// and clears --namespace-label for the test. +func useLabelsFile(t *testing.T, path string) { + t.Helper() + origFile, origFlags := namespaceLabelsFile, deployNamespaceLabels + t.Cleanup(func() { namespaceLabelsFile, deployNamespaceLabels = origFile, origFlags }) + namespaceLabelsFile, deployNamespaceLabels = path, nil +} + +// writeLabelsFile writes content to a labels file in a temp dir and returns +// its path. +func writeLabelsFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "team.labels") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestNamespaceLabelsFileFormat(t *testing.T) { + path := writeLabelsFile(t, "# owner of the cluster\n\n gov.example.com/product-id = KATES1 \r\n # indented comment\r\ncyber.example.com/sensitivity=2\r\nteam=\n") + useLabelsFile(t, path) + labels, sources, err := namespaceUserLabels() + if err != nil { + t.Fatal(err) + } + want := map[string]string{"gov.example.com/product-id": "KATES1", "cyber.example.com/sensitivity": "2", "team": ""} + if !reflect.DeepEqual(labels, want) { + t.Errorf("labels = %v, want %v", labels, want) + } + wantSources := map[string]string{"gov.example.com/product-id": path + ":3", "cyber.example.com/sensitivity": path + ":5", "team": path + ":6"} + if !reflect.DeepEqual(sources, wantSources) { + t.Errorf("sources = %v, want %v", sources, wantSources) + } +} + +func TestNamespaceLabelsFileErrors(t *testing.T) { + for _, tc := range []struct{ name, content, want string }{ + {"missing =", "a=b\nteam\n", ":2: invalid namespace label \"team\": expected key=value"}, + {"duplicate key", "a=b\n# x\na=c\n", `:3: label "a" is already set on line 1`}, + {"quoted value", "a=\"b\"\n", ":1: label \"a\": value"}, + {"inline comment", "a=b # owner\n", ":1: label \"a\": value"}, + {"empty key", "=b\n", ":1: label key \"\""}, + } { + t.Run(tc.name, func(t *testing.T) { + path := writeLabelsFile(t, tc.content) + useLabelsFile(t, path) + _, _, err := namespaceUserLabels() + if err == nil || !strings.HasPrefix(err.Error(), path+":") || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q prefixed by the path", err, tc.want) + } + }) + } +} + +// A missing ./.labels is no labels; a file --labels-file names must exist. +func TestNamespaceLabelsFileMissing(t *testing.T) { + t.Chdir(t.TempDir()) + useLabelsFile(t, "") + labels, _, err := namespaceUserLabels() + if err != nil || len(labels) != 0 { + t.Fatalf("absent ./.labels: labels = %v, err = %v; want none and no error", labels, err) + } + + if err := os.WriteFile(".labels", []byte("team=payments\n"), 0o600); err != nil { + t.Fatal(err) + } + labels, sources, err := namespaceUserLabels() + if err != nil || labels["team"] != "payments" || sources["team"] != ".labels:1" { + t.Fatalf("./.labels: labels = %v, sources = %v, err = %v", labels, sources, err) + } + + useLabelsFile(t, "missing.labels") + if _, _, err := namespaceUserLabels(); err == nil || !strings.Contains(err.Error(), "missing.labels") { + t.Fatalf("explicit missing file: err = %v, want an error naming it", err) + } +} + +func TestValidateNamespaceLabel(t *testing.T) { + long := func(n int) string { return strings.Repeat("a", n) } + for _, tc := range []struct { + key, value string + ok bool + }{ + {"team", "payments", true}, + {"team", "", true}, + {"A_b.c-9", "X_y.z-1", true}, + {long(63), long(63), true}, + {long(253) + "/name", "v", true}, + {"gov.example.com/product-id", "KATES1", true}, + {"pod-security.kubernetes.io/enforce", "restricted", true}, + + {long(64), "v", false}, + {"team", long(64), false}, + {long(254) + "/name", "v", false}, + {"/name", "v", false}, + {"Example.com/name", "v", false}, + {"example.com/", "v", false}, + {"a/b/c", "v", false}, + {"-team", "v", false}, + {"team-", "v", false}, + {"team", "-x", false}, + {"team", `"x"`, false}, + {"team", "a#b", false}, + {"team", "a b", false}, + {"kubernetes.io/name", "v", false}, + {"app.kubernetes.io/name", "v", false}, + {"node.kubernetes.io/role", "v", false}, + {"k8s.io/name", "v", false}, + {"x.k8s.io/name", "v", false}, + {"helm.sh/chart", "v", false}, + {"strimzi.io/cluster", "v", false}, + {"kates.io/lab", "v", false}, + {"team.kates.io/lab", "v", false}, + {"x.pod-security.kubernetes.io/enforce", "v", false}, + } { + err := validateNamespaceLabel(tc.key, tc.value) + if (err == nil) != tc.ok { + t.Errorf("validateNamespaceLabel(%q, %q) = %v, want ok=%v", tc.key, tc.value, err, tc.ok) + } + } +} + +// Every source validates, and every error says which source to fix. +func TestNamespaceLabelErrorsNameTheirSource(t *testing.T) { + useLabelsFile(t, writeLabelsFile(t, "team=ok\n")) + + t.Setenv(namespaceLabelsEnv, "team=ok,app.kubernetes.io/name=x") + if _, _, err := namespaceUserLabels(); err == nil || !strings.HasPrefix(err.Error(), "KATES_NAMESPACE_LABELS: ") || !strings.Contains(err.Error(), "reserved") { + t.Errorf("env: err = %v", err) + } + + t.Setenv(namespaceLabelsEnv, "") + deployNamespaceLabels = []string{"bad"} + if _, _, err := namespaceUserLabels(); err == nil || !strings.HasPrefix(err.Error(), "--namespace-label: ") || !strings.Contains(err.Error(), "expected key=value") { + t.Errorf("flag: err = %v", err) + } + + // Empty values, which the variable and the flag used to refuse. + t.Setenv(namespaceLabelsEnv, "a=") + deployNamespaceLabels = []string{"b="} + labels, _, err := namespaceUserLabels() + if err != nil || labels["a"] != "" || labels["b"] != "" || len(labels) != 3 { + t.Errorf("empty values: labels = %v, err = %v", labels, err) + } +} + +func TestNamespaceLabelPrecedence(t *testing.T) { + path := writeLabelsFile(t, "a=file\nb=file\nc=file\n") + useLabelsFile(t, path) + t.Setenv(namespaceLabelsEnv, "b=env,c=env,d=env1,d=env2") + deployNamespaceLabels = []string{"c=flag1", "c=flag2"} + + labels, sources, err := namespaceUserLabels() + if err != nil { + t.Fatal(err) + } + want := map[string]string{"a": "file", "b": "env", "c": "flag2", "d": "env2"} + wantSources := map[string]string{"a": path + ":1", "b": namespaceLabelsEnv, "c": namespaceLabelFlag, "d": namespaceLabelsEnv} + if !reflect.DeepEqual(labels, want) || !reflect.DeepEqual(sources, wantSources) { + t.Errorf("labels = %v, sources = %v; want %v, %v", labels, sources, want, wantSources) + } + + all, err := deployNamespaceLabelMap() + if err != nil || all[managedByLabel] != "kates" || len(all) != len(want)+1 { + t.Errorf("deployNamespaceLabelMap = %v, %v; want the user labels and managed-by", all, err) + } +} + +func TestDeployNamespaceManifest(t *testing.T) { + got := deployNamespaceManifest("monitoring", map[string]string{ + "gov.example.com/product-id": "KATES1", + "team": "", + managedByLabel: "kates", + }) + want := `apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + annotations: + "kates.io/namespace-labels": "gov.example.com/product-id,team" + labels: + "app.kubernetes.io/managed-by": "kates" + "gov.example.com/product-id": "KATES1" + "team": "" +` + if got != want { + t.Errorf("manifest:\n%s\nwant:\n%s", got, want) + } + // managed-by alone is not the user's: no annotation. + if got := deployNamespaceManifest("kafka", map[string]string{managedByLabel: "kates"}); strings.Contains(got, "annotations") { + t.Errorf("manifest with no user labels has an annotation:\n%s", got) + } +} + +// namespaceKubectl fakes the three kubectl runners ensureDeployNamespace +// uses: get answers with existing (empty is a missing namespace) or fails +// with getErr, create and patch fail with createErr and patchErr. A failed +// patch prints patchOut, an RBAC refusal unless set. +type namespaceKubectl struct { + existing string + getErr, createErr, patchErr error + patchOut string + calls []string + manifest string +} + +func fakeNamespaceKubectl(t *testing.T, existing string) *namespaceKubectl { + t.Helper() + k := &namespaceKubectl{existing: existing} + origOutput, origStdin, origCombined := runExecOutputFn, runExecStdinFn, runExecCombinedFn + t.Cleanup(func() { runExecOutputFn, runExecStdinFn, runExecCombinedFn = origOutput, origStdin, origCombined }) + runExecOutputFn = func(_ context.Context, name string, args ...string) ([]byte, error) { + k.calls = append(k.calls, name+" "+strings.Join(args, " ")) + return []byte(k.existing), k.getErr + } + runExecStdinFn = func(_ context.Context, name string, args []string, stdin string) error { + k.calls = append(k.calls, name+" "+strings.Join(args, " ")) + k.manifest = stdin + return k.createErr + } + runExecCombinedFn = func(_ context.Context, name string, args ...string) ([]byte, error) { + k.calls = append(k.calls, name+" "+strings.Join(args, " ")) + if k.patchErr != nil { + if k.patchOut != "" { + return []byte(k.patchOut), k.patchErr + } + return []byte(`Error from server (Forbidden): namespaces "kafka" is forbidden: User "dev" cannot patch resource "namespaces"`), k.patchErr + } + return nil, nil + } + return k +} + +// patch decodes the JSON of the one kubectl patch call. +func (k *namespaceKubectl) patch(t *testing.T) map[string]any { + t.Helper() + if len(k.calls) != 2 || !strings.HasPrefix(k.calls[1], "kubectl patch namespace kafka --type merge -p ") { + t.Fatalf("calls = %q, want the lookup and one merge patch", k.calls) + } + var patch map[string]any + if err := json.Unmarshal([]byte(strings.TrimPrefix(k.calls[1], "kubectl patch namespace kafka --type merge -p ")), &patch); err != nil { + t.Fatal(err) + } + return patch +} + +const getKafkaNamespace = "kubectl get namespace kafka --ignore-not-found -o json" + +func existingNamespace(labels, annotations map[string]string) string { + data, _ := json.Marshal(map[string]any{"metadata": map[string]any{"name": "kafka", "labels": labels, "annotations": annotations}}) + return string(data) +} + +var kafkaLabels = map[string]string{"gov.example.com/product-id": "KATES1", managedByLabel: "kates"} + +func TestEnsureDeployNamespaceCreatesAMissingOneLabelled(t *testing.T) { + k := fakeNamespaceKubectl(t, "") + if err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels); err != nil { + t.Fatal(err) + } + if want := []string{getKafkaNamespace, "kubectl create -f -"}; !reflect.DeepEqual(k.calls, want) { + t.Fatalf("calls = %q, want %q", k.calls, want) + } + for _, want := range []string{"name: kafka", `"app.kubernetes.io/managed-by": "kates"`, `"gov.example.com/product-id": "KATES1"`, `"kates.io/namespace-labels": "gov.example.com/product-id"`} { + if !strings.Contains(k.manifest, want) { + t.Errorf("manifest lacks %q:\n%s", want, k.manifest) + } + } +} + +// Labels already right: no call at all, so an account that may not patch +// namespaces passes. managed-by is not the existing namespace's business. +func TestEnsureDeployNamespaceLeavesMatchingLabelsAlone(t *testing.T) { + k := fakeNamespaceKubectl(t, existingNamespace(map[string]string{"gov.example.com/product-id": "KATES1", "team": "payments"}, nil)) + if err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels); err != nil { + t.Fatal(err) + } + if want := []string{getKafkaNamespace}; !reflect.DeepEqual(k.calls, want) { + t.Fatalf("calls = %q, want only the lookup", k.calls) + } +} + +func TestEnsureDeployNamespacePatchesChangedAndRemovedLabels(t *testing.T) { + k := fakeNamespaceKubectl(t, existingNamespace( + map[string]string{ + "gov.example.com/product-id": "OLD", + "old.example.com/gone": "x", + "team": "payments", // not Kates': never removed + managedByLabel: "helm", // not Kates' either + }, + map[string]string{namespaceLabelsAnnotation: "gov.example.com/product-id,old.example.com/gone,never.example.com/set"}, + )) + labels := map[string]string{"gov.example.com/product-id": "KATES1", "cyber.example.com/sensitivity": "2", managedByLabel: "kates"} + if err := ensureDeployNamespace(context.Background(), "kafka", labels); err != nil { + t.Fatal(err) + } + want := map[string]any{"metadata": map[string]any{ + "labels": map[string]any{ + "gov.example.com/product-id": "KATES1", + "cyber.example.com/sensitivity": "2", + "old.example.com/gone": nil, + }, + "annotations": map[string]any{namespaceLabelsAnnotation: "cyber.example.com/sensitivity,gov.example.com/product-id"}, + }} + if got := k.patch(t); !reflect.DeepEqual(got, want) { + t.Errorf("patch = %v, want %v", got, want) + } +} + +// The last user label gone: removed, and the annotation with it. +func TestEnsureDeployNamespaceDropsTheAnnotationWithTheLastLabel(t *testing.T) { + k := fakeNamespaceKubectl(t, existingNamespace( + map[string]string{"gov.example.com/product-id": "KATES1"}, + map[string]string{namespaceLabelsAnnotation: "gov.example.com/product-id"}, + )) + if err := ensureDeployNamespace(context.Background(), "kafka", map[string]string{managedByLabel: "kates"}); err != nil { + t.Fatal(err) + } + want := map[string]any{"metadata": map[string]any{ + "labels": map[string]any{"gov.example.com/product-id": nil}, + "annotations": map[string]any{namespaceLabelsAnnotation: nil}, + }} + if got := k.patch(t); !reflect.DeepEqual(got, want) { + t.Errorf("patch = %v, want %v", got, want) + } +} + +func TestEnsureDeployNamespaceForbiddenCreateShowsTheManifest(t *testing.T) { + k := fakeNamespaceKubectl(t, "") + k.createErr = errors.New(`exit status 1: Error from server (Forbidden): namespaces is forbidden: User "dev" cannot create resource "namespaces"`) + err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels) + if err == nil { + t.Fatal("want an error") + } + for _, want := range []string{`cannot create resource "namespaces"`, "may not create namespaces", "cluster admin", "run again", "kind: Namespace", `"gov.example.com/product-id": "KATES1"`, `"kates.io/namespace-labels"`} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error lacks %q:\n%v", want, err) + } + } + + // An admission denial says Forbidden too, but an admin would be refused + // the same way: its reason is the answer, with no admin instruction. + for _, msg := range []string{ + "exit status 1: connection refused", + `exit status 1: Error from server (Forbidden): error when creating "STDIN": admission webhook "validation.gatekeeper.sh" denied the request: you must provide labels: {"owner"}`, + } { + k.createErr = errors.New(msg) + err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels) + if err == nil || !strings.HasPrefix(err.Error(), "create namespace kafka: "+msg) || strings.Contains(err.Error(), "cluster admin") { + t.Errorf("create error %q = %v", msg, err) + } + } +} + +// --verbose streams kubectl's stderr, and must still hand it back in the +// error, or an RBAC refusal reads as a bare "exit status 1". +func TestRunExecStdinDefaultKeepsStderrWhenVerbose(t *testing.T) { + orig := deployVerbose + t.Cleanup(func() { deployVerbose = orig }) + for _, verbose := range []bool{false, true} { + deployVerbose = verbose + err := runExecStdinDefault(context.Background(), "sh", []string{"-c", `cat >/dev/null; echo 'User "dev" cannot create resource "namespaces"' >&2; exit 1`}, "kind: Namespace") + if err == nil || !rbacRefused(err, "create") { + t.Errorf("verbose=%v: error = %v, want kubectl's stderr in it", verbose, err) + } + } +} + +func TestEnsureDeployNamespaceLookupFailureWritesNothing(t *testing.T) { + k := fakeNamespaceKubectl(t, "") + k.getErr = errors.New("exit status 1: Unable to connect to the server") + if err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels); err == nil || !strings.HasPrefix(err.Error(), "check namespace kafka: ") { + t.Errorf("lookup error = %v", err) + } + k.getErr, k.existing = nil, "not json" + if err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels); err == nil || !strings.HasPrefix(err.Error(), "check namespace kafka: ") { + t.Errorf("undecodable namespace = %v", err) + } + if len(k.calls) != 2 { + t.Errorf("calls = %q, want two lookups and no write", k.calls) + } +} + +func TestEnsureDeployNamespaceOtherPatchFailureKeepsKubectlsWords(t *testing.T) { + for _, out := range []string{ + `Error from server (Conflict): Operation cannot be fulfilled on namespaces "kafka"`, + `Error from server (Forbidden): admission webhook "validation.gatekeeper.sh" denied the request: you must provide labels: {"owner"}`, + } { + k := fakeNamespaceKubectl(t, existingNamespace(map[string]string{"gov.example.com/product-id": "OLD"}, nil)) + k.patchErr, k.patchOut = errors.New("exit status 1"), out + err := ensureDeployNamespace(context.Background(), "kafka", kafkaLabels) + if err == nil || !strings.HasPrefix(err.Error(), "patch namespace kafka: ") || !strings.Contains(err.Error(), out) || strings.Contains(err.Error(), "cluster admin") { + t.Errorf("patch failure %q = %v", out, err) + } + } +} + +func TestEnsureDeployNamespaceForbiddenPatchShowsTheLabelCommand(t *testing.T) { + k := fakeNamespaceKubectl(t, existingNamespace( + map[string]string{"gov.example.com/product-id": "OLD", "old.example.com/gone": "x"}, + map[string]string{namespaceLabelsAnnotation: "gov.example.com/product-id,old.example.com/gone"}, + )) + k.patchErr = errors.New("exit status 1") + labels := map[string]string{"gov.example.com/product-id": "KATES1", "cyber.example.com/sensitivity": "2", managedByLabel: "kates"} + err := ensureDeployNamespace(context.Background(), "kafka", labels) + if err == nil { + t.Fatal("want an error") + } + for _, want := range []string{`cannot patch resource "namespaces"`, "cluster admin", "run again", "kubectl label namespace kafka cyber.example.com/sensitivity=2 gov.example.com/product-id=KATES1 old.example.com/gone- --overwrite"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error lacks %q:\n%v", want, err) + } + } +} + +func TestDeployDryRunListsNamespaceLabels(t *testing.T) { + out := stripStyle(captureStdout(t, func() { + renderDeployPreview(uiTestEntries(), map[string]bool{}, + map[string]string{"a": "1", "b": "2", "c": ""}, + map[string]string{"a": ".labels:3", "b": namespaceLabelsEnv, "c": namespaceLabelFlag}) + })) + for _, want := range []string{"Namespace labels", "a=1 .labels:3", "b=2 KATES_NAMESPACE_LABELS", "c= --namespace-label", "app.kubernetes.io/managed-by=kates is added only to the namespaces deploy creates"} { + if !strings.Contains(out, want) { + t.Errorf("preview lacks %q:\n%s", want, out) + } + } + if out := stripStyle(captureStdout(t, func() { renderDeployPreview(uiTestEntries(), map[string]bool{}, nil, nil) })); strings.Contains(out, "Namespace labels") { + t.Errorf("preview without user labels has a labels block:\n%s", out) + } +} + +// A bad label stops kates deploy before it asks the cluster anything. +func TestDeployRefusesABadLabelBeforeTheCluster(t *testing.T) { + useLabelsFile(t, writeLabelsFile(t, "app.kubernetes.io/name=kates\n")) + k := fakeNamespaceKubectl(t, "") + err := runDeploy(deployCmd, nil) + if err == nil || !strings.Contains(err.Error(), "reserved") { + t.Fatalf("err = %v, want the reserved-prefix refusal", err) + } + if len(k.calls) != 0 { + t.Errorf("deploy ran %q before refusing", k.calls) + } +} + +func TestLabelsFileFlagOnEveryNamespaceCreator(t *testing.T) { + for name, cmd := range map[string]*cobra.Command{ + "deploy": deployCmd, "auto": autoCmd, "kyverno apply": kyvernoApplyCmd, "detect": detectCmd, + "migrate up": migrateUpCmd, "migrate run": migrateRunCmd, "migrate source deploy": migrateSourceDeployCmd, + } { + if cmd.Flags().Lookup("labels-file") == nil { + t.Errorf("kates %s has no --labels-file", name) + } + } +} diff --git a/cli/cmd/deploy_plan.go b/cli/cmd/deploy_plan.go index 5d6b1c2e..3cd4e91b 100644 --- a/cli/cmd/deploy_plan.go +++ b/cli/cmd/deploy_plan.go @@ -389,7 +389,7 @@ func registerDashboardComponents(dashboard *deployDashboardModel, ns resolvedNam } // runDeployDryRun shows the deployment preview and exits without deploying. -func runDeployDryRun(sharedEntries []DeploySummaryEntry) error { +func runDeployDryRun(sharedEntries []DeploySummaryEntry, labels, labelSources map[string]string) error { dryCtx, dryCancel := context.WithTimeout(context.Background(), 30*time.Second) defer dryCancel() existingReleases := make(map[string]bool) @@ -399,7 +399,7 @@ func runDeployDryRun(sharedEntries []DeploySummaryEntry) error { existingReleases[key] = true } } - renderDeployPreview(sharedEntries, existingReleases) + renderDeployPreview(sharedEntries, existingReleases, labels, labelSources) return nil } diff --git a/cli/cmd/deploy_preview.go b/cli/cmd/deploy_preview.go index 94c4bfaa..78812596 100644 --- a/cli/cmd/deploy_preview.go +++ b/cli/cmd/deploy_preview.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + "maps" + "slices" "strings" "github.com/bmscomp/kates/cli/output" @@ -12,7 +14,7 @@ import ( // happens afterwards. It shares its column layout with the final summary // (deploy_ui.go) on purpose: the preview and the result should be readable as // the same table, so a component that moved between them is obvious. -func renderDeployPreview(entries []DeploySummaryEntry, existingReleases map[string]bool) { +func renderDeployPreview(entries []DeploySummaryEntry, existingReleases map[string]bool, labels, labelSources map[string]string) { g := output.Glyphs() fmt.Println() fmt.Println(lipgloss.NewStyle().Bold(true).Foreground(clrAccent).Render(" Deployment plan") + @@ -62,6 +64,19 @@ func renderDeployPreview(entries []DeploySummaryEntry, existingReleases map[stri strings.Join(namespaces, " ")) } + // Each label with where it came from: with three sources overriding one + // another, the value alone does not say which file or flag to fix. + if len(labels) > 0 { + fmt.Println() + fmt.Printf(" %s\n", lipgloss.NewStyle().Bold(true).Render("Namespace labels")) + for _, key := range slices.Sorted(maps.Keys(labels)) { + fmt.Printf(" %s=%s %s\n", key, labels[key], + lipgloss.NewStyle().Foreground(clrDim).Render(labelSources[key])) + } + fmt.Printf(" %s\n", lipgloss.NewStyle().Foreground(clrDim). + Render(managedByLabel+"=kates is added only to the namespaces deploy creates.")) + } + fmt.Println() if deployCount > 0 { fmt.Printf(" %s\n", diff --git a/cli/cmd/deploy_ui_test.go b/cli/cmd/deploy_ui_test.go index b74913e1..451f6291 100644 --- a/cli/cmd/deploy_ui_test.go +++ b/cli/cmd/deploy_ui_test.go @@ -153,7 +153,7 @@ func TestDeployRowsShareColumnOffsets(t *testing.T) { func TestPreviewAndSummarySpeakTheSameLanguage(t *testing.T) { entries := uiTestEntries() preview := stripStyle(captureStdout(t, func() { - renderDeployPreview(entries, map[string]bool{}) + renderDeployPreview(entries, map[string]bool{}, nil, nil) })) for i := range entries { entries[i].Status = "deployed" @@ -279,7 +279,7 @@ func TestSummariseCountsAndFindsTheSlowest(t *testing.T) { func TestPreviewNamesTheNamespaces(t *testing.T) { entries := uiTestEntries() out := stripStyle(captureStdout(t, func() { - renderDeployPreview(entries, map[string]bool{}) + renderDeployPreview(entries, map[string]bool{}, nil, nil) })) for _, ns := range []string{"kafka", "kates", "connect", "database", "litmus"} { if !strings.Contains(out, ns) { @@ -291,7 +291,7 @@ func TestPreviewNamesTheNamespaces(t *testing.T) { } // Grammar: the old footer said "1 components". single := stripStyle(captureStdout(t, func() { - renderDeployPreview(entries[:1], map[string]bool{}) + renderDeployPreview(entries[:1], map[string]bool{}, nil, nil) })) if strings.Contains(single, "1 components") { t.Errorf("plural bug is back:\n%s", single) diff --git a/cli/cmd/detect.go b/cli/cmd/detect.go index 47b180f3..a758ed15 100644 --- a/cli/cmd/detect.go +++ b/cli/cmd/detect.go @@ -65,12 +65,18 @@ func init() { detectCmd.Flags().BoolVar(&benchStorage, "bench-storage", false, "Run active live storage benchmarking using ephemeral fio pods (takes ~30s)") detectCmd.Flags().BoolVar(&benchNetwork, "bench-network", false, "Run active network throughput sweeps between AZs using iperf3 (~10s)") detectCmd.Flags().BoolVar(&benchDNS, "bench-dns", false, "Run active CoreDNS latency and throttling sweeps using parallel DNS queries (~5s)") + addLabelsFileFlag(detectCmd) rootCmd.AddCommand(detectCmd) } func runDetect(cmd *cobra.Command, args []string) error { + namespaceLabels, _, err := namespaceUserLabels() + if err != nil { + return err + } executor := detect.NewOSExecutor() collector := detect.NewCollector(executor) + collector.NamespaceLabels = namespaceLabels collector.BenchStorage = benchStorage collector.BenchNetwork = benchNetwork collector.BenchDNS = benchDNS diff --git a/cli/cmd/kyverno_apply.go b/cli/cmd/kyverno_apply.go index ea8dc3f0..64bb2cbd 100644 --- a/cli/cmd/kyverno_apply.go +++ b/cli/cmd/kyverno_apply.go @@ -50,6 +50,9 @@ It uses Helm to apply the exact configuration needed.`, hc := helm.New("") hc.Run(ctx, "repo", "add", "kyverno", "https://kyverno.github.io/kyverno/") hc.Run(ctx, "repo", "update", "kyverno") + if err := ensureConfiguredDeployNamespace(ctx, "kyverno"); err != nil { + return err + } out, err := hc.Run(ctx, "upgrade", "--install", "kyverno", "kyverno/kyverno", "--version", "3.6.4", "-n", "kyverno", "--create-namespace", "--set", "admissionController.replicas=1", @@ -134,6 +137,9 @@ It uses Helm to apply the exact configuration needed.`, } // 2. Apply via Helm + if err := ensureConfiguredDeployNamespace(ctx, "kates"); err != nil { + return err + } output.Hint("Applying policies...") hc := helm.New("") hc.Verbose = true @@ -166,6 +172,7 @@ func init() { kyvernoApplyCmd.Flags().BoolVar(&kyvernoApplyKafka, "with-kafka", false, "Enable Kafka-specific policies (Strimzi)") kyvernoApplyCmd.Flags().BoolVarP(&kyvernoApplyYes, "yes", "y", false, "Skip confirmation prompt") kyvernoApplyCmd.Flags().BoolVar(&kyvernoApplyDryRun, "dry-run", false, "Show what would be applied without executing") + addLabelsFileFlag(kyvernoApplyCmd) kyvernoCmd.AddCommand(kyvernoApplyCmd) } diff --git a/cli/cmd/main_test.go b/cli/cmd/main_test.go index e482b99b..2b02b37f 100644 --- a/cli/cmd/main_test.go +++ b/cli/cmd/main_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "os" + "path/filepath" "testing" ) @@ -32,6 +33,16 @@ func TestMain(m *testing.M) { if os.Getenv("KATES_TEST_RUN_AS_CLI") != "1" { os.Unsetenv("KATES_PROFILES") } + // Namespace labels come from ./.labels and KATES_NAMESPACE_LABELS: a + // stray file in cli/cmd or the developer's variable would label what the + // tests expect bare. An empty labels file stands in for ./.labels; the + // tests of the default path set namespaceLabelsFile back to "". + os.Unsetenv(namespaceLabelsEnv) + namespaceLabelsFile = filepath.Join(home, "empty.labels") + if err := os.WriteFile(namespaceLabelsFile, nil, 0o600); err != nil { + fmt.Fprintln(os.Stderr, "TestMain:", err) + os.Exit(1) + } code := m.Run() os.RemoveAll(home) os.Exit(code) diff --git a/cli/cmd/migrate_blocks_test.go b/cli/cmd/migrate_blocks_test.go index 51f2dea2..7424a12d 100644 --- a/cli/cmd/migrate_blocks_test.go +++ b/cli/cmd/migrate_blocks_test.go @@ -136,8 +136,7 @@ func TestMigrateSourceDeployAndRemove(t *testing.T) { f := newFakeProc(t) scriptCluster(f, true) f.on("kubectl get namespace kafka-old --ignore-not-found -o jsonpath={.metadata.name}", "") - f.on("kubectl create namespace kafka-old", "") - f.on("kubectl label namespace kafka-old", "") + f.on("kubectl create -f -", "") f.on("helm upgrade --install old charts/legacy-kafka -n kafka-old -f .build/migrate/m391-430/source-values.yaml --wait --timeout 600s", "") f.on("helm test old -n kafka-old --timeout 600s", "") captureOutput(t, "table") diff --git a/cli/cmd/migrate_source.go b/cli/cmd/migrate_source.go index d5d4400e..96b5119b 100644 --- a/cli/cmd/migrate_source.go +++ b/cli/cmd/migrate_source.go @@ -98,6 +98,7 @@ func init() { d.BoolVarP(&migrateSourceDeployFlags.Yes, "yes", "y", false, "Assume yes and never prompt (fails instead of asking)") d.IntVar(&migrateSourceDeployFlags.Timeout, "timeout", migrateDefaultTimeout, "Install and test budget in seconds") d.StringVar(&migrateSourceDeployFlags.Registry, "registry", kafkaversion.DefaultLegacyRegistry, "Registry of the built legacy images") + addLabelsFileFlag(migrateSourceDeployCmd) d.StringVar(&migrateSourceDeployFlags.TargetCluster, "target-cluster", migrate.DefaultTargetCluster, "Name of the target Kafka the source will be mirrored to") d.StringVar(&migrateSourceDeployFlags.TargetNS, "target-namespace", migrate.DefaultTargetNamespace, "Namespace of the target Kafka") diff --git a/cli/cmd/migrate_up.go b/cli/cmd/migrate_up.go index ac8bd32c..513bf2c6 100644 --- a/cli/cmd/migrate_up.go +++ b/cli/cmd/migrate_up.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" - "sort" "strings" "time" @@ -14,6 +14,7 @@ import ( "github.com/bmscomp/kates/cli/internal/podrun" "github.com/bmscomp/kates/cli/output" + "github.com/bmscomp/kates/cli/pkg/detect" "github.com/bmscomp/kates/cli/pkg/kafkaversion" "github.com/bmscomp/kates/cli/pkg/migrate" ) @@ -50,6 +51,7 @@ cutover and down find the lab again.`, func init() { addMigratePairFlags(migrateUpCmd, &migrateUpFlags) migrateUpCmd.Flags().BoolVar(&migrateUpFlags.SkipBuild, "skip-build", false, "Do not build/load the source image (assume it is present)") + addLabelsFileFlag(migrateUpCmd) migrateCmd.AddCommand(migrateUpCmd) } @@ -448,8 +450,10 @@ func (r *labRun) deploySource(ctx context.Context, s *migrate.State, src *migrat } // ensureLabNamespace creates a namespace labelled as the lab's when it does -// not exist, and reports whether it did. An existing namespace is somebody's -// and is left as it is. +// not exist, and reports whether it did. It carries the user's namespace +// labels too, set in the one create call so a policy requiring them admits +// it; the lab's labels win a clash. An existing namespace is somebody's and +// is left as it is. func ensureLabNamespace(ctx context.Context, namespace string, labels map[string]string) (created bool, err error) { out, err := defaultRunner.Run(ctx, "kubectl", "get", "namespace", namespace, "--ignore-not-found", "-o", "jsonpath={.metadata.name}") if err != nil { @@ -458,31 +462,17 @@ func ensureLabNamespace(ctx context.Context, namespace string, labels map[string if strings.TrimSpace(out) != "" { return false, nil } - if _, err := defaultRunner.Run(ctx, "kubectl", "create", "namespace", namespace); err != nil { - return false, fmt.Errorf("create namespace %s: %w", namespace, err) + all, _, err := namespaceUserLabels() + if err != nil { + return false, err } - args := []string{"label", "namespace", namespace, "--overwrite"} - args = append(args, labelPairs(labels)...) - if _, err := defaultRunner.Run(ctx, "kubectl", args...); err != nil { - return true, fmt.Errorf("label namespace %s: %w", namespace, err) + maps.Copy(all, labels) + if _, err := defaultRunner.RunInput(ctx, detect.NamespaceManifest(namespace, all, nil), "kubectl", "create", "-f", "-"); err != nil { + return false, fmt.Errorf("create namespace %s: %w", namespace, err) } return true, nil } -// labelPairs renders labels as key=value arguments, sorted. -func labelPairs(labels map[string]string) []string { - keys := make([]string, 0, len(labels)) - for k := range labels { - keys = append(keys, k) - } - sort.Strings(keys) - out := make([]string, 0, len(keys)) - for _, k := range keys { - out = append(out, k+"="+labels[k]) - } - return out -} - // sourceDiagnostics prints a source's pods and broker log after a failed // install. func (r *labRun) sourceDiagnostics(ctx context.Context, src *migrate.Source) { diff --git a/cli/cmd/migrate_up_test.go b/cli/cmd/migrate_up_test.go index 4ae0852b..da124e2a 100644 --- a/cli/cmd/migrate_up_test.go +++ b/cli/cmd/migrate_up_test.go @@ -3,6 +3,7 @@ package cmd import ( "context" "os" + "slices" "strings" "testing" @@ -13,8 +14,7 @@ import ( // prerequisites: the namespace, the two helm installs, the waits. func scriptUp(f *fakeProc) { f.on("kubectl get namespace kafka-m282-430-src --ignore-not-found -o jsonpath={.metadata.name}", "") - f.on("kubectl create namespace kafka-m282-430-src", "namespace/kafka-m282-430-src created") - f.on("kubectl label namespace kafka-m282-430-src", "namespace/kafka-m282-430-src labeled") + f.on("kubectl create -f -", "namespace/kafka-m282-430-src created") f.on("helm upgrade --install m282-430-src charts/legacy-kafka", "") f.on("helm test m282-430-src -n kafka-m282-430-src", "") f.on("helm upgrade --install mm2-m282-430 charts/mirror-maker2", "") @@ -43,8 +43,7 @@ func TestMigrateUpPhasesAndHelmArgv(t *testing.T) { "kubectl -n kafka wait kafka/krafter --for=condition=Ready --timeout=300s", "kubectl -n kafka get secret kates-mm2 --ignore-not-found", targetTopicsList, - "kubectl create namespace kafka-m282-430-src", - "kubectl label namespace kafka-m282-430-src --overwrite kates.io/lab=m282-430 kates.io/lab-role=source", + "kubectl create -f -", "helm upgrade --install m282-430-src charts/legacy-kafka -n kafka-m282-430-src -f .build/migrate/m282-430/source-values.yaml --wait --timeout 600s", "helm test m282-430-src -n kafka-m282-430-src --timeout 600s", "helm upgrade --install mm2-m282-430 charts/mirror-maker2 -n kafka -f charts/mirror-maker2/values-kind.yaml -f charts/mirror-maker2/values-migrate-2x.yaml -f .build/migrate/m282-430/mirror-values.yaml --timeout 10m", @@ -59,6 +58,13 @@ func TestMigrateUpPhasesAndHelmArgv(t *testing.T) { } last = i } + // The namespace is created labelled, in that one call. + ns := f.find("kubectl create -f -")[0].stdin + for _, want := range []string{"kind: Namespace", "name: kafka-m282-430-src", `"kates.io/lab": "m282-430"`, `"kates.io/lab-role": "source"`} { + if !strings.Contains(ns, want) { + t.Errorf("the namespace manifest lacks %q:\n%s", want, ns) + } + } // --skip-build: no docker, no kind load. for _, l := range f.lines() { if strings.HasPrefix(l, "docker") || strings.HasPrefix(l, "kind") { @@ -100,14 +106,53 @@ func TestMigrateUpPhasesAndHelmArgv(t *testing.T) { } } +// A lab namespace carries the user's namespace labels beside the lab's own, +// set in the create call, so a cluster that requires labels admits it. +func TestMigrateUpLabNamespaceCarriesTheUserLabels(t *testing.T) { + migrateTestRepo(t) + useLabelsFile(t, "") + if err := os.WriteFile(".labels", []byte("gov.example.com/product-id=KATES1\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(namespaceLabelsEnv, "cyber.example.com/sensitivity=2") + f := newFakeProc(t) + scriptCluster(f, true) + scriptPrerequisites(f) + scriptUp(f) + out := captureOutput(t, "table") + + flags := newPairFlags("2.8.2") + flags.SkipBuild = true + if err := runMigrateUp(context.Background(), flags); err != nil { + t.Fatalf("up: %v\n%s", err, out.String()) + } + creates := f.find("kubectl create -f -") + if len(creates) != 1 { + t.Fatalf("want the namespace created in one call, got %d", len(creates)) + } + for _, want := range []string{ + "name: kafka-m282-430-src", + `"cyber.example.com/sensitivity": "2"`, + `"gov.example.com/product-id": "KATES1"`, + `"kates.io/lab": "m282-430"`, + `"kates.io/lab-role": "source"`, + } { + if !strings.Contains(creates[0].stdin, want) { + t.Errorf("the namespace manifest lacks %q:\n%s", want, creates[0].stdin) + } + } + if f.hasLine("kubectl label namespace") { + t.Error("the namespace was labelled after its creation") + } +} + // scriptUpFanIn scripts the two-source lab m282-391-430. func scriptUpFanIn(f *fakeProc) { const lab = "m282-391-430" for _, alias := range []string{"src282", "src391"} { ns := "kafka-" + lab + "-" + alias f.on("kubectl get namespace "+ns+" --ignore-not-found -o jsonpath={.metadata.name}", "") - f.on("kubectl create namespace "+ns, "namespace/"+ns+" created") - f.on("kubectl label namespace "+ns, "labeled") + f.on("kubectl create -f -", "namespace/"+ns+" created") f.on("helm upgrade --install "+lab+"-"+alias+" charts/legacy-kafka", "") f.on("helm test "+lab+"-"+alias+" -n "+ns, "") } @@ -131,9 +176,21 @@ func TestMigrateUpFanIn(t *testing.T) { } // One namespace, one release and one Helm test per source, then the one // mirror release. + var manifests []string + for _, c := range f.find("kubectl create -f -") { + manifests = append(manifests, c.stdin) + } + for _, alias := range []string{"src282", "src391"} { + want := "name: kafka-m282-391-430-" + alias + "\n" + + " labels:\n" + + ` "kates.io/lab": "m282-391-430"` + "\n" + + ` "kates.io/lab-role": "source"` + "\n" + + ` "kates.io/lab-source": "` + alias + `"` + "\n" + if !slices.ContainsFunc(manifests, func(m string) bool { return strings.Contains(m, want) }) { + t.Errorf("no namespace manifest carries\n%s\namong:\n%s", want, strings.Join(manifests, "---\n")) + } + } for _, want := range []string{ - "kubectl label namespace kafka-m282-391-430-src282 --overwrite kates.io/lab=m282-391-430 kates.io/lab-role=source kates.io/lab-source=src282", - "kubectl label namespace kafka-m282-391-430-src391 --overwrite kates.io/lab=m282-391-430 kates.io/lab-role=source kates.io/lab-source=src391", "helm upgrade --install m282-391-430-src282 charts/legacy-kafka -n kafka-m282-391-430-src282 -f .build/migrate/m282-391-430/source-values-src282.yaml --wait --timeout 600s", "helm upgrade --install m282-391-430-src391 charts/legacy-kafka -n kafka-m282-391-430-src391 -f .build/migrate/m282-391-430/source-values-src391.yaml --wait --timeout 600s", "helm upgrade --install mm2-m282-391-430 charts/mirror-maker2 -n kafka -f charts/mirror-maker2/values-kind.yaml -f charts/mirror-maker2/values-migrate-2x.yaml -f .build/migrate/m282-391-430/mirror-values.yaml --timeout 10m", @@ -282,7 +339,7 @@ func TestMigrateUpFailsFastOnPrerequisites(t *testing.T) { if err == nil { t.Fatal("want a failure") } - if f.hasLine("helm upgrade") || f.hasLine("kubectl create namespace") { + if f.hasLine("helm upgrade") || f.hasLine("kubectl create") { t.Fatalf("nothing may be created after a failed prerequisite:\n %s", strings.Join(f.lines(), "\n ")) } text := stripAnsi(out.String()) @@ -306,7 +363,7 @@ func TestMigrateUpRefusesOverlappingLab(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "--topics") || !strings.Contains(err.Error(), "lab other") { t.Fatalf("want the overlap refusal naming --topics, got %v", err) } - if f.hasLine("kubectl create namespace") { + if f.hasLine("kubectl create") { t.Fatal("nothing may be created after the refusal") } } @@ -326,7 +383,7 @@ func TestMigrateUpRefusesATopicTheTargetHas(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "kates.orders") || !strings.Contains(err.Error(), "--topics") { t.Fatalf("want the refusal naming kates.orders and --topics, got %v", err) } - if f.hasLine("kubectl create namespace") { + if f.hasLine("kubectl create") { t.Fatal("nothing may be created after the refusal") } if !f.hasLine("kubectl -n kafka delete pod kates-migrate-target") { @@ -404,7 +461,7 @@ func TestMigrateUpRefusesWhenTheTargetCannotBeListed(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "cannot list the topics") { t.Fatalf("want a refusal when the target cannot be listed, got %v", err) } - if f.hasLine("kubectl create namespace") { + if f.hasLine("kubectl create") { t.Fatal("nothing may be created when the check cannot run") } }) diff --git a/cli/cmd/migrate_verify.go b/cli/cmd/migrate_verify.go index 2bd1616b..1b32c180 100644 --- a/cli/cmd/migrate_verify.go +++ b/cli/cmd/migrate_verify.go @@ -823,6 +823,7 @@ func init() { migrateRunCmd.Flags().BoolVar(&migrateRunFlags.SkipBuild, "skip-build", false, "Do not build/load the source image (assume it is present)") migrateRunCmd.Flags().StringVar(&migrateRunFlags.SourceVersion, "source-version", "", "Alias of --from (the scripts' spelling)") _ = migrateRunCmd.Flags().MarkDeprecated("source-version", "use --from") + addLabelsFileFlag(migrateRunCmd) migrateCmd.AddCommand(migrateVerifyCmd, migrateCutoverCmd, migrateRollbackCmd, migrateRunCmd) } diff --git a/cli/pkg/detect/collector.go b/cli/pkg/detect/collector.go index 8a6a390b..3fa48d37 100644 --- a/cli/pkg/detect/collector.go +++ b/cli/pkg/detect/collector.go @@ -8,7 +8,10 @@ import ( "encoding/pem" "errors" "fmt" + "maps" + "os" "regexp" + "slices" "sort" "strconv" "strings" @@ -32,8 +35,13 @@ type Collector struct { // dns-detect pod the cluster-domain lookup falls back to, and the Bench* // probes whatever their flags say. Their parts of the report stay empty. // kates deploy --dry-run sets it: a plan preview must not write. - ReadOnly bool - OnProgress func(string) + ReadOnly bool + // NamespaceLabels go onto the namespaces the Secret-creation audit and + // the latency matrix create, under the probes' own labels, so a policy + // that requires labels on every namespace admits them. kates deploy and + // kates auto set them to the user's namespace labels. + NamespaceLabels map[string]string + OnProgress func(string) } func NewCollector(exec CommandExecutor) *Collector { @@ -932,15 +940,64 @@ func (c *Collector) checkKafkaCapacity(report *DetectReport, reservePct float64) return "Insufficient: missing " + strings.Join(deficits, ", ") } +// createProbeNamespace creates a probe's namespace in one call, already +// carrying NamespaceLabels and the probe's labels (which win a clash): a +// policy that requires labels refuses a bare namespace before any label call. +// Pod Security labels stay off: the probes' pods are plain busybox, fio and +// iperf3 pods that a restricted namespace refuses. +// The executor has no stdin, so the manifest goes through a file. +func (c *Collector) createProbeNamespace(ns, runID string) error { + labels := map[string]string{} + for key, value := range c.NamespaceLabels { + if !strings.HasPrefix(key, "pod-security.kubernetes.io/") { + labels[key] = value + } + } + labels["kates-detect-experimental"] = "true" + labels["kates-detect-run"] = runID + f, err := os.CreateTemp("", "kates-detect-namespace-*.yaml") + if err != nil { + return err + } + defer os.Remove(f.Name()) + _, err = f.WriteString(NamespaceManifest(ns, labels, nil)) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + _, err = c.exec.Exec("kubectl", "create", "-f", f.Name()) + return err +} + +// NamespaceManifest renders a Namespace for kubectl create -f, every label +// and annotation key and value quoted. +func NamespaceManifest(name string, labels, annotations map[string]string) string { + var b strings.Builder + fmt.Fprintf(&b, "apiVersion: v1\nkind: Namespace\nmetadata:\n name: %s\n", name) + for _, field := range []struct { + name string + values map[string]string + }{{"annotations", annotations}, {"labels", labels}} { + if len(field.values) == 0 { + continue + } + fmt.Fprintf(&b, " %s:\n", field.name) + for _, key := range slices.Sorted(maps.Keys(field.values)) { + fmt.Fprintf(&b, " %s: %s\n", strconv.Quote(key), strconv.Quote(field.values[key])) + } + } + return b.String() +} + func (c *Collector) checkSecretCreation(ctx context.Context) SecretCreationAudit { audit := SecretCreationAudit{} runID := strconv.FormatInt(time.Now().UnixNano(), 36) ns := fmt.Sprintf("kates-detect-secrets-%s", runID) - // Create namespace - _, err := c.exec.Exec("kubectl", "create", "ns", ns) - if err != nil { + if err := c.createProbeNamespace(ns, runID); err != nil { audit.NamespaceCreated = false audit.SecretCreated = false audit.ErrorMsg = fmt.Sprintf("failed to create namespace: %v", err) @@ -953,12 +1010,9 @@ func (c *Collector) checkSecretCreation(ctx context.Context) SecretCreationAudit c.exec.Exec("kubectl", "delete", "ns", ns, "--wait=false") }() - // Label the namespace - _, _ = c.exec.Exec("kubectl", "label", "ns", ns, "kates-detect-experimental=true", fmt.Sprintf("kates-detect-run=%s", runID)) - // Try to create the secret. A refusal (an admission webhook's included) // is on kubectl's stderr, which the executor returns in the error. - _, err = c.exec.Exec("kubectl", "create", "secret", "generic", "kates-detect-test-sec", + _, err := c.exec.Exec("kubectl", "create", "secret", "generic", "kates-detect-test-sec", "--from-literal=test-key=test-val", "-n", ns) if err != nil { @@ -1032,9 +1086,7 @@ func (c *Collector) getAZLatency(ctx context.Context, nodes []NodeInfo) []Latenc ns := fmt.Sprintf("kates-detect-latency-%s", runID) c.progress("Auditing inter-AZ network latency: setting up temporary namespace...") - // Create namespace - _, err := c.exec.Exec("kubectl", "create", "ns", ns) - if err != nil { + if err := c.createProbeNamespace(ns, runID); err != nil { return nil } @@ -1044,9 +1096,6 @@ func (c *Collector) getAZLatency(ctx context.Context, nodes []NodeInfo) []Latenc c.exec.Exec("kubectl", "delete", "ns", ns, "--wait=false") }() - // Label the namespace - _, _ = c.exec.Exec("kubectl", "label", "ns", ns, "kates-detect-experimental=true", fmt.Sprintf("kates-detect-run=%s", runID)) - c.progress("Auditing inter-AZ network latency: creating prober pods...") // Spin up a prober pod in each zone @@ -1834,9 +1883,7 @@ func (c *Collector) runLiveStorageBench(ctx context.Context, report *DetectRepor ns := fmt.Sprintf("kates-detect-bench-%s", runID) c.progress("Storage benchmarking: setting up temporary namespace...") - // Create namespace - _, err := c.exec.Exec("kubectl", "create", "ns", ns) - if err != nil { + if err := c.createProbeNamespace(ns, runID); err != nil { return } @@ -1846,9 +1893,6 @@ func (c *Collector) runLiveStorageBench(ctx context.Context, report *DetectRepor _, _ = c.exec.Exec("kubectl", "delete", "ns", ns, "--wait=false") }() - // Label the namespace - _, _ = c.exec.Exec("kubectl", "label", "ns", ns, "kates-detect-experimental=true", fmt.Sprintf("kates-detect-run=%s", runID)) - // Benchmark each StorageClass sequentially to avoid disk self-contention for idx, sc := range report.Storage { scName := sc.Name @@ -1977,8 +2021,7 @@ func (c *Collector) runAZBandwidthBench(ctx context.Context, report *DetectRepor ns := fmt.Sprintf("kates-detect-bandwidth-%s", runID) c.progress("Auditing inter-AZ network bandwidth: setting up temporary namespace...") - _, err := c.exec.Exec("kubectl", "create", "ns", ns) - if err != nil { + if err := c.createProbeNamespace(ns, runID); err != nil { return } @@ -1987,9 +2030,6 @@ func (c *Collector) runAZBandwidthBench(ctx context.Context, report *DetectRepor _, _ = c.exec.Exec("kubectl", "delete", "ns", ns, "--wait=false") }() - // Label the namespace - _, _ = c.exec.Exec("kubectl", "label", "ns", ns, "kates-detect-experimental=true", fmt.Sprintf("kates-detect-run=%s", runID)) - c.progress("Auditing inter-AZ network bandwidth: creating iperf3 prober pods...") // Spin up a prober pod in each zone @@ -2140,8 +2180,7 @@ func (c *Collector) runDNSResolutionProbe(ctx context.Context, report *DetectRep ns := fmt.Sprintf("kates-detect-dns-%s", runID) c.progress("Auditing CoreDNS performance: setting up temporary namespace...") - _, err := c.exec.Exec("kubectl", "create", "ns", ns) - if err != nil { + if err := c.createProbeNamespace(ns, runID); err != nil { return } @@ -2150,9 +2189,6 @@ func (c *Collector) runDNSResolutionProbe(ctx context.Context, report *DetectRep _, _ = c.exec.Exec("kubectl", "delete", "ns", ns, "--wait=false") }() - // Label the namespace - _, _ = c.exec.Exec("kubectl", "label", "ns", ns, "kates-detect-experimental=true", fmt.Sprintf("kates-detect-run=%s", runID)) - c.progress("Auditing CoreDNS performance: creating prober pod...") nodeName := report.Nodes[0].Name diff --git a/cli/pkg/detect/collector_test.go b/cli/pkg/detect/collector_test.go index feff8351..0bb21b0f 100644 --- a/cli/pkg/detect/collector_test.go +++ b/cli/pkg/detect/collector_test.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" + "os" "regexp" "strings" + "sync" "testing" ) @@ -101,6 +103,7 @@ func setupHealthyMock() *MockExecutor { m.Set(`{"items": []}`, "kubectl", "get", "pods", "-n", "kafka", "-l", "name=strimzi-cluster-operator", "-o", "json") // Secrets check mocks + m.Set("namespace created", "kubectl create -f") m.Set("namespace created", "kubectl create ns") m.Set("namespace labeled", "kubectl label ns") m.Set("namespace deleted", "kubectl delete ns") @@ -360,8 +363,7 @@ func TestCheckKafkaCapacity(t *testing.T) { func TestCheckSecretCreation_KyvernoDenial(t *testing.T) { m := NewMockExecutor() - m.Set("namespace created", "kubectl create ns") - m.Set("namespace labeled", "kubectl label ns") + m.Set("namespace created", "kubectl create -f") m.Set("namespace deleted", "kubectl delete ns") // Kyverno denial response output @@ -539,12 +541,12 @@ func TestCollect_WriteProbesCreateThings(t *testing.T) { probe string parts []string }{ - {"secret audit namespace", []string{"kubectl create ns kates-detect-secrets-"}}, + {"secret audit namespace", []string{"kubectl delete ns kates-detect-secrets-"}}, {"secret audit Secret", []string{"kubectl create secret generic kates-detect-test-sec"}}, - {"latency namespace", []string{"kubectl create ns kates-detect-latency-"}}, + {"latency namespace", []string{"kubectl delete ns kates-detect-latency-"}}, + {"probe namespace manifests", []string{"kubectl create -f ", "kates-detect-namespace-"}}, {"latency prober pods", []string{"kubectl run prober-"}}, {"dns-detect fallback pod", []string{"kubectl run", "dns-detect"}}, - {"namespace labels", []string{"kubectl label ns"}}, {"namespace cleanup", []string{"kubectl delete ns"}}, } { if !containsCall(writes, want.parts...) { @@ -552,11 +554,11 @@ func TestCollect_WriteProbesCreateThings(t *testing.T) { want.probe, want.parts, strings.Join(writes, "\n ")) } } - // Each Bench* probe starts by creating its namespace, so four more than - // the two the secret and latency probes create. + // Every probe creates its namespace from a manifest: secret, latency and + // the three benches. var namespaces int for _, w := range writes { - if strings.HasPrefix(w, "kubectl create ns ") { + if strings.HasPrefix(w, "kubectl create -f ") { namespaces++ } } @@ -565,6 +567,85 @@ func TestCollect_WriteProbesCreateThings(t *testing.T) { } } +// manifestExecutor answers as its MockExecutor does, and keeps every file +// `kubectl create -f` is given, read before the probe removes it. +type manifestExecutor struct { + *MockExecutor + lock sync.Mutex + manifests map[string]string +} + +func (e *manifestExecutor) Exec(name string, args ...string) (string, error) { + if name == "kubectl" && len(args) == 3 && args[0] == "create" && args[1] == "-f" { + data, err := os.ReadFile(args[2]) + if err != nil { + return "", err + } + e.lock.Lock() + e.manifests[args[2]] = string(data) + e.lock.Unlock() + } + return e.MockExecutor.Exec(name, args...) +} + +// A policy that requires labels on every namespace refuses a bare one, so +// the secret and latency probes create theirs in one call that already +// carries the user's labels, under the probe's own, which win a clash. +func TestCollect_ProbeNamespacesCarryTheUserLabels(t *testing.T) { + m := &manifestExecutor{MockExecutor: setupWriteProbeMock(), manifests: map[string]string{}} + c := NewCollector(m) + c.BenchStorage, c.BenchNetwork, c.BenchDNS = true, true, true + c.NamespaceLabels = map[string]string{ + "gov.example.com/product-id": "KATES1", + "kates-detect-experimental": "no", + "pod-security.kubernetes.io/enforce": "restricted", + } + if _, err := c.Collect(context.Background()); err != nil { + t.Fatalf("Collect failed: %v", err) + } + + if len(m.manifests) != 5 { + t.Fatalf("want one manifest per probe namespace, got %d: %v", len(m.manifests), m.manifests) + } + for _, prefix := range []string{"kates-detect-secrets-", "kates-detect-latency-", "kates-detect-bench-", "kates-detect-bandwidth-", "kates-detect-dns-"} { + var manifest string + for _, mf := range m.manifests { + if strings.Contains(mf, "name: "+prefix) { + manifest = mf + } + } + if manifest == "" { + t.Errorf("no manifest creates a %s namespace: %v", prefix, m.manifests) + continue + } + for _, want := range []string{ + "kind: Namespace", + `"gov.example.com/product-id": "KATES1"`, + `"kates-detect-experimental": "true"`, + `"kates-detect-run": "`, + } { + if !strings.Contains(manifest, want) { + t.Errorf("the %s manifest lacks %q:\n%s", prefix, want, manifest) + } + } + // The probes' pods are not restricted-ready; a Pod Security label + // would have them refused. + if strings.Contains(manifest, "pod-security.kubernetes.io") { + t.Errorf("the %s manifest carries a Pod Security label:\n%s", prefix, manifest) + } + } + for _, call := range m.Calls() { + if strings.HasPrefix(call, "kubectl label ns kates-detect-") || strings.HasPrefix(call, "kubectl create ns ") { + t.Errorf("a probe namespace was created bare or labelled after its creation: %s", call) + } + } + for path := range m.manifests { + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the manifest %s was left behind: %v", path, err) + } + } +} + // kates deploy --dry-run collects read-only: whatever it previews, it must // not create, label or delete anything to do it, benches included. func TestCollect_ReadOnlyIssuesNoWrites(t *testing.T) { diff --git a/cli/pkg/detect/renderer_test.go b/cli/pkg/detect/renderer_test.go index 96b88b5e..4bb20d79 100644 --- a/cli/pkg/detect/renderer_test.go +++ b/cli/pkg/detect/renderer_test.go @@ -190,7 +190,7 @@ func TestRenderHTML(t *testing.T) { func TestLiveStorageBench_Success(t *testing.T) { m := NewMockExecutor() - m.Responses["kubectl create ns"] = "" + m.Responses["kubectl create -f"] = "" m.Responses["kubectl label ns"] = "" m.Responses["kubectl delete ns"] = "" m.Responses["sh -c cat <|groups|group `. +`up`, `run` and `migrate source deploy` take `--labels-file` (default: `.labels` in the current directory). A source namespace they create carries those labels and `KATES_NAMESPACE_LABELS`, with the lab's own labels winning on a clash; one that exists is left as it is. [Namespace Labels](12-deployment.md#namespace-labels) gives the format. + ::: {.callout-note} In this version every source is `legacy-kafka`: a Strimzi-operated source (an in-window version, or a dropped line under its own operator with `--operator-scope namespace`) is described by `plan` and refused by `up` with "not yet implemented โ€” use --source-provider legacy". That is true of a `--from` in a set as much as on its own: a fan-in that includes a Strimzi source stays `plan`-only. `--to` other than the primary's version is `plan`-only for the same reason. ::: @@ -1813,6 +1824,7 @@ kates kyverno apply --mode Enforce --yes --with-netpol | `--with-netpol` | Enable NetworkPolicy generation | | `--yes`, `-y` | Skip confirmation prompt | | `--dry-run` | Show what would be applied without executing | +| `--labels-file` | Labels for the `kyverno` and `kates` namespaces (default: `.labels` in the current directory); see [Namespace Labels](12-deployment.md#namespace-labels) | **See also:** [Security & Compliance](17-security.md) for Kyverno policy deep dive and custom policy authoring. diff --git a/docs/book/12-deployment.md b/docs/book/12-deployment.md index a3c5e061..d1d6ecd5 100644 --- a/docs/book/12-deployment.md +++ b/docs/book/12-deployment.md @@ -752,6 +752,115 @@ kates test list `kates ports` is the one-step alternative: it forwards the API to `localhost:8080` instead, writes that address and the key into a context of its own, `ports`, and makes it the current context, leaving every other context as it was. The Quick Start in [Introduction](01-introduction.md#quick-start) describes it, along with the commands for the single-namespace topology. +## Namespace Labels + +Many shared clusters admit a namespace only when it carries the labels their policies ask for, such as an owning team or a cost center, and `kates deploy` deploys into namespaces of its own. You list those labels once, in a `.labels` file, and the namespaces Kates creates or deploys into carry them. You need the directory you run `kates` from, usually the repository root, and an account that may create and label namespaces; the setup takes a few minutes. + +### Step 1 โ€” Write the Labels File + +The repository root holds a commented `.labels.example`, and `.gitignore` lists `.labels`, so your copy stays on your machine. Copy the example where you run `kates`, which is the repository root for `make all`: + +```bash +cp .labels.example .labels +``` + +Then uncomment or add one `key=value` per line. This file gives every namespace an owning team and a cost center: + +```properties +# Read by the cluster's admission policies +team=payments +cost-center=4711 +``` + +Kates trims the spaces around each key and value, skips blank lines and lines whose first non-space character is `#`, and accepts Windows line endings. There are no inline comments and no quoting, so a `#` or a quote inside a value makes the label invalid. A line without `=` is an error, and so is a key that appears twice, which the error reports with both line numbers. Every error from the file starts with its path and line, such as `.labels:3:`. + +A missing `.labels` in the current directory means no labels from a file. `--labels-file ` reads another file instead, and that one must exist. + +### Step 2 โ€” Check the Labels + +Labels come from three sources. For a key that more than one sets, the later source in this table wins: + +| Source | Form | Order | +|:--------------------------------|:------------------------------------|:-----------------| +| `.labels`, or the `--labels-file` file | One `key=value` per line | First | +| `KATES_NAMESPACE_LABELS` | Comma-separated `key=value` entries | Second | +| `--namespace-label` | One `key=value` per flag, repeatable | Last | + +A key given twice inside the variable, or in two flags, takes its last value. Every source follows the same rules, which are the Kubernetes label syntax plus a set of reserved prefixes: + +| Part | Rule | +|:------------------|:----------------------------------------------------------------| +| Key name | 1โ€“63 letters, digits, `-`, `_` or `.`, starting and ending with a letter or digit | +| Key prefix | Optional, before a `/`: a lowercase DNS subdomain of at most 253 characters, never empty | +| Value | Empty, or up to 63 characters under the key name's rule | +| Reserved prefixes | `kubernetes.io`, `k8s.io`, `helm.sh`, `strimzi.io`, `kates.io` and their subdomains are refused; `pod-security.kubernetes.io` itself is allowed | + +So `app.kubernetes.io/part-of` is refused and `pod-security.kubernetes.io/enforce` is accepted. Each error names its source: the file and line, `KATES_NAMESPACE_LABELS` or `--namespace-label`. `kates deploy` resolves and checks the labels before its pre-flight checks, so a bad label stops it before it inspects the cluster. + +A dry run shows what the deploy would apply: + +```bash +kates deploy --dry-run +``` + +When there is at least one label, the plan has a Namespace labels block. It lists each label as `key=value` with where it came from, such as `.labels:3`, `KATES_NAMESPACE_LABELS` or `--namespace-label`. A line under it says that `app.kubernetes.io/managed-by=kates` goes only onto the namespaces the deploy creates. The [deploy](10-cli-reference.md#deploy) entry says what else a dry run reads and writes. + +### Step 3 โ€” Deploy + +Run the deploy as usual, with `make all` or directly: + +```bash +kates deploy +``` + +What Kates does with each namespace depends on whether it exists and on the `kates.io/namespace-labels` annotation, which lists the keys of the labels Kates put there: + +| The namespace | What Kates does | +|:--------------------------------------|:----------------------------------------------------------| +| Doesn't exist | Creates it with your labels, `app.kubernetes.io/managed-by=kates`, and the annotation listing your keys | +| Exists, carries every label of yours at your value, and has no label to remove | Nothing: it makes no call at all | +| Exists, and a label of yours is missing or has another value | One patch sets it and adds its key to the annotation | +| Exists, and carries a label whose key the annotation lists and your labels leave out | The same patch removes that label and drops its key from the annotation | + +The managed-by label goes only onto a namespace Kates creates, and Kates never adds, changes or removes it on one that exists. With no labels of yours, a new namespace gets only the managed-by label and no annotation. + +Because a namespace whose labels are already right gets no call, an account that may not change namespaces still deploys there. When the account may not create a namespace, the deploy stops with kubectl's error and the Namespace manifest, labels and annotation included. A cluster admin applies it with `kubectl create -f`, and you run the deploy again. When the account may not change a namespace whose labels differ, the error prints a `kubectl label namespace โ€ฆ --overwrite` command instead, with the labels to set and the keys to remove as `key-`. Both instructions follow only a refusal by the account's permissions. When an admission policy refuses the namespace, for example because a label it requires is missing, the error is the policy's own message and nothing more, because a cluster admin would be refused the same way: add the label it names to `.labels` and run again. + +Before it installs anything, `kates deploy` inspects the cluster, and some of its probes create the `kates-detect-secrets-` and `kates-detect-latency-` namespaces to measure it. Kates creates them with your labels and their own `kates-detect` labels, which win on a clash. Your `pod-security.kubernetes.io` labels stay off them, because the probes' pods would not pass a restricted namespace. Where the account may not create namespaces, or a policy refuses them, `--no-write-probes` skips the probes that create anything, as `--dry-run` does. + +### Step 4 โ€” Verify the Labels + +Read the labels and the annotation back from one of the deploy's namespaces, `kafka` in the isolated topology: + +```bash +kubectl get namespace kafka --show-labels +kubectl get namespace kafka \ + -o jsonpath='{.metadata.annotations.kates\.io/namespace-labels}' +``` + +The first command shows your labels, and `app.kubernetes.io/managed-by=kates` when the deploy created the namespace. The second prints the comma-separated keys Kates manages there, in sorted order, and nothing when Kates has set no label there. + +### Remove a Label + +Delete its line from `.labels` and run `kates deploy` again. In each namespace the run deploys into, Kates removes the labels whose keys the annotation lists and your labels leave out. The annotation lists only the labels Kates created or changed, so a label someone else set stays. So does a label of yours that an existing namespace already carried with your value, because Kates never had to set it; `kubectl label namespace -` removes it. + +### Where Else the Labels Apply + +Other commands that create namespaces read the labels file and `KATES_NAMESPACE_LABELS` too, and take `--labels-file`; `--namespace-label` is the deploy's alone. This table lists what each one labels: + +| Command | Namespaces it labels | +|:------------------------------------|:--------------------------------------------------------------| +| `kates deploy` | Each component's namespace; `cert-manager`, `kyverno` and `strimzi-operator` only on a run that installs them, so a run that finds them installed leaves their labels as they are | +| `kates auto` | Its namespace, before its Helm installs; nothing under `--dry-run` | +| `kates kyverno apply` | `kyverno` before it installs Kyverno and `kates` before the `kates` chart; nothing under `--dry-run` | +| `kates migrate up`, `kates migrate run`, `kates migrate source deploy` | A lab namespace it creates, where the lab's own labels win on a clash; an existing one is left alone | +| The probes of `kates deploy`, `kates auto` and `kates detect` | The `kates-detect-*-` namespaces they create: secrets and latency, and with `kates detect --bench-*` the bench, bandwidth and dns ones; never your `pod-security.kubernetes.io` labels | +| `ensure_namespace` in `scripts/common.sh` | Every namespace the deploy scripts ensure, new or existing | + +`kates auto` and `kates kyverno apply` label their namespaces the way `kates deploy` does. The scripts are simpler. `ensure_namespace` reads only `.labels` in the directory you run the script from, not `--labels-file` or `KATES_NAMESPACE_LABELS`, and applies its lines with `kubectl label namespace โ€ฆ --overwrite`. It adds no managed-by label and no annotation, so a label whose line you delete stays on the namespace. It checks only that each line starts with a letter or digit and holds a `=`, and stops on any other line; kubectl checks the rest of the syntax, and nothing refuses the reserved prefixes. Without a `.labels`, or with one that has no labels, it labels nothing. + +Where your account may not create namespaces at all, [Running Without Cluster Admin](#running-without-cluster-admin) installs the `kates` chart into a namespace the team already has. + ## Running Without Cluster Admin On a shared cluster, a team usually gets a namespace and the built-in `edit` role in it, not cluster-admin. That is enough to run performance tests, INTEGRITY tests and [conformity checks](appendix-a-glossary.md#gl-conformity-check) with Kates, once an admin has installed [Strimzi](appendix-a-glossary.md#gl-strimzi) and the Kafka cluster. Performance and INTEGRITY runs make no Kubernetes API call: the [Kates API](appendix-a-glossary.md#gl-kates-api) talks only to Kafka. Two things need care, and the `kates` chart's `rbac.scope=namespace` handles both. The default install creates a ClusterRole, which `edit` can't do, and a conformity check reads the Kafka namespace, where the team has no rights. `kates deploy` stays an admin's tool, because it installs Strimzi, cert-manager, monitoring and Litmus. diff --git a/scripts/common.sh b/scripts/common.sh index b1c3dc21..f9db1863 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -43,10 +43,29 @@ require_chart() { fi } +# Also applies ./.labels (git-ignored, see .labels.example), created or not. Unlike +# kates, no managed-by label and no tracking: a line you delete leaves its label. ensure_namespace() { + # A read error inside the process substitution below would go unseen. + if [ -f .labels ] && [ ! -r .labels ]; then + error ".labels: not readable" + return 1 + fi if ! kubectl get namespace "$1" > /dev/null 2>&1; then kubectl create namespace "$1" fi + [ -f .labels ] || return 0 + local labels=() line + while IFS= read -r line || [ -n "$line" ]; do + # Anything else would reach kubectl as a namespace name, a removal (key-) or a flag. + case "$line" in + [A-Za-z0-9]*=*) labels+=("$line") ;; + *) error ".labels: '${line}' is not key=value"; return 1 ;; + esac + done < <(tr -d '\r' < .labels | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' \ + -e '/^#/d' -e '/^$/d' -e 's/[[:space:]]*=[[:space:]]*/=/') + [ ${#labels[@]} -gt 0 ] || return 0 + kubectl label namespace "$1" "${labels[@]}" --overwrite } svc_exists() { diff --git a/scripts/deploy-kafka-generic.sh b/scripts/deploy-kafka-generic.sh index aa6d18fa..2c7d2459 100755 --- a/scripts/deploy-kafka-generic.sh +++ b/scripts/deploy-kafka-generic.sh @@ -73,7 +73,8 @@ elif [ -x "${ROOT_DIR}/build/kates" ]; then "${ROOT_DIR}/build/kates" detect --generate-values --values-output "${DETECTED_VALUES}" --quiet else info " kates binary not found. Building it now..." - cd "${ROOT_DIR}" && make cli-build >/dev/null + # A subshell, so ensure_namespace still reads .labels where the script was run. + (cd "${ROOT_DIR}" && make cli-build >/dev/null) "${ROOT_DIR}/cli/dist/kates" detect --generate-values --values-output "${DETECTED_VALUES}" --quiet fi From 21dd8c01c21c6896dfd36d7af7f3c0a69877d6be Mon Sep 17 00:00:00 2001 From: Said Boudjelda Date: Wed, 21 Oct 2026 21:30:29 +0200 Subject: [PATCH 2/2] fix(cli): withKubeContext copies through a capped slice, with no size arithmetic CodeQL's go/allocation-size-overflow flagged make([]string, 0, len(args)+1) once .labels file data reached kubectl's arguments. The +1 cannot overflow, but appending to args[:end:end] does the same copy without it. The test now gives the input spare capacity, so a write into the caller's array shows. --- cli/cmd/deploy_kubecontext.go | 6 +++--- cli/cmd/deploy_kubecontext_test.go | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cli/cmd/deploy_kubecontext.go b/cli/cmd/deploy_kubecontext.go index bfb4fb80..3f375b0c 100644 --- a/cli/cmd/deploy_kubecontext.go +++ b/cli/cmd/deploy_kubecontext.go @@ -63,9 +63,9 @@ func withKubeContext(kubeContext, name string, args []string) []string { } } - pinned := make([]string, 0, len(args)+1) - pinned = append(pinned, args[:end]...) - pinned = append(pinned, flag+"="+kubeContext) + // The three-index slice caps the capacity at end, so append copies into a + // new array instead of writing over args. + pinned := append(args[:end:end], flag+"="+kubeContext) return append(pinned, args[end:]...) } diff --git a/cli/cmd/deploy_kubecontext_test.go b/cli/cmd/deploy_kubecontext_test.go index c85b3015..ddcaa041 100644 --- a/cli/cmd/deploy_kubecontext_test.go +++ b/cli/cmd/deploy_kubecontext_test.go @@ -47,13 +47,15 @@ func TestWithKubeContext(t *testing.T) { []string{"exec", "pod", "--context=kind-panda", "--", "tool", "--context=x"}}, } { t.Run(tt.name, func(t *testing.T) { - in := append([]string(nil), tt.args...) + // Spare capacity, so an append into the caller's array would show. + in := make([]string, len(tt.args), len(tt.args)+4) + copy(in, tt.args) got := withKubeContext(tt.kubeContext, tt.cmd, in) if !reflect.DeepEqual(got, tt.want) { t.Errorf("withKubeContext(%q, %q, %q)\n got %q\nwant %q", tt.kubeContext, tt.cmd, tt.args, got, tt.want) } - if !reflect.DeepEqual(in, tt.args) { - t.Errorf("withKubeContext modified its input: %q", in) + if !reflect.DeepEqual(in, tt.args) || in[:cap(in)][len(in)] != "" { + t.Errorf("withKubeContext modified its input: %q", in[:cap(in)]) } }) }