From a64dc0d118845335f78f83c27896c03a9d2a51a8 Mon Sep 17 00:00:00 2001 From: Abrar Shivani Date: Thu, 20 Aug 2026 12:07:09 -0700 Subject: [PATCH] add unit tests for the clusterinfo package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clusterinfo answers questions about the cluster the operator runs in: which container runtime the GPU nodes use, whether this is OpenShift, whether there is a cluster proxy, whether the cluster supports DRA. Other parts of the operator use those answers to decide what to install. #2831 added tests for getRuntimeString; the rest of the package had none. The package builds its Kubernetes clients inside each function from a *rest.Config, so there is nothing to inject a mock into. The tests point rest.Config at an httptest server that serves the endpoints the package reads. Real client-go runs against it, so URL building, decoding and 404 handling are exercised rather than simulated. The stub fails a test if a client requests a path it has no handler for, which matters because getOpenshiftVersion reads a 404 as "not OpenShift" — without that check a wrong URL would pass for the wrong reason. getRuntimeString keeps the five cases from #2831 and gains three: a bare prefix with no version suffix, a runtime name appearing as a substring rather than a prefix, and case sensitivity. They move from clusterinfo_test.go to runtime_test.go, which now holds every test for the container runtime path. Nine behaviours look wrong. None are fixed here. The tests record what the code does today and live in clusterinfo_characterization_test.go, apart from the tests asserting intended behaviour, so fixing one of them fails only tests in that file. That is checked by applying each fix and confirming where the failures land. Coverage is 99.3% of statements. The uncovered line calls os.Exit. No production code is changed. Signed-off-by: Abrar Shivani --- .../clusterinfo_characterization_test.go | 313 +++++++++++ controllers/clusterinfo/clusterinfo_test.go | 316 ++++++++++-- controllers/clusterinfo/dra_test.go | 291 +++++++++++ controllers/clusterinfo/logging_test.go | 131 +++++ controllers/clusterinfo/openshift_dtk_test.go | 249 +++++++++ .../clusterinfo/openshift_proxy_test.go | 196 +++++++ .../clusterinfo/openshift_version_test.go | 274 ++++++++++ controllers/clusterinfo/runtime_test.go | 351 +++++++++++++ controllers/clusterinfo/stubs_test.go | 488 ++++++++++++++++++ 9 files changed, 2574 insertions(+), 35 deletions(-) create mode 100644 controllers/clusterinfo/clusterinfo_characterization_test.go create mode 100644 controllers/clusterinfo/dra_test.go create mode 100644 controllers/clusterinfo/logging_test.go create mode 100644 controllers/clusterinfo/openshift_dtk_test.go create mode 100644 controllers/clusterinfo/openshift_proxy_test.go create mode 100644 controllers/clusterinfo/openshift_version_test.go create mode 100644 controllers/clusterinfo/runtime_test.go create mode 100644 controllers/clusterinfo/stubs_test.go diff --git a/controllers/clusterinfo/clusterinfo_characterization_test.go b/controllers/clusterinfo/clusterinfo_characterization_test.go new file mode 100644 index 000000000..55adaa40c --- /dev/null +++ b/controllers/clusterinfo/clusterinfo_characterization_test.go @@ -0,0 +1,313 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// Characterization tests: every test in this file locks in behaviour that looks wrong, so that a +// change to it surfaces in review instead of silently altering what the operator does. They are +// deliberately not assertions of intended behaviour, and a failure here is as likely to mean the +// implementation was fixed as that it regressed — read the referenced bug before touching the test. + +// expectedErrorLogRecord pins the error value alongside the message, because suspected bug #1 is +// that the failed lookup survives only as a log line, so the value it carries is the whole record. +type expectedErrorLogRecord struct { + message string + statusReason metav1.StatusReason + statusCode int32 +} + +func TestGetContainerRuntimeCharacterization(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorLogs []expectedErrorLogRecord + expectedRequestPaths []string + }{ + { + // Suspected bug #1: getContainerRuntime logs the getOpenshiftVersion error and carries on with the + // empty version, so the crio short-circuit is skipped and the failed read is only ever a log line. + description: "an OpenShift version lookup failure is swallowed and node inspection continues", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedErrorLogs: []expectedErrorLogRecord{{ + message: "failed to retrieve", + statusReason: metav1.StatusReasonInternalError, + statusCode: http.StatusInternalServerError, + }}, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #8: a Completed history entry with an empty version returns the same ("", nil) + // pair as "not OpenShift", so a real OpenShift cluster has its runtime read off its nodes instead. + description: "an OpenShift cluster with an empty completed version falls through to node inspection", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, ""))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #9: only a containerd node breaks out of the scan, so every later node that parses + // overwrites the verdict and a docker/cri-o cluster's runtime flips with the apiserver's order. + description: "without a containerd node the last node listed decides: docker then cri-o", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "cri-o://1.28.2")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #9 with the same nodes reversed: the scan's last write wins either way, so the + // verdict tracks list order and no precedence between docker and cri-o is being applied. + description: "without a containerd node the last node listed decides: cri-o then docker", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("cri-o://1.28.2", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + containerRuntime, err := getContainerRuntime(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + capturedErrorRecords := recorder.capturedRecordsAtLevel(logRecordLevelError) + require.Len(t, capturedErrorRecords, len(testCase.expectedErrorLogs)) + for index, expectedErrorLog := range testCase.expectedErrorLogs { + require.Equal(t, expectedErrorLog.message, capturedErrorRecords[index].message) + requireAPIStatusError(t, capturedErrorRecords[index].err, + expectedErrorLog.statusReason, expectedErrorLog.statusCode) + } + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + }) + } +} + +func TestNewCharacterization(t *testing.T) { + // Suspected bug #1 from the caller: getContainerRuntime swallows the denied lookup, so New lists nodes + // first and only its own second lookup aborts, under the "failed to get openshift version" wrapper. + t.Run("a denied OpenShift lookup aborts New only after nodes have been listed", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Nil(t, clusterInfoAPI) + require.ErrorContains(t, err, "failed to get openshift version") + requireAPIStatusError(t, err, metav1.StatusReasonForbidden, http.StatusForbidden) + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion, pathNodes, pathClusterVersion}) + }) +} + +func TestGetOpenshiftVersionCharacterization(t *testing.T) { + // Suspected bug #8: strings.Split("", ".") yields one empty element, so a Completed entry carrying no + // version returns ("", nil) and an OpenShift cluster is indistinguishable from a plain Kubernetes one. + t.Run("a completed entry with an empty version yields the not-OpenShift sentinel", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, ""))), + }) + + openshiftVersion, err := getOpenshiftVersion(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.Empty(t, openshiftVersion) + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion}) + }) +} + +func TestGetDRAResourceGVRCharacterization(t *testing.T) { + // Suspected bug #3: the DeviceClass scan spans every API group before the resource.k8s.io filter, so an + // unrelated CRD of that Kind bypasses the unsupported early return and fails a GPUCluster reconcile. + t.Run("a DeviceClass only in an unrelated group is an error, not an unsupported verdict", func(t *testing.T) { + ctx := testContext(t) + clusterWithoutAnyDeviceClass := newStubAPIServer(t, discoveryHandlers()) + unrelatedDeviceClassGroup := unrelatedDeviceClassAPIGroup() + clusterWithOnlyAnUnrelatedDeviceClass := newStubAPIServer(t, discoveryHandlers(unrelatedDeviceClassGroup)) + + absentDeviceClassGVR, absentDeviceClassDRASupported, absentDeviceClassErr := + getDRAResourceGVR(ctx, clusterWithoutAnyDeviceClass.restConfig) + + require.NoError(t, absentDeviceClassErr) + require.False(t, absentDeviceClassDRASupported) + require.Equal(t, schema.GroupVersionResource{}, absentDeviceClassGVR) + + unrelatedDeviceClassGVR, unrelatedDeviceClassDRASupported, unrelatedDeviceClassErr := + getDRAResourceGVR(ctx, clusterWithOnlyAnUnrelatedDeviceClass.restConfig) + + require.ErrorContains(t, unrelatedDeviceClassErr, + "could not determine the GVR for the DeviceClass resource from discovered group/versions: devices.example.com/v1") + require.False(t, unrelatedDeviceClassDRASupported) + require.Equal(t, schema.GroupVersionResource{}, unrelatedDeviceClassGVR) + assertRequestedPathSet(t, clusterWithoutAnyDeviceClass, discoveryPaths()) + assertRequestedPathSet(t, clusterWithOnlyAnUnrelatedDeviceClass, discoveryPaths(unrelatedDeviceClassGroup)) + }) + + // Suspected bug #7: an ErrGroupDiscoveryFailed is treated as continuable without checking which group + // failed, so an unreachable resource.k8s.io reads as "DRA not supported" with a nil error, not a retry. + t.Run("an unreachable resource.k8s.io reports DRA unsupported with no error", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + unreachableDRAGroup := unreachableDRAAPIGroup("v1") + apiServer := newStubAPIServer(t, discoveryHandlers(unreachableDRAGroup)) + + draResourceGVR, draSupported, err := getDRAResourceGVR(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + // The log line is the only operator-visible sign that the unsupported verdict was reached by + // giving up rather than by looking, so its wording and the cause it carries are behaviour. + capturedRecords := recorder.capturedRecords() + require.Len(t, capturedRecords, 1) + partialDiscoveryFailureRecord := capturedRecords[0] + require.Equal(t, logRecordLevelInfo, partialDiscoveryFailureRecord.level) + require.Equal(t, "partial API discovery failure; continuing with discovered groups", + partialDiscoveryFailureRecord.message) + // logr clamps V's negative level to zero before the sink sees it, so the warning level is + // unobservable; the recorded zero still catches a raise to debug, which would hide the line. + require.Equal(t, 0, partialDiscoveryFailureRecord.verbosity) + require.Len(t, partialDiscoveryFailureRecord.keysAndValues, 2) + require.Equal(t, "error", partialDiscoveryFailureRecord.keysAndValues[0]) + require.Contains(t, partialDiscoveryFailureRecord.keysAndValues[1], "resource.k8s.io/v1") + assertRequestedPathSet(t, apiServer, discoveryPaths(unreachableDRAGroup)) + }) +} + +func TestClusterInfoGetDRAResourceGVRCharacterization(t *testing.T) { + // Suspected bug #7 from the caller: the unsupported verdict carries no error, so a WithOneShot(true) + // instance caches it for life. main.go passes false, so today's accessor re-runs discovery every call. + t.Run("oneshot caches an unsupported verdict caused by a transient discovery failure", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, mergeHandlers(discoveryHandlers(unreachableDRAAPIGroup("v1")), + map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + pathDriverToolkitImageStream: respondNotFound(), + })) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + require.NoError(t, err) + groupListRequestCountDuringNew := len(apiServer.requestsTo(pathAPIGroups)) + // The exact count is client-go's, not the operator's: ServerPreferredResources repeats the whole + // walk defaultRetries times on a partial failure, so only "New ran discovery at all" is pinned here. + require.NotZero(t, groupListRequestCountDuringNew) + + draResourceGVR, draSupported, err := clusterInfoAPI.GetDRAResourceGVR() + + require.NoError(t, err) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + require.Len(t, apiServer.requestsTo(pathAPIGroups), groupListRequestCountDuringNew) + }) +} + +func TestGetOpenshiftDTKImagesCharacterization(t *testing.T) { + // Suspected bug #2: the IsNotFound branch logs and then falls through to logger.Error instead of + // returning, so a plain Kubernetes cluster logs an OpenShift-only ImageStream's absence as an error. + t.Run("a missing ImageStream is logged as an error as well as a plain absence", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{pathDriverToolkitImageStream: respondNotFound()}) + + require.Nil(t, getOpenshiftDTKImages(ctx, apiServer.restConfig)) + + require.Equal(t, + []string{"ocpHasDriverToolkitImageStream: driver-toolkit imagestream not found"}, + recorder.capturedMessages(logRecordLevelInfo)) + require.Equal(t, + []string{"Couldn't get the driver-toolkit imagestream"}, + recorder.capturedMessages(logRecordLevelError)) + assertRequestedPathSequence(t, apiServer, []string{pathDriverToolkitImageStream}) + }) +} + +func TestGetOpenshiftProxySpecCharacterization(t *testing.T) { + // Suspected bug #4: the ocpconfigv1.NewForConfig error is logged but not returned, so a nil + // *ConfigV1Client reaches Proxies().Get and the operator panics instead of reporting the failure. + t.Run("client construction failure panics instead of returning an error", func(t *testing.T) { + ctx := testContext(t) + + require.Panics(t, func() { + _, _ = getOpenshiftProxySpec(ctx, brokenRESTConfig()) + }) + }) +} + +func TestClusterInfoGetOpenshiftProxySpecCharacterization(t *testing.T) { + // Suspected bug #5: New's oneshot path never calls getOpenshiftProxySpec, so proxySpec stays nil and a + // future WithOneShot(true) caller would drop a configured cluster Proxy from the driver DaemonSet. + t.Run("a oneshot instance built by New never fetches the proxy", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, openshiftClusterHandlers) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + require.NoError(t, err) + + openshiftProxySpec, err := clusterInfoAPI.GetOpenshiftProxySpec() + + require.NoError(t, err) + require.Nil(t, openshiftProxySpec) + require.Empty(t, apiServer.requestsTo(pathProxy)) + }) +} + +func TestClusterInfoGetOpenshiftDriverToolkitImagesCharacterization(t *testing.T) { + // Suspected bug #6: the oneshot accessor returns the cached map itself rather than a copy. No current + // caller mutates it, so this is a trap for a future one rather than a defect an operator sees today. + t.Run("oneshot hands out the cached map itself, not a copy", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: true, + openshiftDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + } + + driverToolkitImages := clusterInfoUnderTest.GetOpenshiftDriverToolkitImages() + driverToolkitImages["412.86"] = "quay.io/openshift/driver-toolkit@sha256:bbb" + + require.Equal(t, driverToolkitImages, clusterInfoUnderTest.openshiftDriverToolkitImages) + }) +} diff --git a/controllers/clusterinfo/clusterinfo_test.go b/controllers/clusterinfo/clusterinfo_test.go index 9fc4890d3..0077abe56 100644 --- a/controllers/clusterinfo/clusterinfo_test.go +++ b/controllers/clusterinfo/clusterinfo_test.go @@ -17,66 +17,312 @@ package clusterinfo import ( + "context" + "net/http" "testing" + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" "github.com/NVIDIA/gpu-operator/internal/consts" ) -func TestGetRuntimeString(t *testing.T) { - tests := []struct { - description string - runtimeVer string - expected string - expectError bool +func requireClusterInfo(t *testing.T, clusterInfoAPI Interface) *clusterInfo { + t.Helper() + + clusterInfoUnderTest, ok := clusterInfoAPI.(*clusterInfo) + require.True(t, ok, "New returned %T, not *clusterInfo", clusterInfoAPI) + return clusterInfoUnderTest +} + +func TestNew(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + options []Option + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedContainerRuntime string + expectedOpenshiftVersion string + expectedDriverToolkitImages map[string]string + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedRequestPaths []string }{ { - description: "docker runtime", - runtimeVer: "docker://20.10.7", - expected: consts.Docker, + description: "non-oneshot construction queries nothing", + handlers: openshiftClusterHandlers, + options: []Option{WithOneShot(false)}, + }, + { + description: "oneshot defaults to off", + handlers: openshiftClusterHandlers, + }, + { + description: "oneshot on a cluster that is not OpenShift", + handlers: mergeHandlers(discoveryHandlers(), map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + pathDriverToolkitImageStream: respondNotFound(), + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{})), + }), + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.Containerd, + expectedOpenshiftVersion: "", + expectedDriverToolkitImages: nil, + expectedDRASupported: false, + expectedRequestPaths: append([]string{pathClusterVersion, pathNodes, pathDriverToolkitImageStream}, + discoveryPaths()...), }, { - description: "containerd runtime", - runtimeVer: "containerd://1.6.8", - expected: consts.Containerd, + description: "oneshot on an OpenShift cluster never lists nodes", + handlers: openshiftClusterHandlers, + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.CRIO, + expectedOpenshiftVersion: "4.14", + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + expectedRequestPaths: append([]string{pathClusterVersion, pathDriverToolkitImageStream}, + discoveryPaths(openshiftClusterDRAGroups...)...), }, { - description: "cri-o runtime", - runtimeVer: "cri-o://1.24.1", - expected: consts.CRIO, + description: "oneshot fails when the container runtime cannot be determined", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + options: []Option{WithOneShot(true)}, + expectedErrorContains: "failed to get container runtime", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, }, { - description: "unrecognized runtime returns error", - runtimeVer: "unknown-runtime://1.0.0", - expectError: true, + description: "oneshot tolerates a DriverToolkit lookup failure", + handlers: mergeHandlers(openshiftClusterHandlers, map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondServerError(), + }), + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.CRIO, + expectedOpenshiftVersion: "4.14", + expectedDriverToolkitImages: nil, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + expectedRequestPaths: append([]string{pathClusterVersion, pathDriverToolkitImageStream}, + discoveryPaths(openshiftClusterDRAGroups...)...), }, { - description: "empty runtime version returns error", - runtimeVer: "", - expectError: true, + description: "oneshot fails when DRA support cannot be determined", + handlers: mergeHandlers(openshiftClusterHandlers, map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondNotFound(), + pathAPIGroups: respondServerError(), + }), + options: []Option{WithOneShot(true)}, + expectedErrorContains: "failed to determine DRA support", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{ + pathClusterVersion, pathDriverToolkitImageStream, pathAPIVersions, pathAPIGroups, + }, }, } - for _, tc := range tests { - t.Run(tc.description, func(t *testing.T) { - node := corev1.Node{ - Status: corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ - ContainerRuntimeVersion: tc.runtimeVer, - }, - }, - } + // Suspected bug #5 owns whether New reaches the Proxy endpoint. Cases that could reach it still + // register a Proxy handler, because this exclusion does not reach the unhandled-path tripwire. + pathsOwnedByCharacterizationTests := []string{pathProxy} + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) - result, err := getRuntimeString(node) + options := append([]Option{WithKubernetesConfig(apiServer.restConfig)}, testCase.options...) - if tc.expectError { - require.Error(t, err) + clusterInfoAPI, err := New(ctx, options...) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + } + require.Nil(t, clusterInfoAPI) + assertRequestedPathSetExcluding(t, apiServer, testCase.expectedRequestPaths, pathsOwnedByCharacterizationTests...) return } + require.NoError(t, err) - require.Equal(t, tc.expected, result) + require.NotNil(t, clusterInfoAPI) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + + assert.Equal(t, testCase.expectedContainerRuntime, clusterInfoUnderTest.containerRuntime) + assert.Equal(t, testCase.expectedOpenshiftVersion, clusterInfoUnderTest.openshiftVersion) + assert.Equal(t, testCase.expectedDriverToolkitImages, clusterInfoUnderTest.openshiftDriverToolkitImages) + assert.Equal(t, testCase.expectedDRAResourceGVR, clusterInfoUnderTest.draResourceGVR) + assert.Equal(t, testCase.expectedDRASupported, clusterInfoUnderTest.draSupported) + assertRequestedPathSetExcluding(t, apiServer, testCase.expectedRequestPaths, pathsOwnedByCharacterizationTests...) }) } + + // Only the abort is contractual: the wording and which endpoints are reached on the way are + // suspected bug #1, owned by clusterinfo_characterization_test.go. + t.Run("oneshot fails when the OpenShift version cannot be determined", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.PartialUpdate, "4.15.0"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Error(t, err) + require.Nil(t, clusterInfoAPI) + assertEveryRequestWasARead(t, apiServer) + }) + + t.Run("oneshot fails when the OpenShift version lookup is denied", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Error(t, err) + requireAPIStatusError(t, err, metav1.StatusReasonForbidden, http.StatusForbidden) + require.Nil(t, clusterInfoAPI) + assertEveryRequestWasARead(t, apiServer) + }) + + t.Run("options are applied in order and the last one wins", func(t *testing.T) { + ctx := testContext(t) + firstConfig := &rest.Config{Host: "https://first.example.com:6443"} + secondConfig := &rest.Config{Host: "https://second.example.com:6443"} + + clusterInfoAPI, err := New(ctx, + WithKubernetesConfig(firstConfig), + WithKubernetesConfig(secondConfig), + WithOneShot(true), + WithOneShot(false), + ) + + require.NoError(t, err) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + require.Equal(t, secondConfig, clusterInfoUnderTest.config) + require.False(t, clusterInfoUnderTest.oneshot) + }) + + t.Run("New stores the context it was given", func(t *testing.T) { + ctx := testContext(t) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(&rest.Config{Host: "https://example.com:6443"})) + + require.NoError(t, err) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + require.Equal(t, ctx, clusterInfoUnderTest.ctx) + }) +} + +// Nothing in an accessor's return values reveals which context it passed down, so each subtest +// below spends the stored context and asserts the consequence, making a context.Background() visible. +func TestClusterInfoAccessorsUseTheStoredContext(t *testing.T) { + // The stub registers no handler but the catch-all, whose tripwire fires during cleanup if a + // request arrives: a cancelled context must stop each accessor before it reaches the wire. + newClusterInfoWithCancelledContext := func(t *testing.T) *clusterInfo { + t.Helper() + + ctx, cancel := context.WithCancel(testContext(t)) + cancel() + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{}) + return &clusterInfo{ctx: ctx, config: apiServer.restConfig} + } + + t.Run("GetContainerRuntime fails on a cancelled context", func(t *testing.T) { + containerRuntime, err := newClusterInfoWithCancelledContext(t).GetContainerRuntime() + + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, containerRuntime) + }) + + t.Run("GetOpenshiftVersion fails on a cancelled context", func(t *testing.T) { + openshiftVersion, err := newClusterInfoWithCancelledContext(t).GetOpenshiftVersion() + + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, openshiftVersion) + }) + + t.Run("GetOpenshiftProxySpec fails on a cancelled context", func(t *testing.T) { + openshiftProxySpec, err := newClusterInfoWithCancelledContext(t).GetOpenshiftProxySpec() + + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, openshiftProxySpec) + }) + + t.Run("GetOpenshiftDriverToolkitImages yields no images on a cancelled context", func(t *testing.T) { + require.Nil(t, newClusterInfoWithCancelledContext(t).GetOpenshiftDriverToolkitImages()) + }) + + // Discovery builds its own requests with context.TODO, so cancelling the stored context cannot + // stop this accessor; the logger it carries is the only trace that the stored context was used. + t.Run("GetDRAResourceGVR logs through the stored context", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, discoveryHandlers(draAPIGroup("v1"))) + clusterInfoUnderTest := &clusterInfo{ctx: ctx, config: apiServer.restConfig} + + draResourceGVR, draSupported, err := clusterInfoUnderTest.GetDRAResourceGVR() + + require.NoError(t, err) + require.True(t, draSupported) + require.Equal(t, draGVR("v1"), draResourceGVR) + require.Equal(t, + []string{"Discovered DeviceClass resource"}, + recorder.capturedMessages(logRecordLevelInfo)) + }) +} + +func TestOptions(t *testing.T) { + t.Run("WithKubernetesConfig stores the config it was given", func(t *testing.T) { + config := &rest.Config{Host: "https://example.com:6443"} + clusterInfoUnderTest := &clusterInfo{} + + WithKubernetesConfig(config)(clusterInfoUnderTest) + + require.Equal(t, config, clusterInfoUnderTest.config) + }) + + // A nil config is what makes New fall back to config.GetConfigOrDie, which calls os.Exit(1). + // Every test in this package therefore passes WithKubernetesConfig explicitly. + t.Run("WithKubernetesConfig accepts nil", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{config: &rest.Config{Host: "https://example.com:6443"}} + + WithKubernetesConfig(nil)(clusterInfoUnderTest) + + require.Nil(t, clusterInfoUnderTest.config) + }) + + t.Run("WithOneShot sets the flag", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{} + + WithOneShot(true)(clusterInfoUnderTest) + + require.True(t, clusterInfoUnderTest.oneshot) + }) + + t.Run("WithOneShot clears the flag", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{oneshot: true} + + WithOneShot(false)(clusterInfoUnderTest) + + require.False(t, clusterInfoUnderTest.oneshot) + }) } diff --git a/controllers/clusterinfo/dra_test.go b/controllers/clusterinfo/dra_test.go new file mode 100644 index 000000000..1cd606fd4 --- /dev/null +++ b/controllers/clusterinfo/dra_test.go @@ -0,0 +1,291 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" +) + +func TestGetDRAResourceGVR(t *testing.T) { + draGroupWithoutDeviceClass := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "resourceclaims", Kind: "ResourceClaim"}}, + } + unreachableMetricsGroup := discoveredAPIGroup{ + name: "metrics.k8s.io", + versionsInPreferenceOrder: []string{"v1beta1"}, + versionEndpointsFail: true, + } + // This group and draGroupNamingDeviceClassDifferently are the two halves of "the scan matches on + // Kind, never on the resource name". + deviceClassLookalikeGroup := discoveredAPIGroup{ + name: "policy.example.com", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "deviceclasses", Kind: "DeviceClassPolicy"}}, + } + draGroupNamingDeviceClassDifferently := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "gpudeviceclasses", Kind: "DeviceClass"}}, + } + // Two DeviceClass-kind resources under different names are distinct GroupResources, so discovery + // keeps v1beta1's deviceclasses and v1's gpudeviceclasses side by side instead of collapsing them. + draGroupServingTwoDeviceClassNames := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1beta1", "v1"}, + resourcesByVersion: map[string][]metav1.APIResource{ + "v1beta1": {deviceClassAPIResource()}, + "v1": {{Name: "gpudeviceclasses", Kind: "DeviceClass"}}, + }, + } + + testCases := []struct { + description string + groups []discoveredAPIGroup + handlerOverrides map[string]http.HandlerFunc + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedErrorContains string + expectedRequestPaths []string + }{ + { + description: "DeviceClass served at v1", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "DeviceClass served at v1beta2", + groups: []discoveredAPIGroup{draAPIGroup("v1beta2")}, + expectedDRAResourceGVR: draGVR("v1beta2"), + expectedDRASupported: true, + }, + { + description: "DeviceClass served at v1beta1", + groups: []discoveredAPIGroup{draAPIGroup("v1beta1")}, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "one deviceclasses at several versions collapses to the preferred v1", + groups: []discoveredAPIGroup{draAPIGroup("v1", "v1beta1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "one deviceclasses at several versions collapses to the preferred v1beta1", + groups: []discoveredAPIGroup{draAPIGroup("v1beta1", "v1")}, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "the ordered version preference outranks the group's preferred version", + groups: []discoveredAPIGroup{draGroupServingTwoDeviceClassNames}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a cluster without the resource.k8s.io group does not support DRA", + expectedDRASupported: false, + }, + { + description: "resource.k8s.io without a DeviceClass resource does not support DRA", + groups: []discoveredAPIGroup{draGroupWithoutDeviceClass}, + expectedDRASupported: false, + }, + { + description: "a resource named deviceclasses under another Kind does not support DRA", + groups: []discoveredAPIGroup{deviceClassLookalikeGroup}, + expectedDRASupported: false, + }, + { + // The Resource of the returned GVR is a constant, so it stays "deviceclasses" whatever + // name the group listed the Kind under. + description: "a DeviceClass listed under another resource name still supports DRA", + groups: []discoveredAPIGroup{draGroupNamingDeviceClassDifferently}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a DeviceClass in an unrelated group does not displace resource.k8s.io", + groups: []discoveredAPIGroup{draAPIGroup("v1"), unrelatedDeviceClassAPIGroup()}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a DeviceClass served only at an unsupported version is an error", + groups: []discoveredAPIGroup{draAPIGroup("v1alpha3")}, + expectedErrorContains: "could not determine the GVR for the DeviceClass resource", + }, + { + description: "an unreachable API group does not mask resource.k8s.io", + groups: []discoveredAPIGroup{draAPIGroup("v1"), unreachableMetricsGroup}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + // A failure listing the groups themselves is not an ErrGroupDiscoveryFailed, so there + // are no partial results to keep and discovery stops before any group is fetched. + description: "a failure listing the API groups is fatal", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIGroups: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions, pathAPIGroups}, + }, + { + description: "a failure listing the legacy API versions is fatal", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIVersions: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, mergeHandlers(discoveryHandlers(testCase.groups...), testCase.handlerOverrides)) + + draResourceGVR, draSupported, err := getDRAResourceGVR(ctx, apiServer.restConfig) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedDRASupported, draSupported) + require.Equal(t, testCase.expectedDRAResourceGVR, draResourceGVR) + } + + expectedPaths := testCase.expectedRequestPaths + if expectedPaths == nil { + expectedPaths = discoveryPaths(testCase.groups...) + } + assertRequestedPathSet(t, apiServer, expectedPaths) + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause + // is pinned by unwrapping it and comparing against the error the same constructor produces. + t.Run("client construction failure", func(t *testing.T) { + _, expectedClientError := discovery.NewDiscoveryClientForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + draResourceGVR, draSupported, err := getDRAResourceGVR(testContext(t), brokenRESTConfig()) + + require.ErrorContains(t, err, "error building discovery client") + require.EqualError(t, errors.Unwrap(err), expectedClientError.Error()) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + }) +} + +func TestClusterInfoGetDRAResourceGVR(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedDRAResourceGVR schema.GroupVersionResource + cachedDRASupported bool + groups []discoveredAPIGroup + handlerOverrides map[string]http.HandlerFunc + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedErrorContains string + expectedRequestPaths []string + }{ + { + // A nil rest.Config is the assertion: any API call would fail loudly. + description: "oneshot returns the cached GVR without any API call", + oneshot: true, + cachedDRAResourceGVR: draGVR("v1beta1"), + cachedDRASupported: true, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "oneshot reports a cluster cached as not supporting DRA", + oneshot: true, + expectedDRASupported: false, + }, + { + description: "non-oneshot queries the cluster", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "non-oneshot ignores the cached GVR", + cachedDRAResourceGVR: draGVR("v1beta1"), + cachedDRASupported: true, + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "non-oneshot propagates errors", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIGroups: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions, pathAPIGroups}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + draResourceGVR: testCase.cachedDRAResourceGVR, + draSupported: testCase.cachedDRASupported, + } + var apiServer *stubAPIServer + if !testCase.oneshot { + apiServer = newStubAPIServer(t, mergeHandlers(discoveryHandlers(testCase.groups...), testCase.handlerOverrides)) + clusterInfoUnderTest.config = apiServer.restConfig + } + + draResourceGVR, draSupported, err := clusterInfoUnderTest.GetDRAResourceGVR() + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedDRASupported, draSupported) + require.Equal(t, testCase.expectedDRAResourceGVR, draResourceGVR) + } + + if apiServer != nil { + expectedPaths := testCase.expectedRequestPaths + if expectedPaths == nil { + expectedPaths = discoveryPaths(testCase.groups...) + } + assertRequestedPathSet(t, apiServer, expectedPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/logging_test.go b/controllers/clusterinfo/logging_test.go new file mode 100644 index 000000000..81929971a --- /dev/null +++ b/controllers/clusterinfo/logging_test.go @@ -0,0 +1,131 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "context" + "slices" + "sync" + "testing" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +type logRecordLevel string + +const ( + logRecordLevelInfo logRecordLevel = "info" + logRecordLevelError logRecordLevel = "error" +) + +type logRecord struct { + level logRecordLevel + verbosity int + message string + err error + keysAndValues []any +} + +// logRecorder is held by pointer because logr copies a sink on every WithName and WithValues call, +// and every copy has to append to the same slice. +type logRecorder struct { + mu sync.Mutex + records []logRecord +} + +func (r *logRecorder) record(newRecord logRecord) { + r.mu.Lock() + defer r.mu.Unlock() + r.records = append(r.records, newRecord) +} + +func (r *logRecorder) capturedRecords() []logRecord { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.records) +} + +func (r *logRecorder) capturedRecordsAtLevel(level logRecordLevel) []logRecord { + var matchingRecords []logRecord + for _, capturedRecord := range r.capturedRecords() { + if capturedRecord.level == level { + matchingRecords = append(matchingRecords, capturedRecord) + } + } + return matchingRecords +} + +func (r *logRecorder) capturedMessages(level logRecordLevel) []string { + var messages []string + for _, capturedRecord := range r.capturedRecordsAtLevel(level) { + messages = append(messages, capturedRecord.message) + } + return messages +} + +// funcr.New is not used here because it routes Info and Error through one write function, which +// would leave the level recoverable only by parsing the formatted output. +type capturingLogSink struct { + funcr.Formatter + recorder *logRecorder +} + +func (s capturingLogSink) WithName(name string) logr.LogSink { + s.AddName(name) + return &s +} + +func (s capturingLogSink) WithValues(kvList ...any) logr.LogSink { + s.AddValues(kvList) + return &s +} + +func (s capturingLogSink) Info(verbosity int, message string, keysAndValues ...any) { + s.recorder.record(logRecord{ + level: logRecordLevelInfo, + verbosity: verbosity, + message: message, + keysAndValues: keysAndValues, + }) +} + +func (s capturingLogSink) Error(err error, message string, keysAndValues ...any) { + s.recorder.record(logRecord{ + level: logRecordLevelError, + message: message, + err: err, + keysAndValues: keysAndValues, + }) +} + +func capturingLogger() (logr.Logger, *logRecorder) { + recorder := &logRecorder{} + capturingSink := capturingLogSink{ + Formatter: funcr.NewFormatter(funcr.Options{Verbosity: consts.LogLevelDebug}), + recorder: recorder, + } + return logr.New(&capturingSink), recorder +} + +func testContextWithCapturedLogs(t *testing.T) (context.Context, *logRecorder) { + logger, recorder := capturingLogger() + return log.IntoContext(t.Context(), logger), recorder +} diff --git a/controllers/clusterinfo/openshift_dtk_test.go b/controllers/clusterinfo/openshift_dtk_test.go new file mode 100644 index 000000000..82a148c9a --- /dev/null +++ b/controllers/clusterinfo/openshift_dtk_test.go @@ -0,0 +1,249 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +const brokenImageStreamWarningMessage = "WARNING: ocpHasDriverToolkitImageStream: driver-toolkit imagestream is broken, see RHBZ#2015024" + +func TestGetOpenshiftDTKImages(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedDriverToolkitImages map[string]string + expectsBrokenImageStreamWarning bool + }{ + { + description: "every usable tag becomes a map entry", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + }, + { + description: "the floating latest tag is skipped", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest", "quay.io/openshift/driver-toolkit@sha256:zzz"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + }, + { + description: "a tag whose name only begins with latest is kept", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest-rhel9", "quay.io/openshift/driver-toolkit@sha256:ccc"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "latest-rhel9": "quay.io/openshift/driver-toolkit@sha256:ccc", + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + }, + }, + { + description: "a tag pointing at an ImageStreamTag is kept with that reference", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTagReferencingTag("410.84", "driver-toolkit:v4.10"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "driver-toolkit:v4.10", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + }, + { + description: "a tag without a From reference is skipped", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTagWithoutFrom("410.84"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{"412.86": "quay.io/openshift/driver-toolkit@sha256:bbb"}, + }, + { + description: "a tag with an empty name is skipped, see RHBZ#2015024", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("", "quay.io/openshift/driver-toolkit@sha256:zzz"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + expectsBrokenImageStreamWarning: true, + }, + { + description: "only a latest tag yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest", "quay.io/openshift/driver-toolkit@sha256:zzz"), + )), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "an ImageStream with no tags yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream()), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "a server error yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondServerError(), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "forbidden yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondForbidden(), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "duplicate tag names collapse to the last one", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:bbb"}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + driverToolkitImages := getOpenshiftDTKImages(ctx, apiServer.restConfig) + + require.Equal(t, testCase.expectedDriverToolkitImages, driverToolkitImages) + + // A skipped tag leaves the returned map identical to one where the tag was never there, so + // the warning is the only report that the ImageStream is broken rather than merely thin. + capturedInfoMessages := recorder.capturedMessages(logRecordLevelInfo) + if testCase.expectsBrokenImageStreamWarning { + require.Contains(t, capturedInfoMessages, brokenImageStreamWarningMessage) + } else { + require.NotContains(t, capturedInfoMessages, brokenImageStreamWarningMessage) + } + + assertRequestedPathSequence(t, apiServer, []string{pathDriverToolkitImageStream}) + }) + } + + t.Run("client construction failure is logged and yields a nil map", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + + require.Nil(t, getOpenshiftDTKImages(ctx, brokenRESTConfig())) + + // getOpenshiftDTKImages returns no error, so the log is the failure's only observable. + capturedErrorRecords := recorder.capturedRecordsAtLevel(logRecordLevelError) + require.Len(t, capturedErrorRecords, 1) + require.Equal(t, "failed to build openshift image stream client", capturedErrorRecords[0].message) + require.Error(t, capturedErrorRecords[0].err) + }) +} + +func TestClusterInfoGetOpenshiftDriverToolkitImages(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedDriverToolkitImages map[string]string + handlers map[string]http.HandlerFunc + expectedDriverToolkitImages map[string]string + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached map without any API call", + oneshot: true, + cachedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + }, + { + description: "oneshot returns a nil cached map", + oneshot: true, + cachedDriverToolkitImages: nil, + expectedDriverToolkitImages: nil, + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + expectedRequestPaths: []string{pathDriverToolkitImageStream}, + }, + { + description: "non-oneshot returns nil when the ImageStream is missing", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondNotFound(), + }, + expectedDriverToolkitImages: nil, + expectedRequestPaths: []string{pathDriverToolkitImageStream}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + openshiftDriverToolkitImages: testCase.cachedDriverToolkitImages, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + driverToolkitImages := clusterInfoUnderTest.GetOpenshiftDriverToolkitImages() + + require.Equal(t, testCase.expectedDriverToolkitImages, driverToolkitImages) + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/openshift_proxy_test.go b/controllers/clusterinfo/openshift_proxy_test.go new file mode 100644 index 000000000..4ae1372a8 --- /dev/null +++ b/controllers/clusterinfo/openshift_proxy_test.go @@ -0,0 +1,196 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetOpenshiftProxySpec(t *testing.T) { + populatedOpenshiftProxySpec := configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + HTTPSProxy: "https://proxy.example.com:3129", + NoProxy: ".cluster.local,10.0.0.0/8", + TrustedCA: configv1.ConfigMapNameReference{Name: "user-ca-bundle"}, + } + + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedOpenshiftProxySpec *configv1.ProxySpec + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedErrorIsNotFound bool + }{ + { + description: "fully populated proxy", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(populatedOpenshiftProxySpec)), + }, + expectedOpenshiftProxySpec: &populatedOpenshiftProxySpec, + }, + { + // A zero spec behind a non-nil pointer does not trip the nil guard in internal/state/driver.go. + description: "a cluster with no proxy configured yields a zero spec, not nil", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{})), + }, + expectedOpenshiftProxySpec: &configv1.ProxySpec{}, + }, + { + // Unlike getOpenshiftVersion, this function does not convert NotFound into a sentinel. + description: "a missing Proxy resource is an error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondNotFound(), + }, + expectedErrorReason: metav1.StatusReasonNotFound, + expectedErrorCode: http.StatusNotFound, + expectedErrorIsNotFound: true, + }, + { + description: "server error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondServerError(), + }, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + }, + { + description: "forbidden", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondForbidden(), + }, + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + openshiftProxySpec, err := getOpenshiftProxySpec(ctx, apiServer.restConfig) + + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Equal(t, testCase.expectedErrorIsNotFound, apierrors.IsNotFound(err)) + require.Nil(t, openshiftProxySpec) + } else { + require.NoError(t, err) + require.NotNil(t, openshiftProxySpec) + require.Equal(t, *testCase.expectedOpenshiftProxySpec, *openshiftProxySpec) + } + + assertRequestedPathSequence(t, apiServer, []string{pathProxy}) + }) + } +} + +func TestClusterInfoGetOpenshiftProxySpec(t *testing.T) { + cachedOpenshiftProxySpec := &configv1.ProxySpec{HTTPProxy: "http://proxy.example.com:3128"} + fetchedOpenshiftProxySpec := configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + HTTPSProxy: "https://proxy.example.com:3129", + NoProxy: ".cluster.local", + TrustedCA: configv1.ConfigMapNameReference{Name: "user-ca-bundle"}, + } + + testCases := []struct { + description string + oneshot bool + cachedOpenshiftProxySpec *configv1.ProxySpec + handlers map[string]http.HandlerFunc + expectedOpenshiftProxySpec *configv1.ProxySpec + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedErrorIsNotFound bool + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached spec without any API call", + oneshot: true, + cachedOpenshiftProxySpec: cachedOpenshiftProxySpec, + expectedOpenshiftProxySpec: cachedOpenshiftProxySpec, + }, + { + description: "oneshot returns a nil cached spec without an error", + oneshot: true, + cachedOpenshiftProxySpec: nil, + expectedOpenshiftProxySpec: nil, + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(fetchedOpenshiftProxySpec)), + }, + expectedOpenshiftProxySpec: &fetchedOpenshiftProxySpec, + expectedRequestPaths: []string{pathProxy}, + }, + { + description: "non-oneshot propagates a missing Proxy resource as an error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondNotFound(), + }, + expectedErrorReason: metav1.StatusReasonNotFound, + expectedErrorCode: http.StatusNotFound, + expectedErrorIsNotFound: true, + expectedRequestPaths: []string{pathProxy}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + proxySpec: testCase.cachedOpenshiftProxySpec, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + openshiftProxySpec, err := clusterInfoUnderTest.GetOpenshiftProxySpec() + + switch { + case testCase.expectedErrorReason != "": + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Equal(t, testCase.expectedErrorIsNotFound, apierrors.IsNotFound(err)) + require.Nil(t, openshiftProxySpec) + case testCase.oneshot: + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftProxySpec, openshiftProxySpec) + default: + require.NoError(t, err) + require.NotNil(t, openshiftProxySpec) + require.Equal(t, *testCase.expectedOpenshiftProxySpec, *openshiftProxySpec) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/openshift_version_test.go b/controllers/clusterinfo/openshift_version_test.go new file mode 100644 index 000000000..47509b78e --- /dev/null +++ b/controllers/clusterinfo/openshift_version_test.go @@ -0,0 +1,274 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + ocpconfigv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetOpenshiftVersion(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedOpenshiftVersion string + expectsError bool + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + }{ + { + description: "completed history entry yields major.minor", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "two-component version is returned unchanged", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "single-component version yields the major only", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4"))), + }, + expectedOpenshiftVersion: "4", + }, + { + description: "pre-release version keeps only the first two segments", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10-rc.1"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "partial entries are skipped in favour of the completed one", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory(configv1.PartialUpdate, "4.15.0"), + updateHistory(configv1.CompletedUpdate, "4.14.10"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "an entry with an unset state is skipped in favour of the completed one", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory("", "4.15.0"), + updateHistory(configv1.CompletedUpdate, "4.14.10"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "the first completed entry wins, not the newest", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory(configv1.CompletedUpdate, "4.14.10"), + updateHistory(configv1.CompletedUpdate, "4.12.1"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "no completed entry in history", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.PartialUpdate, "4.15.0"))), + }, + expectsError: true, + expectedErrorContains: "failed to find Completed Cluster Version", + }, + { + description: "empty history", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion()), + }, + expectsError: true, + expectedErrorContains: "failed to find Completed Cluster Version", + }, + { + description: "no ClusterVersion resource means the cluster is not OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + }, + expectedOpenshiftVersion: "", + }, + { + description: "server error is reported, not treated as not-OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + }, + expectsError: true, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + }, + { + description: "forbidden is reported, not treated as not-OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + }, + expectsError: true, + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + }, + { + description: "malformed response body", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithRawBody(http.StatusOK, "{"), + }, + expectsError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + openshiftVersion, err := getOpenshiftVersion(ctx, apiServer.restConfig) + + if testCase.expectsError { + require.Error(t, err) + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + } + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + } + // NotFound is the one status this function must never surface: it is converted to + // the not-OpenShift sentinel, so seeing it here would mean the conversion was lost. + require.False(t, apierrors.IsNotFound(err)) + require.Empty(t, openshiftVersion) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftVersion, openshiftVersion) + } + + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion}) + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause is + // pinned by comparing against the error the same constructor produces for the same config. + t.Run("client construction failure returns the client's own error", func(t *testing.T) { + _, expectedClientError := ocpconfigv1.NewForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + openshiftVersion, err := getOpenshiftVersion(testContext(t), brokenRESTConfig()) + + require.EqualError(t, err, expectedClientError.Error()) + require.False(t, apierrors.IsNotFound(err)) + require.Empty(t, openshiftVersion) + }) +} + +func TestClusterInfoGetOpenshiftVersion(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedOpenshiftVersion string + handlers map[string]http.HandlerFunc + expectedOpenshiftVersion string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached version without any API call", + oneshot: true, + cachedOpenshiftVersion: "4.14", + expectedOpenshiftVersion: "4.14", + }, + { + description: "oneshot returns an empty cached version", + oneshot: true, + cachedOpenshiftVersion: "", + expectedOpenshiftVersion: "", + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + }, + expectedOpenshiftVersion: "4.14", + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "non-oneshot reports a cluster that is not OpenShift as an empty version", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + }, + expectedOpenshiftVersion: "", + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "non-oneshot propagates errors", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + }, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + openshiftVersion: testCase.cachedOpenshiftVersion, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + openshiftVersion, err := clusterInfoUnderTest.GetOpenshiftVersion() + + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Empty(t, openshiftVersion) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftVersion, openshiftVersion) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/runtime_test.go b/controllers/clusterinfo/runtime_test.go new file mode 100644 index 000000000..460418fe1 --- /dev/null +++ b/controllers/clusterinfo/runtime_test.go @@ -0,0 +1,351 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +func TestGetRuntimeString(t *testing.T) { + testCases := []struct { + description string + containerRuntimeVersion string + expectedContainerRuntime string + expectedErrorContains string + }{ + { + description: "docker runtime", + containerRuntimeVersion: "docker://20.10.23", + expectedContainerRuntime: consts.Docker, + }, + { + description: "containerd runtime", + containerRuntimeVersion: "containerd://1.7.11", + expectedContainerRuntime: consts.Containerd, + }, + { + description: "cri-o runtime maps to the crio constant", + containerRuntimeVersion: "cri-o://1.28.2", + expectedContainerRuntime: consts.CRIO, + }, + { + description: "unrecognised runtime", + containerRuntimeVersion: "rkt://1.0.0", + expectedErrorContains: "runtime not recognized: rkt://1.0.0", + }, + { + description: "node has not reported a runtime yet", + containerRuntimeVersion: "", + expectedErrorContains: "runtime not recognized: ", + }, + { + description: "bare prefix without a version suffix", + containerRuntimeVersion: "containerd", + expectedContainerRuntime: consts.Containerd, + }, + { + description: "runtime name appearing as a substring but not a prefix", + containerRuntimeVersion: "my-docker://1.0", + expectedErrorContains: "runtime not recognized: my-docker://1.0", + }, + { + description: "matching is case sensitive", + containerRuntimeVersion: "Docker://20.10", + expectedErrorContains: "runtime not recognized: Docker://20.10", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + node := corev1.Node{ + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: testCase.containerRuntimeVersion}, + }, + } + + containerRuntime, err := getRuntimeString(node) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.Empty(t, containerRuntime) + return + } + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + }) + } +} + +func TestGetContainerRuntime(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedRequestPaths []string + }{ + { + description: "OpenShift short-circuits to crio without listing nodes", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "single containerd node", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "single docker node", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "single cri-o node on a cluster that is not OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("cri-o://1.28.2")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "containerd wins when it appears last", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "containerd wins when it appears first", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("containerd://1.7.11", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "no GPU nodes defaults to containerd", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList()), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "every node reports an unrecognised runtime, defaulting to containerd", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("rkt://1.0", "kata://3.2.0")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "an unrecognised node does not abort the scan", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("rkt://1.0", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Only this ordering catches a missing skip; in the case above the later node overwrites it anyway. + description: "an unrecognised node after a recognised one does not overwrite the result", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "rkt://1.0")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "node listing failure is fatal", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "node listing denied by RBAC is fatal", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondForbidden(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + containerRuntime, err := getContainerRuntime(ctx, apiServer.restConfig) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Empty(t, containerRuntime) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + } + + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + for _, request := range apiServer.requestsTo(pathNodes) { + assert.Equal(t, consts.GPUPresentLabel+"=true", request.query.Get("labelSelector")) + } + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause is + // pinned by comparing against the error the same constructor produces for the same config. + t.Run("client construction failure returns the client's own error", func(t *testing.T) { + _, expectedClientError := corev1client.NewForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + containerRuntime, err := getContainerRuntime(testContext(t), brokenRESTConfig()) + + require.EqualError(t, err, expectedClientError.Error()) + require.Empty(t, containerRuntime) + }) +} + +func TestClusterInfoGetContainerRuntime(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedContainerRuntime string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorContains string + expectedRequestPaths []string + }{ + { + // A nil rest.Config is the assertion: any API call would fail loudly. + description: "oneshot returns the cached runtime without any API call", + oneshot: true, + cachedContainerRuntime: consts.Containerd, + expectedContainerRuntime: consts.Containerd, + }, + { + description: "oneshot returns an empty cached runtime rather than re-fetching", + oneshot: true, + cachedContainerRuntime: "", + expectedContainerRuntime: "", + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "non-oneshot propagates errors", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "non-oneshot ignores the cached runtime", + cachedContainerRuntime: consts.Docker, + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + containerRuntime: testCase.cachedContainerRuntime, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + containerRuntime, err := clusterInfoUnderTest.GetContainerRuntime() + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.Empty(t, containerRuntime) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/stubs_test.go b/controllers/clusterinfo/stubs_test.go new file mode 100644 index 000000000..a236a3098 --- /dev/null +++ b/controllers/clusterinfo/stubs_test.go @@ -0,0 +1,488 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "sync" + "testing" + + "github.com/go-logr/logr" + configv1 "github.com/openshift/api/config/v1" + imagev1 "github.com/openshift/api/image/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +const ( + pathNodes = "/api/v1/nodes" + pathClusterVersion = "/apis/config.openshift.io/v1/clusterversions/version" + pathProxy = "/apis/config.openshift.io/v1/proxies/cluster" + pathDriverToolkitImageStream = "/apis/image.openshift.io/v1/namespaces/openshift/imagestreams/driver-toolkit" + + // The core group's resource list is reached through the legacy /api prefix, not through /apis. + pathAPIVersions = "/api" + pathAPIGroups = "/apis" + pathCoreResources = "/api/v1" +) + +func pathGroupVersionResources(groupVersion string) string { + return "/apis/" + groupVersion +} + +type recordedRequest struct { + method string + path string + query url.Values +} + +// stubAPIServer keeps "a handler answered 404" distinct from "no handler was registered for this +// URL", so a wrong-path assumption cannot pass as a legitimate NotFound. +type stubAPIServer struct { + restConfig *rest.Config + + mu sync.Mutex + requests []recordedRequest + unhandledRequestPaths []string +} + +// An unregistered path fails the test during cleanup rather than at request time, so the case's +// own assertions report first. +func newStubAPIServer(t *testing.T, handlers map[string]http.HandlerFunc) *stubAPIServer { + t.Helper() + + apiServer := &stubAPIServer{} + mux := http.NewServeMux() + for path, handler := range handlers { + mux.HandleFunc(path, apiServer.recordingHandler(handler)) + } + mux.HandleFunc("/", apiServer.recordingHandler(func(w http.ResponseWriter, r *http.Request) { + apiServer.noteUnhandled(r.URL.Path) + respondNotFound()(w, r) + })) + + server := httptest.NewServer(mux) + apiServer.restConfig = &rest.Config{ + Host: server.URL, + // Pinning JSON makes rest.Request.UseProtobufAsDefault a no-op, so the core/v1 node + // client — which prefers protobuf — also talks JSON to this stub. + ContentConfig: rest.ContentConfig{ContentType: "application/json"}, + } + + t.Cleanup(func() { + require.Empty(t, apiServer.unhandledPaths(), "client requested paths that no stub handler was registered for") + }) + t.Cleanup(server.Close) + + return apiServer +} + +func (s *stubAPIServer) recordingHandler(handler http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests = append(s.requests, recordedRequest{method: r.Method, path: r.URL.Path, query: r.URL.Query()}) + s.mu.Unlock() + handler(w, r) + } +} + +func (s *stubAPIServer) noteUnhandled(path string) { + s.mu.Lock() + defer s.mu.Unlock() + s.unhandledRequestPaths = append(s.unhandledRequestPaths, path) +} + +func (s *stubAPIServer) unhandledPaths() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.unhandledRequestPaths) +} + +// recordedRequests is the only place the read side takes the mutex; the accessors below derive +// from its clone, so none of them can hand out a slice the recording handler still appends to. +func (s *stubAPIServer) recordedRequests() []recordedRequest { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.requests) +} + +func (s *stubAPIServer) requestedPaths() []string { + var paths []string + for _, request := range s.recordedRequests() { + paths = append(paths, request.path) + } + return paths +} + +func (s *stubAPIServer) requestsTo(path string) []recordedRequest { + var matchingRequests []recordedRequest + for _, request := range s.recordedRequests() { + if request.path == path { + matchingRequests = append(matchingRequests, request) + } + } + return matchingRequests +} + +func (s *stubAPIServer) distinctRequestedPaths() []string { + return slices.Compact(slices.Sorted(slices.Values(s.requestedPaths()))) +} + +func assertRequestedPathSequence(t *testing.T, apiServer *stubAPIServer, expectedPaths []string) { + t.Helper() + + assert.Equal(t, expectedPaths, apiServer.requestedPaths()) + assertEveryRequestWasARead(t, apiServer) +} + +// assertRequestedPathSet, like the sequence form, asserts every request was a read, but discovery +// fans out in parallel and repeats the walk on failure, so its order and repeat count are not pinned. +func assertRequestedPathSet(t *testing.T, apiServer *stubAPIServer, expectedPaths []string) { + t.Helper() + + assert.ElementsMatch(t, expectedPaths, apiServer.distinctRequestedPaths()) + assertEveryRequestWasARead(t, apiServer) +} + +func assertRequestedPathSetExcluding(t *testing.T, apiServer *stubAPIServer, expectedPaths []string, excludedPaths ...string) { + t.Helper() + + comparedPaths := slices.DeleteFunc(apiServer.distinctRequestedPaths(), func(path string) bool { + return slices.Contains(excludedPaths, path) + }) + assert.ElementsMatch(t, expectedPaths, comparedPaths) + assertEveryRequestWasARead(t, apiServer) +} + +func assertEveryRequestWasARead(t *testing.T, apiServer *stubAPIServer) { + t.Helper() + + for _, request := range apiServer.recordedRequests() { + assert.Equal(t, http.MethodGet, request.method, "unexpected %s to %s", request.method, request.path) + } +} + +// The Content-Type header is what client-go uses to choose a decoder; without it Go's +// DetectContentType labels a JSON body text/plain and decoding fails with "no serializer". +func respondWithJSON(statusCode int, responseObject any) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + body, err := json.Marshal(responseObject) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write(body) + } +} + +func respondWithRawBody(statusCode int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + } +} + +func respondWithFailureStatus(statusCode int32, reason metav1.StatusReason, message string) http.HandlerFunc { + return respondWithJSON(int(statusCode), &metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, + Message: message, + Reason: reason, + Code: statusCode, + }) +} + +func respondNotFound() http.HandlerFunc { + return respondWithFailureStatus(http.StatusNotFound, metav1.StatusReasonNotFound, + "the server could not find the requested resource") +} + +// respondServerError deliberately omits Retry-After. client-go only retries a 5xx that carries it, +// so the request fails immediately instead of sleeping through ten retries. +func respondServerError() http.HandlerFunc { + return respondWithFailureStatus(http.StatusInternalServerError, metav1.StatusReasonInternalError, + "the server encountered an internal error") +} + +func respondForbidden() http.HandlerFunc { + return respondWithFailureStatus(http.StatusForbidden, metav1.StatusReasonForbidden, + "forbidden: user cannot get the requested resource") +} + +// brokenRESTConfig makes every *client.NewForConfig call fail: transport.TLSConfigFor rejects a +// config that supplies CA data alongside Insecure. +func brokenRESTConfig() *rest.Config { + return &rest.Config{ + Host: "https://127.0.0.1:1", + TLSClientConfig: rest.TLSClientConfig{Insecure: true, CAData: []byte("not-a-cert")}, + } +} + +func testContext(t *testing.T) context.Context { + return log.IntoContext(t.Context(), logr.Discard()) +} + +// requireAPIStatusError discriminates on the typed error the client built rather than on the stubs' +// own message text, so it also fails if production code drops the %w that carries the cause. +func requireAPIStatusError(t *testing.T, err error, expectedReason metav1.StatusReason, expectedCode int32) { + t.Helper() + + var statusError *apierrors.StatusError + require.ErrorAs(t, err, &statusError) + require.Equal(t, expectedReason, statusError.Status().Reason) + require.Equal(t, expectedCode, statusError.Status().Code) +} + +func clusterVersion(history ...configv1.UpdateHistory) *configv1.ClusterVersion { + return &configv1.ClusterVersion{ + TypeMeta: metav1.TypeMeta{APIVersion: configv1.GroupVersion.String(), Kind: "ClusterVersion"}, + ObjectMeta: metav1.ObjectMeta{Name: "version"}, + Status: configv1.ClusterVersionStatus{History: history}, + } +} + +func updateHistory(state configv1.UpdateState, version string) configv1.UpdateHistory { + return configv1.UpdateHistory{State: state, Version: version} +} + +func clusterProxy(spec configv1.ProxySpec) *configv1.Proxy { + return &configv1.Proxy{ + TypeMeta: metav1.TypeMeta{APIVersion: configv1.GroupVersion.String(), Kind: "Proxy"}, + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Spec: spec, + } +} + +func driverToolkitImageStream(tags ...imagev1.TagReference) *imagev1.ImageStream { + return &imagev1.ImageStream{ + TypeMeta: metav1.TypeMeta{APIVersion: imagev1.GroupVersion.String(), Kind: "ImageStream"}, + ObjectMeta: metav1.ObjectMeta{Name: "driver-toolkit", Namespace: consts.OpenshiftNamespace}, + Spec: imagev1.ImageStreamSpec{Tags: tags}, + } +} + +func imageStreamTag(name, image string) imagev1.TagReference { + return imagev1.TagReference{ + Name: name, + From: &corev1.ObjectReference{Kind: "DockerImage", Name: image}, + } +} + +// Real ImageStreams carry both the ImageStreamTag and the DockerImage reference forms. +func imageStreamTagReferencingTag(name, referencedTag string) imagev1.TagReference { + return imagev1.TagReference{ + Name: name, + From: &corev1.ObjectReference{Kind: "ImageStreamTag", Name: referencedTag}, + } +} + +func imageStreamTagWithoutFrom(name string) imagev1.TagReference { + return imagev1.TagReference{Name: name} +} + +func gpuNodeList(containerRuntimeVersions ...string) *corev1.NodeList { + nodes := make([]corev1.Node, 0, len(containerRuntimeVersions)) + for index, containerRuntimeVersion := range containerRuntimeVersions { + nodes = append(nodes, corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("gpu-node-%d", index), + Labels: map[string]string{consts.GPUPresentLabel: "true"}, + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: containerRuntimeVersion}, + }, + }) + } + return &corev1.NodeList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "NodeList"}, + Items: nodes, + } +} + +// The first version listed is the group's preferred version, which is the one discovery keeps for +// a resource served by several. +type discoveredAPIGroup struct { + name string + versionsInPreferenceOrder []string + resources []metav1.APIResource + // Discovery deduplicates on group plus resource name, so only per-version resource sets let two + // versions of a resource both survive. + resourcesByVersion map[string][]metav1.APIResource + versionEndpointsFail bool +} + +func (group discoveredAPIGroup) resourcesServedAt(version string) []metav1.APIResource { + if resourcesForVersion, found := group.resourcesByVersion[version]; found { + return resourcesForVersion + } + return group.resources +} + +func draGVR(version string) schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "resource.k8s.io", Version: version, Resource: "deviceclasses"} +} + +// draAPIGroup lists the group's other resources ahead of deviceclasses, so the scan has to walk +// past non-matching entries rather than finding its match at index 0. +func draAPIGroup(versionsInPreferenceOrder ...string) discoveredAPIGroup { + return discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: versionsInPreferenceOrder, + resources: []metav1.APIResource{ + {Name: "resourceclaims", Kind: "ResourceClaim"}, + {Name: "resourceclaimtemplates", Kind: "ResourceClaimTemplate"}, + {Name: "resourceslices", Kind: "ResourceSlice"}, + deviceClassAPIResource(), + }, + } +} + +// A group that is advertised but whose version endpoints all fail is what drives +// ServerPreferredResources into its partial-failure path. +func unreachableDRAAPIGroup(versionsInPreferenceOrder ...string) discoveredAPIGroup { + draGroup := draAPIGroup(versionsInPreferenceOrder...) + draGroup.versionEndpointsFail = true + return draGroup +} + +func unrelatedDeviceClassAPIGroup() discoveredAPIGroup { + return discoveredAPIGroup{ + name: "devices.example.com", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{deviceClassAPIResource()}, + } +} + +func deviceClassAPIResource() metav1.APIResource { + return metav1.APIResource{Name: "deviceclasses", Kind: "DeviceClass"} +} + +// ServerPreferredResources fetches every version of every group, not only each group's preferred +// version, so each listed version needs a handler. +func discoveryHandlers(groups ...discoveredAPIGroup) map[string]http.HandlerFunc { + handlers := map[string]http.HandlerFunc{ + pathAPIVersions: respondWithJSON(http.StatusOK, apiVersions("v1")), + pathAPIGroups: respondWithJSON(http.StatusOK, apiGroupList(groups...)), + pathCoreResources: respondWithJSON(http.StatusOK, + apiResourceList("v1", metav1.APIResource{Name: "nodes", Kind: "Node"})), + } + for _, group := range groups { + for _, version := range group.versionsInPreferenceOrder { + groupVersion := group.name + "/" + version + if group.versionEndpointsFail { + handlers[pathGroupVersionResources(groupVersion)] = respondServerError() + continue + } + handlers[pathGroupVersionResources(groupVersion)] = respondWithJSON(http.StatusOK, + apiResourceList(groupVersion, group.resourcesServedAt(version)...)) + } + } + return handlers +} + +// Must stay in sync with discoveryHandlers. +func discoveryPaths(groups ...discoveredAPIGroup) []string { + paths := []string{pathAPIVersions, pathAPIGroups, pathCoreResources} + for _, group := range groups { + for _, version := range group.versionsInPreferenceOrder { + paths = append(paths, pathGroupVersionResources(group.name+"/"+version)) + } + } + return paths +} + +func mergeHandlers(handlerMaps ...map[string]http.HandlerFunc) map[string]http.HandlerFunc { + mergedHandlers := map[string]http.HandlerFunc{} + for _, handlerMap := range handlerMaps { + maps.Copy(mergedHandlers, handlerMap) + } + return mergedHandlers +} + +var openshiftClusterDRAGroups = []discoveredAPIGroup{draAPIGroup("v1")} + +var openshiftClusterHandlers = mergeHandlers(discoveryHandlers(openshiftClusterDRAGroups...), + map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("cri-o://1.28.2")), + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + })), + }) + +func apiVersions(versions ...string) *metav1.APIVersions { + return &metav1.APIVersions{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIVersions"}, + Versions: versions, + } +} + +func apiGroupList(groups ...discoveredAPIGroup) *metav1.APIGroupList { + groupList := &metav1.APIGroupList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIGroupList"}, + } + for _, group := range groups { + groupList.Groups = append(groupList.Groups, apiGroup(group)) + } + return groupList +} + +func apiGroup(group discoveredAPIGroup) metav1.APIGroup { + advertisedGroup := metav1.APIGroup{Name: group.name} + for _, version := range group.versionsInPreferenceOrder { + advertisedGroup.Versions = append(advertisedGroup.Versions, metav1.GroupVersionForDiscovery{ + GroupVersion: group.name + "/" + version, + Version: version, + }) + } + if len(advertisedGroup.Versions) == 0 { + panic(fmt.Sprintf("API group fixture %q lists no versions, so it has no preferred version", group.name)) + } + advertisedGroup.PreferredVersion = advertisedGroup.Versions[0] + return advertisedGroup +} + +func apiResourceList(groupVersion string, resources ...metav1.APIResource) *metav1.APIResourceList { + return &metav1.APIResourceList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIResourceList"}, + GroupVersion: groupVersion, + APIResources: resources, + } +}