diff --git a/controllers/clusterinfo/clusterinfo_characterization_test.go b/controllers/clusterinfo/clusterinfo_characterization_test.go new file mode 100644 index 000000000..55adaa40c --- /dev/null +++ b/controllers/clusterinfo/clusterinfo_characterization_test.go @@ -0,0 +1,313 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// Characterization tests: every test in this file locks in behaviour that looks wrong, so that a +// change to it surfaces in review instead of silently altering what the operator does. They are +// deliberately not assertions of intended behaviour, and a failure here is as likely to mean the +// implementation was fixed as that it regressed — read the referenced bug before touching the test. + +// expectedErrorLogRecord pins the error value alongside the message, because suspected bug #1 is +// that the failed lookup survives only as a log line, so the value it carries is the whole record. +type expectedErrorLogRecord struct { + message string + statusReason metav1.StatusReason + statusCode int32 +} + +func TestGetContainerRuntimeCharacterization(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorLogs []expectedErrorLogRecord + expectedRequestPaths []string + }{ + { + // Suspected bug #1: getContainerRuntime logs the getOpenshiftVersion error and carries on with the + // empty version, so the crio short-circuit is skipped and the failed read is only ever a log line. + description: "an OpenShift version lookup failure is swallowed and node inspection continues", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedErrorLogs: []expectedErrorLogRecord{{ + message: "failed to retrieve", + statusReason: metav1.StatusReasonInternalError, + statusCode: http.StatusInternalServerError, + }}, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #8: a Completed history entry with an empty version returns the same ("", nil) + // pair as "not OpenShift", so a real OpenShift cluster has its runtime read off its nodes instead. + description: "an OpenShift cluster with an empty completed version falls through to node inspection", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, ""))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #9: only a containerd node breaks out of the scan, so every later node that parses + // overwrites the verdict and a docker/cri-o cluster's runtime flips with the apiserver's order. + description: "without a containerd node the last node listed decides: docker then cri-o", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "cri-o://1.28.2")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Suspected bug #9 with the same nodes reversed: the scan's last write wins either way, so the + // verdict tracks list order and no precedence between docker and cri-o is being applied. + description: "without a containerd node the last node listed decides: cri-o then docker", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("cri-o://1.28.2", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + containerRuntime, err := getContainerRuntime(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + capturedErrorRecords := recorder.capturedRecordsAtLevel(logRecordLevelError) + require.Len(t, capturedErrorRecords, len(testCase.expectedErrorLogs)) + for index, expectedErrorLog := range testCase.expectedErrorLogs { + require.Equal(t, expectedErrorLog.message, capturedErrorRecords[index].message) + requireAPIStatusError(t, capturedErrorRecords[index].err, + expectedErrorLog.statusReason, expectedErrorLog.statusCode) + } + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + }) + } +} + +func TestNewCharacterization(t *testing.T) { + // Suspected bug #1 from the caller: getContainerRuntime swallows the denied lookup, so New lists nodes + // first and only its own second lookup aborts, under the "failed to get openshift version" wrapper. + t.Run("a denied OpenShift lookup aborts New only after nodes have been listed", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Nil(t, clusterInfoAPI) + require.ErrorContains(t, err, "failed to get openshift version") + requireAPIStatusError(t, err, metav1.StatusReasonForbidden, http.StatusForbidden) + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion, pathNodes, pathClusterVersion}) + }) +} + +func TestGetOpenshiftVersionCharacterization(t *testing.T) { + // Suspected bug #8: strings.Split("", ".") yields one empty element, so a Completed entry carrying no + // version returns ("", nil) and an OpenShift cluster is indistinguishable from a plain Kubernetes one. + t.Run("a completed entry with an empty version yields the not-OpenShift sentinel", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, ""))), + }) + + openshiftVersion, err := getOpenshiftVersion(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.Empty(t, openshiftVersion) + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion}) + }) +} + +func TestGetDRAResourceGVRCharacterization(t *testing.T) { + // Suspected bug #3: the DeviceClass scan spans every API group before the resource.k8s.io filter, so an + // unrelated CRD of that Kind bypasses the unsupported early return and fails a GPUCluster reconcile. + t.Run("a DeviceClass only in an unrelated group is an error, not an unsupported verdict", func(t *testing.T) { + ctx := testContext(t) + clusterWithoutAnyDeviceClass := newStubAPIServer(t, discoveryHandlers()) + unrelatedDeviceClassGroup := unrelatedDeviceClassAPIGroup() + clusterWithOnlyAnUnrelatedDeviceClass := newStubAPIServer(t, discoveryHandlers(unrelatedDeviceClassGroup)) + + absentDeviceClassGVR, absentDeviceClassDRASupported, absentDeviceClassErr := + getDRAResourceGVR(ctx, clusterWithoutAnyDeviceClass.restConfig) + + require.NoError(t, absentDeviceClassErr) + require.False(t, absentDeviceClassDRASupported) + require.Equal(t, schema.GroupVersionResource{}, absentDeviceClassGVR) + + unrelatedDeviceClassGVR, unrelatedDeviceClassDRASupported, unrelatedDeviceClassErr := + getDRAResourceGVR(ctx, clusterWithOnlyAnUnrelatedDeviceClass.restConfig) + + require.ErrorContains(t, unrelatedDeviceClassErr, + "could not determine the GVR for the DeviceClass resource from discovered group/versions: devices.example.com/v1") + require.False(t, unrelatedDeviceClassDRASupported) + require.Equal(t, schema.GroupVersionResource{}, unrelatedDeviceClassGVR) + assertRequestedPathSet(t, clusterWithoutAnyDeviceClass, discoveryPaths()) + assertRequestedPathSet(t, clusterWithOnlyAnUnrelatedDeviceClass, discoveryPaths(unrelatedDeviceClassGroup)) + }) + + // Suspected bug #7: an ErrGroupDiscoveryFailed is treated as continuable without checking which group + // failed, so an unreachable resource.k8s.io reads as "DRA not supported" with a nil error, not a retry. + t.Run("an unreachable resource.k8s.io reports DRA unsupported with no error", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + unreachableDRAGroup := unreachableDRAAPIGroup("v1") + apiServer := newStubAPIServer(t, discoveryHandlers(unreachableDRAGroup)) + + draResourceGVR, draSupported, err := getDRAResourceGVR(ctx, apiServer.restConfig) + + require.NoError(t, err) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + // The log line is the only operator-visible sign that the unsupported verdict was reached by + // giving up rather than by looking, so its wording and the cause it carries are behaviour. + capturedRecords := recorder.capturedRecords() + require.Len(t, capturedRecords, 1) + partialDiscoveryFailureRecord := capturedRecords[0] + require.Equal(t, logRecordLevelInfo, partialDiscoveryFailureRecord.level) + require.Equal(t, "partial API discovery failure; continuing with discovered groups", + partialDiscoveryFailureRecord.message) + // logr clamps V's negative level to zero before the sink sees it, so the warning level is + // unobservable; the recorded zero still catches a raise to debug, which would hide the line. + require.Equal(t, 0, partialDiscoveryFailureRecord.verbosity) + require.Len(t, partialDiscoveryFailureRecord.keysAndValues, 2) + require.Equal(t, "error", partialDiscoveryFailureRecord.keysAndValues[0]) + require.Contains(t, partialDiscoveryFailureRecord.keysAndValues[1], "resource.k8s.io/v1") + assertRequestedPathSet(t, apiServer, discoveryPaths(unreachableDRAGroup)) + }) +} + +func TestClusterInfoGetDRAResourceGVRCharacterization(t *testing.T) { + // Suspected bug #7 from the caller: the unsupported verdict carries no error, so a WithOneShot(true) + // instance caches it for life. main.go passes false, so today's accessor re-runs discovery every call. + t.Run("oneshot caches an unsupported verdict caused by a transient discovery failure", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, mergeHandlers(discoveryHandlers(unreachableDRAAPIGroup("v1")), + map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + pathDriverToolkitImageStream: respondNotFound(), + })) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + require.NoError(t, err) + groupListRequestCountDuringNew := len(apiServer.requestsTo(pathAPIGroups)) + // The exact count is client-go's, not the operator's: ServerPreferredResources repeats the whole + // walk defaultRetries times on a partial failure, so only "New ran discovery at all" is pinned here. + require.NotZero(t, groupListRequestCountDuringNew) + + draResourceGVR, draSupported, err := clusterInfoAPI.GetDRAResourceGVR() + + require.NoError(t, err) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + require.Len(t, apiServer.requestsTo(pathAPIGroups), groupListRequestCountDuringNew) + }) +} + +func TestGetOpenshiftDTKImagesCharacterization(t *testing.T) { + // Suspected bug #2: the IsNotFound branch logs and then falls through to logger.Error instead of + // returning, so a plain Kubernetes cluster logs an OpenShift-only ImageStream's absence as an error. + t.Run("a missing ImageStream is logged as an error as well as a plain absence", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{pathDriverToolkitImageStream: respondNotFound()}) + + require.Nil(t, getOpenshiftDTKImages(ctx, apiServer.restConfig)) + + require.Equal(t, + []string{"ocpHasDriverToolkitImageStream: driver-toolkit imagestream not found"}, + recorder.capturedMessages(logRecordLevelInfo)) + require.Equal(t, + []string{"Couldn't get the driver-toolkit imagestream"}, + recorder.capturedMessages(logRecordLevelError)) + assertRequestedPathSequence(t, apiServer, []string{pathDriverToolkitImageStream}) + }) +} + +func TestGetOpenshiftProxySpecCharacterization(t *testing.T) { + // Suspected bug #4: the ocpconfigv1.NewForConfig error is logged but not returned, so a nil + // *ConfigV1Client reaches Proxies().Get and the operator panics instead of reporting the failure. + t.Run("client construction failure panics instead of returning an error", func(t *testing.T) { + ctx := testContext(t) + + require.Panics(t, func() { + _, _ = getOpenshiftProxySpec(ctx, brokenRESTConfig()) + }) + }) +} + +func TestClusterInfoGetOpenshiftProxySpecCharacterization(t *testing.T) { + // Suspected bug #5: New's oneshot path never calls getOpenshiftProxySpec, so proxySpec stays nil and a + // future WithOneShot(true) caller would drop a configured cluster Proxy from the driver DaemonSet. + t.Run("a oneshot instance built by New never fetches the proxy", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, openshiftClusterHandlers) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + require.NoError(t, err) + + openshiftProxySpec, err := clusterInfoAPI.GetOpenshiftProxySpec() + + require.NoError(t, err) + require.Nil(t, openshiftProxySpec) + require.Empty(t, apiServer.requestsTo(pathProxy)) + }) +} + +func TestClusterInfoGetOpenshiftDriverToolkitImagesCharacterization(t *testing.T) { + // Suspected bug #6: the oneshot accessor returns the cached map itself rather than a copy. No current + // caller mutates it, so this is a trap for a future one rather than a defect an operator sees today. + t.Run("oneshot hands out the cached map itself, not a copy", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: true, + openshiftDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + } + + driverToolkitImages := clusterInfoUnderTest.GetOpenshiftDriverToolkitImages() + driverToolkitImages["412.86"] = "quay.io/openshift/driver-toolkit@sha256:bbb" + + require.Equal(t, driverToolkitImages, clusterInfoUnderTest.openshiftDriverToolkitImages) + }) +} diff --git a/controllers/clusterinfo/clusterinfo_test.go b/controllers/clusterinfo/clusterinfo_test.go index 9fc4890d3..0077abe56 100644 --- a/controllers/clusterinfo/clusterinfo_test.go +++ b/controllers/clusterinfo/clusterinfo_test.go @@ -17,66 +17,312 @@ package clusterinfo import ( + "context" + "net/http" "testing" + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" "github.com/NVIDIA/gpu-operator/internal/consts" ) -func TestGetRuntimeString(t *testing.T) { - tests := []struct { - description string - runtimeVer string - expected string - expectError bool +func requireClusterInfo(t *testing.T, clusterInfoAPI Interface) *clusterInfo { + t.Helper() + + clusterInfoUnderTest, ok := clusterInfoAPI.(*clusterInfo) + require.True(t, ok, "New returned %T, not *clusterInfo", clusterInfoAPI) + return clusterInfoUnderTest +} + +func TestNew(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + options []Option + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedContainerRuntime string + expectedOpenshiftVersion string + expectedDriverToolkitImages map[string]string + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedRequestPaths []string }{ { - description: "docker runtime", - runtimeVer: "docker://20.10.7", - expected: consts.Docker, + description: "non-oneshot construction queries nothing", + handlers: openshiftClusterHandlers, + options: []Option{WithOneShot(false)}, + }, + { + description: "oneshot defaults to off", + handlers: openshiftClusterHandlers, + }, + { + description: "oneshot on a cluster that is not OpenShift", + handlers: mergeHandlers(discoveryHandlers(), map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + pathDriverToolkitImageStream: respondNotFound(), + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{})), + }), + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.Containerd, + expectedOpenshiftVersion: "", + expectedDriverToolkitImages: nil, + expectedDRASupported: false, + expectedRequestPaths: append([]string{pathClusterVersion, pathNodes, pathDriverToolkitImageStream}, + discoveryPaths()...), }, { - description: "containerd runtime", - runtimeVer: "containerd://1.6.8", - expected: consts.Containerd, + description: "oneshot on an OpenShift cluster never lists nodes", + handlers: openshiftClusterHandlers, + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.CRIO, + expectedOpenshiftVersion: "4.14", + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + expectedRequestPaths: append([]string{pathClusterVersion, pathDriverToolkitImageStream}, + discoveryPaths(openshiftClusterDRAGroups...)...), }, { - description: "cri-o runtime", - runtimeVer: "cri-o://1.24.1", - expected: consts.CRIO, + description: "oneshot fails when the container runtime cannot be determined", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + options: []Option{WithOneShot(true)}, + expectedErrorContains: "failed to get container runtime", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, }, { - description: "unrecognized runtime returns error", - runtimeVer: "unknown-runtime://1.0.0", - expectError: true, + description: "oneshot tolerates a DriverToolkit lookup failure", + handlers: mergeHandlers(openshiftClusterHandlers, map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondServerError(), + }), + options: []Option{WithOneShot(true)}, + expectedContainerRuntime: consts.CRIO, + expectedOpenshiftVersion: "4.14", + expectedDriverToolkitImages: nil, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + expectedRequestPaths: append([]string{pathClusterVersion, pathDriverToolkitImageStream}, + discoveryPaths(openshiftClusterDRAGroups...)...), }, { - description: "empty runtime version returns error", - runtimeVer: "", - expectError: true, + description: "oneshot fails when DRA support cannot be determined", + handlers: mergeHandlers(openshiftClusterHandlers, map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondNotFound(), + pathAPIGroups: respondServerError(), + }), + options: []Option{WithOneShot(true)}, + expectedErrorContains: "failed to determine DRA support", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{ + pathClusterVersion, pathDriverToolkitImageStream, pathAPIVersions, pathAPIGroups, + }, }, } - for _, tc := range tests { - t.Run(tc.description, func(t *testing.T) { - node := corev1.Node{ - Status: corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ - ContainerRuntimeVersion: tc.runtimeVer, - }, - }, - } + // Suspected bug #5 owns whether New reaches the Proxy endpoint. Cases that could reach it still + // register a Proxy handler, because this exclusion does not reach the unhandled-path tripwire. + pathsOwnedByCharacterizationTests := []string{pathProxy} + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) - result, err := getRuntimeString(node) + options := append([]Option{WithKubernetesConfig(apiServer.restConfig)}, testCase.options...) - if tc.expectError { - require.Error(t, err) + clusterInfoAPI, err := New(ctx, options...) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + } + require.Nil(t, clusterInfoAPI) + assertRequestedPathSetExcluding(t, apiServer, testCase.expectedRequestPaths, pathsOwnedByCharacterizationTests...) return } + require.NoError(t, err) - require.Equal(t, tc.expected, result) + require.NotNil(t, clusterInfoAPI) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + + assert.Equal(t, testCase.expectedContainerRuntime, clusterInfoUnderTest.containerRuntime) + assert.Equal(t, testCase.expectedOpenshiftVersion, clusterInfoUnderTest.openshiftVersion) + assert.Equal(t, testCase.expectedDriverToolkitImages, clusterInfoUnderTest.openshiftDriverToolkitImages) + assert.Equal(t, testCase.expectedDRAResourceGVR, clusterInfoUnderTest.draResourceGVR) + assert.Equal(t, testCase.expectedDRASupported, clusterInfoUnderTest.draSupported) + assertRequestedPathSetExcluding(t, apiServer, testCase.expectedRequestPaths, pathsOwnedByCharacterizationTests...) }) } + + // Only the abort is contractual: the wording and which endpoints are reached on the way are + // suspected bug #1, owned by clusterinfo_characterization_test.go. + t.Run("oneshot fails when the OpenShift version cannot be determined", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.PartialUpdate, "4.15.0"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Error(t, err) + require.Nil(t, clusterInfoAPI) + assertEveryRequestWasARead(t, apiServer) + }) + + t.Run("oneshot fails when the OpenShift version lookup is denied", func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(apiServer.restConfig), WithOneShot(true)) + + require.Error(t, err) + requireAPIStatusError(t, err, metav1.StatusReasonForbidden, http.StatusForbidden) + require.Nil(t, clusterInfoAPI) + assertEveryRequestWasARead(t, apiServer) + }) + + t.Run("options are applied in order and the last one wins", func(t *testing.T) { + ctx := testContext(t) + firstConfig := &rest.Config{Host: "https://first.example.com:6443"} + secondConfig := &rest.Config{Host: "https://second.example.com:6443"} + + clusterInfoAPI, err := New(ctx, + WithKubernetesConfig(firstConfig), + WithKubernetesConfig(secondConfig), + WithOneShot(true), + WithOneShot(false), + ) + + require.NoError(t, err) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + require.Equal(t, secondConfig, clusterInfoUnderTest.config) + require.False(t, clusterInfoUnderTest.oneshot) + }) + + t.Run("New stores the context it was given", func(t *testing.T) { + ctx := testContext(t) + + clusterInfoAPI, err := New(ctx, WithKubernetesConfig(&rest.Config{Host: "https://example.com:6443"})) + + require.NoError(t, err) + clusterInfoUnderTest := requireClusterInfo(t, clusterInfoAPI) + require.Equal(t, ctx, clusterInfoUnderTest.ctx) + }) +} + +// Nothing in an accessor's return values reveals which context it passed down, so each subtest +// below spends the stored context and asserts the consequence, making a context.Background() visible. +func TestClusterInfoAccessorsUseTheStoredContext(t *testing.T) { + // The stub registers no handler but the catch-all, whose tripwire fires during cleanup if a + // request arrives: a cancelled context must stop each accessor before it reaches the wire. + newClusterInfoWithCancelledContext := func(t *testing.T) *clusterInfo { + t.Helper() + + ctx, cancel := context.WithCancel(testContext(t)) + cancel() + apiServer := newStubAPIServer(t, map[string]http.HandlerFunc{}) + return &clusterInfo{ctx: ctx, config: apiServer.restConfig} + } + + t.Run("GetContainerRuntime fails on a cancelled context", func(t *testing.T) { + containerRuntime, err := newClusterInfoWithCancelledContext(t).GetContainerRuntime() + + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, containerRuntime) + }) + + t.Run("GetOpenshiftVersion fails on a cancelled context", func(t *testing.T) { + openshiftVersion, err := newClusterInfoWithCancelledContext(t).GetOpenshiftVersion() + + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, openshiftVersion) + }) + + t.Run("GetOpenshiftProxySpec fails on a cancelled context", func(t *testing.T) { + openshiftProxySpec, err := newClusterInfoWithCancelledContext(t).GetOpenshiftProxySpec() + + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, openshiftProxySpec) + }) + + t.Run("GetOpenshiftDriverToolkitImages yields no images on a cancelled context", func(t *testing.T) { + require.Nil(t, newClusterInfoWithCancelledContext(t).GetOpenshiftDriverToolkitImages()) + }) + + // Discovery builds its own requests with context.TODO, so cancelling the stored context cannot + // stop this accessor; the logger it carries is the only trace that the stored context was used. + t.Run("GetDRAResourceGVR logs through the stored context", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, discoveryHandlers(draAPIGroup("v1"))) + clusterInfoUnderTest := &clusterInfo{ctx: ctx, config: apiServer.restConfig} + + draResourceGVR, draSupported, err := clusterInfoUnderTest.GetDRAResourceGVR() + + require.NoError(t, err) + require.True(t, draSupported) + require.Equal(t, draGVR("v1"), draResourceGVR) + require.Equal(t, + []string{"Discovered DeviceClass resource"}, + recorder.capturedMessages(logRecordLevelInfo)) + }) +} + +func TestOptions(t *testing.T) { + t.Run("WithKubernetesConfig stores the config it was given", func(t *testing.T) { + config := &rest.Config{Host: "https://example.com:6443"} + clusterInfoUnderTest := &clusterInfo{} + + WithKubernetesConfig(config)(clusterInfoUnderTest) + + require.Equal(t, config, clusterInfoUnderTest.config) + }) + + // A nil config is what makes New fall back to config.GetConfigOrDie, which calls os.Exit(1). + // Every test in this package therefore passes WithKubernetesConfig explicitly. + t.Run("WithKubernetesConfig accepts nil", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{config: &rest.Config{Host: "https://example.com:6443"}} + + WithKubernetesConfig(nil)(clusterInfoUnderTest) + + require.Nil(t, clusterInfoUnderTest.config) + }) + + t.Run("WithOneShot sets the flag", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{} + + WithOneShot(true)(clusterInfoUnderTest) + + require.True(t, clusterInfoUnderTest.oneshot) + }) + + t.Run("WithOneShot clears the flag", func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{oneshot: true} + + WithOneShot(false)(clusterInfoUnderTest) + + require.False(t, clusterInfoUnderTest.oneshot) + }) } diff --git a/controllers/clusterinfo/dra_test.go b/controllers/clusterinfo/dra_test.go new file mode 100644 index 000000000..1cd606fd4 --- /dev/null +++ b/controllers/clusterinfo/dra_test.go @@ -0,0 +1,291 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" +) + +func TestGetDRAResourceGVR(t *testing.T) { + draGroupWithoutDeviceClass := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "resourceclaims", Kind: "ResourceClaim"}}, + } + unreachableMetricsGroup := discoveredAPIGroup{ + name: "metrics.k8s.io", + versionsInPreferenceOrder: []string{"v1beta1"}, + versionEndpointsFail: true, + } + // This group and draGroupNamingDeviceClassDifferently are the two halves of "the scan matches on + // Kind, never on the resource name". + deviceClassLookalikeGroup := discoveredAPIGroup{ + name: "policy.example.com", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "deviceclasses", Kind: "DeviceClassPolicy"}}, + } + draGroupNamingDeviceClassDifferently := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{{Name: "gpudeviceclasses", Kind: "DeviceClass"}}, + } + // Two DeviceClass-kind resources under different names are distinct GroupResources, so discovery + // keeps v1beta1's deviceclasses and v1's gpudeviceclasses side by side instead of collapsing them. + draGroupServingTwoDeviceClassNames := discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: []string{"v1beta1", "v1"}, + resourcesByVersion: map[string][]metav1.APIResource{ + "v1beta1": {deviceClassAPIResource()}, + "v1": {{Name: "gpudeviceclasses", Kind: "DeviceClass"}}, + }, + } + + testCases := []struct { + description string + groups []discoveredAPIGroup + handlerOverrides map[string]http.HandlerFunc + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedErrorContains string + expectedRequestPaths []string + }{ + { + description: "DeviceClass served at v1", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "DeviceClass served at v1beta2", + groups: []discoveredAPIGroup{draAPIGroup("v1beta2")}, + expectedDRAResourceGVR: draGVR("v1beta2"), + expectedDRASupported: true, + }, + { + description: "DeviceClass served at v1beta1", + groups: []discoveredAPIGroup{draAPIGroup("v1beta1")}, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "one deviceclasses at several versions collapses to the preferred v1", + groups: []discoveredAPIGroup{draAPIGroup("v1", "v1beta1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "one deviceclasses at several versions collapses to the preferred v1beta1", + groups: []discoveredAPIGroup{draAPIGroup("v1beta1", "v1")}, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "the ordered version preference outranks the group's preferred version", + groups: []discoveredAPIGroup{draGroupServingTwoDeviceClassNames}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a cluster without the resource.k8s.io group does not support DRA", + expectedDRASupported: false, + }, + { + description: "resource.k8s.io without a DeviceClass resource does not support DRA", + groups: []discoveredAPIGroup{draGroupWithoutDeviceClass}, + expectedDRASupported: false, + }, + { + description: "a resource named deviceclasses under another Kind does not support DRA", + groups: []discoveredAPIGroup{deviceClassLookalikeGroup}, + expectedDRASupported: false, + }, + { + // The Resource of the returned GVR is a constant, so it stays "deviceclasses" whatever + // name the group listed the Kind under. + description: "a DeviceClass listed under another resource name still supports DRA", + groups: []discoveredAPIGroup{draGroupNamingDeviceClassDifferently}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a DeviceClass in an unrelated group does not displace resource.k8s.io", + groups: []discoveredAPIGroup{draAPIGroup("v1"), unrelatedDeviceClassAPIGroup()}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "a DeviceClass served only at an unsupported version is an error", + groups: []discoveredAPIGroup{draAPIGroup("v1alpha3")}, + expectedErrorContains: "could not determine the GVR for the DeviceClass resource", + }, + { + description: "an unreachable API group does not mask resource.k8s.io", + groups: []discoveredAPIGroup{draAPIGroup("v1"), unreachableMetricsGroup}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + // A failure listing the groups themselves is not an ErrGroupDiscoveryFailed, so there + // are no partial results to keep and discovery stops before any group is fetched. + description: "a failure listing the API groups is fatal", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIGroups: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions, pathAPIGroups}, + }, + { + description: "a failure listing the legacy API versions is fatal", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIVersions: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, mergeHandlers(discoveryHandlers(testCase.groups...), testCase.handlerOverrides)) + + draResourceGVR, draSupported, err := getDRAResourceGVR(ctx, apiServer.restConfig) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedDRASupported, draSupported) + require.Equal(t, testCase.expectedDRAResourceGVR, draResourceGVR) + } + + expectedPaths := testCase.expectedRequestPaths + if expectedPaths == nil { + expectedPaths = discoveryPaths(testCase.groups...) + } + assertRequestedPathSet(t, apiServer, expectedPaths) + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause + // is pinned by unwrapping it and comparing against the error the same constructor produces. + t.Run("client construction failure", func(t *testing.T) { + _, expectedClientError := discovery.NewDiscoveryClientForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + draResourceGVR, draSupported, err := getDRAResourceGVR(testContext(t), brokenRESTConfig()) + + require.ErrorContains(t, err, "error building discovery client") + require.EqualError(t, errors.Unwrap(err), expectedClientError.Error()) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + }) +} + +func TestClusterInfoGetDRAResourceGVR(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedDRAResourceGVR schema.GroupVersionResource + cachedDRASupported bool + groups []discoveredAPIGroup + handlerOverrides map[string]http.HandlerFunc + expectedDRAResourceGVR schema.GroupVersionResource + expectedDRASupported bool + expectedErrorContains string + expectedRequestPaths []string + }{ + { + // A nil rest.Config is the assertion: any API call would fail loudly. + description: "oneshot returns the cached GVR without any API call", + oneshot: true, + cachedDRAResourceGVR: draGVR("v1beta1"), + cachedDRASupported: true, + expectedDRAResourceGVR: draGVR("v1beta1"), + expectedDRASupported: true, + }, + { + description: "oneshot reports a cluster cached as not supporting DRA", + oneshot: true, + expectedDRASupported: false, + }, + { + description: "non-oneshot queries the cluster", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "non-oneshot ignores the cached GVR", + cachedDRAResourceGVR: draGVR("v1beta1"), + cachedDRASupported: true, + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + expectedDRAResourceGVR: draGVR("v1"), + expectedDRASupported: true, + }, + { + description: "non-oneshot propagates errors", + groups: []discoveredAPIGroup{draAPIGroup("v1")}, + handlerOverrides: map[string]http.HandlerFunc{pathAPIGroups: respondServerError()}, + expectedErrorContains: "error getting server resources from discovery client", + expectedRequestPaths: []string{pathAPIVersions, pathAPIGroups}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + draResourceGVR: testCase.cachedDRAResourceGVR, + draSupported: testCase.cachedDRASupported, + } + var apiServer *stubAPIServer + if !testCase.oneshot { + apiServer = newStubAPIServer(t, mergeHandlers(discoveryHandlers(testCase.groups...), testCase.handlerOverrides)) + clusterInfoUnderTest.config = apiServer.restConfig + } + + draResourceGVR, draSupported, err := clusterInfoUnderTest.GetDRAResourceGVR() + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.False(t, draSupported) + require.Equal(t, schema.GroupVersionResource{}, draResourceGVR) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedDRASupported, draSupported) + require.Equal(t, testCase.expectedDRAResourceGVR, draResourceGVR) + } + + if apiServer != nil { + expectedPaths := testCase.expectedRequestPaths + if expectedPaths == nil { + expectedPaths = discoveryPaths(testCase.groups...) + } + assertRequestedPathSet(t, apiServer, expectedPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/logging_test.go b/controllers/clusterinfo/logging_test.go new file mode 100644 index 000000000..81929971a --- /dev/null +++ b/controllers/clusterinfo/logging_test.go @@ -0,0 +1,131 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "context" + "slices" + "sync" + "testing" + + "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +type logRecordLevel string + +const ( + logRecordLevelInfo logRecordLevel = "info" + logRecordLevelError logRecordLevel = "error" +) + +type logRecord struct { + level logRecordLevel + verbosity int + message string + err error + keysAndValues []any +} + +// logRecorder is held by pointer because logr copies a sink on every WithName and WithValues call, +// and every copy has to append to the same slice. +type logRecorder struct { + mu sync.Mutex + records []logRecord +} + +func (r *logRecorder) record(newRecord logRecord) { + r.mu.Lock() + defer r.mu.Unlock() + r.records = append(r.records, newRecord) +} + +func (r *logRecorder) capturedRecords() []logRecord { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.records) +} + +func (r *logRecorder) capturedRecordsAtLevel(level logRecordLevel) []logRecord { + var matchingRecords []logRecord + for _, capturedRecord := range r.capturedRecords() { + if capturedRecord.level == level { + matchingRecords = append(matchingRecords, capturedRecord) + } + } + return matchingRecords +} + +func (r *logRecorder) capturedMessages(level logRecordLevel) []string { + var messages []string + for _, capturedRecord := range r.capturedRecordsAtLevel(level) { + messages = append(messages, capturedRecord.message) + } + return messages +} + +// funcr.New is not used here because it routes Info and Error through one write function, which +// would leave the level recoverable only by parsing the formatted output. +type capturingLogSink struct { + funcr.Formatter + recorder *logRecorder +} + +func (s capturingLogSink) WithName(name string) logr.LogSink { + s.AddName(name) + return &s +} + +func (s capturingLogSink) WithValues(kvList ...any) logr.LogSink { + s.AddValues(kvList) + return &s +} + +func (s capturingLogSink) Info(verbosity int, message string, keysAndValues ...any) { + s.recorder.record(logRecord{ + level: logRecordLevelInfo, + verbosity: verbosity, + message: message, + keysAndValues: keysAndValues, + }) +} + +func (s capturingLogSink) Error(err error, message string, keysAndValues ...any) { + s.recorder.record(logRecord{ + level: logRecordLevelError, + message: message, + err: err, + keysAndValues: keysAndValues, + }) +} + +func capturingLogger() (logr.Logger, *logRecorder) { + recorder := &logRecorder{} + capturingSink := capturingLogSink{ + Formatter: funcr.NewFormatter(funcr.Options{Verbosity: consts.LogLevelDebug}), + recorder: recorder, + } + return logr.New(&capturingSink), recorder +} + +func testContextWithCapturedLogs(t *testing.T) (context.Context, *logRecorder) { + logger, recorder := capturingLogger() + return log.IntoContext(t.Context(), logger), recorder +} diff --git a/controllers/clusterinfo/openshift_dtk_test.go b/controllers/clusterinfo/openshift_dtk_test.go new file mode 100644 index 000000000..82a148c9a --- /dev/null +++ b/controllers/clusterinfo/openshift_dtk_test.go @@ -0,0 +1,249 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +const brokenImageStreamWarningMessage = "WARNING: ocpHasDriverToolkitImageStream: driver-toolkit imagestream is broken, see RHBZ#2015024" + +func TestGetOpenshiftDTKImages(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedDriverToolkitImages map[string]string + expectsBrokenImageStreamWarning bool + }{ + { + description: "every usable tag becomes a map entry", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + }, + { + description: "the floating latest tag is skipped", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest", "quay.io/openshift/driver-toolkit@sha256:zzz"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + }, + { + description: "a tag whose name only begins with latest is kept", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest-rhel9", "quay.io/openshift/driver-toolkit@sha256:ccc"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "latest-rhel9": "quay.io/openshift/driver-toolkit@sha256:ccc", + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + }, + }, + { + description: "a tag pointing at an ImageStreamTag is kept with that reference", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTagReferencingTag("410.84", "driver-toolkit:v4.10"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "driver-toolkit:v4.10", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + }, + { + description: "a tag without a From reference is skipped", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTagWithoutFrom("410.84"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{"412.86": "quay.io/openshift/driver-toolkit@sha256:bbb"}, + }, + { + description: "a tag with an empty name is skipped, see RHBZ#2015024", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("", "quay.io/openshift/driver-toolkit@sha256:zzz"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + expectsBrokenImageStreamWarning: true, + }, + { + description: "only a latest tag yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("latest", "quay.io/openshift/driver-toolkit@sha256:zzz"), + )), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "an ImageStream with no tags yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream()), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "a server error yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondServerError(), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "forbidden yields a nil map", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondForbidden(), + }, + expectedDriverToolkitImages: nil, + }, + { + description: "duplicate tag names collapse to the last one", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:bbb"}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + driverToolkitImages := getOpenshiftDTKImages(ctx, apiServer.restConfig) + + require.Equal(t, testCase.expectedDriverToolkitImages, driverToolkitImages) + + // A skipped tag leaves the returned map identical to one where the tag was never there, so + // the warning is the only report that the ImageStream is broken rather than merely thin. + capturedInfoMessages := recorder.capturedMessages(logRecordLevelInfo) + if testCase.expectsBrokenImageStreamWarning { + require.Contains(t, capturedInfoMessages, brokenImageStreamWarningMessage) + } else { + require.NotContains(t, capturedInfoMessages, brokenImageStreamWarningMessage) + } + + assertRequestedPathSequence(t, apiServer, []string{pathDriverToolkitImageStream}) + }) + } + + t.Run("client construction failure is logged and yields a nil map", func(t *testing.T) { + ctx, recorder := testContextWithCapturedLogs(t) + + require.Nil(t, getOpenshiftDTKImages(ctx, brokenRESTConfig())) + + // getOpenshiftDTKImages returns no error, so the log is the failure's only observable. + capturedErrorRecords := recorder.capturedRecordsAtLevel(logRecordLevelError) + require.Len(t, capturedErrorRecords, 1) + require.Equal(t, "failed to build openshift image stream client", capturedErrorRecords[0].message) + require.Error(t, capturedErrorRecords[0].err) + }) +} + +func TestClusterInfoGetOpenshiftDriverToolkitImages(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedDriverToolkitImages map[string]string + handlers map[string]http.HandlerFunc + expectedDriverToolkitImages map[string]string + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached map without any API call", + oneshot: true, + cachedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + expectedDriverToolkitImages: map[string]string{"410.84": "quay.io/openshift/driver-toolkit@sha256:aaa"}, + }, + { + description: "oneshot returns a nil cached map", + oneshot: true, + cachedDriverToolkitImages: nil, + expectedDriverToolkitImages: nil, + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + }, + expectedDriverToolkitImages: map[string]string{ + "410.84": "quay.io/openshift/driver-toolkit@sha256:aaa", + "412.86": "quay.io/openshift/driver-toolkit@sha256:bbb", + }, + expectedRequestPaths: []string{pathDriverToolkitImageStream}, + }, + { + description: "non-oneshot returns nil when the ImageStream is missing", + handlers: map[string]http.HandlerFunc{ + pathDriverToolkitImageStream: respondNotFound(), + }, + expectedDriverToolkitImages: nil, + expectedRequestPaths: []string{pathDriverToolkitImageStream}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + openshiftDriverToolkitImages: testCase.cachedDriverToolkitImages, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + driverToolkitImages := clusterInfoUnderTest.GetOpenshiftDriverToolkitImages() + + require.Equal(t, testCase.expectedDriverToolkitImages, driverToolkitImages) + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/openshift_proxy_test.go b/controllers/clusterinfo/openshift_proxy_test.go new file mode 100644 index 000000000..4ae1372a8 --- /dev/null +++ b/controllers/clusterinfo/openshift_proxy_test.go @@ -0,0 +1,196 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetOpenshiftProxySpec(t *testing.T) { + populatedOpenshiftProxySpec := configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + HTTPSProxy: "https://proxy.example.com:3129", + NoProxy: ".cluster.local,10.0.0.0/8", + TrustedCA: configv1.ConfigMapNameReference{Name: "user-ca-bundle"}, + } + + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedOpenshiftProxySpec *configv1.ProxySpec + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedErrorIsNotFound bool + }{ + { + description: "fully populated proxy", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(populatedOpenshiftProxySpec)), + }, + expectedOpenshiftProxySpec: &populatedOpenshiftProxySpec, + }, + { + // A zero spec behind a non-nil pointer does not trip the nil guard in internal/state/driver.go. + description: "a cluster with no proxy configured yields a zero spec, not nil", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{})), + }, + expectedOpenshiftProxySpec: &configv1.ProxySpec{}, + }, + { + // Unlike getOpenshiftVersion, this function does not convert NotFound into a sentinel. + description: "a missing Proxy resource is an error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondNotFound(), + }, + expectedErrorReason: metav1.StatusReasonNotFound, + expectedErrorCode: http.StatusNotFound, + expectedErrorIsNotFound: true, + }, + { + description: "server error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondServerError(), + }, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + }, + { + description: "forbidden", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondForbidden(), + }, + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + openshiftProxySpec, err := getOpenshiftProxySpec(ctx, apiServer.restConfig) + + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Equal(t, testCase.expectedErrorIsNotFound, apierrors.IsNotFound(err)) + require.Nil(t, openshiftProxySpec) + } else { + require.NoError(t, err) + require.NotNil(t, openshiftProxySpec) + require.Equal(t, *testCase.expectedOpenshiftProxySpec, *openshiftProxySpec) + } + + assertRequestedPathSequence(t, apiServer, []string{pathProxy}) + }) + } +} + +func TestClusterInfoGetOpenshiftProxySpec(t *testing.T) { + cachedOpenshiftProxySpec := &configv1.ProxySpec{HTTPProxy: "http://proxy.example.com:3128"} + fetchedOpenshiftProxySpec := configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + HTTPSProxy: "https://proxy.example.com:3129", + NoProxy: ".cluster.local", + TrustedCA: configv1.ConfigMapNameReference{Name: "user-ca-bundle"}, + } + + testCases := []struct { + description string + oneshot bool + cachedOpenshiftProxySpec *configv1.ProxySpec + handlers map[string]http.HandlerFunc + expectedOpenshiftProxySpec *configv1.ProxySpec + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedErrorIsNotFound bool + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached spec without any API call", + oneshot: true, + cachedOpenshiftProxySpec: cachedOpenshiftProxySpec, + expectedOpenshiftProxySpec: cachedOpenshiftProxySpec, + }, + { + description: "oneshot returns a nil cached spec without an error", + oneshot: true, + cachedOpenshiftProxySpec: nil, + expectedOpenshiftProxySpec: nil, + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(fetchedOpenshiftProxySpec)), + }, + expectedOpenshiftProxySpec: &fetchedOpenshiftProxySpec, + expectedRequestPaths: []string{pathProxy}, + }, + { + description: "non-oneshot propagates a missing Proxy resource as an error", + handlers: map[string]http.HandlerFunc{ + pathProxy: respondNotFound(), + }, + expectedErrorReason: metav1.StatusReasonNotFound, + expectedErrorCode: http.StatusNotFound, + expectedErrorIsNotFound: true, + expectedRequestPaths: []string{pathProxy}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + proxySpec: testCase.cachedOpenshiftProxySpec, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + openshiftProxySpec, err := clusterInfoUnderTest.GetOpenshiftProxySpec() + + switch { + case testCase.expectedErrorReason != "": + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Equal(t, testCase.expectedErrorIsNotFound, apierrors.IsNotFound(err)) + require.Nil(t, openshiftProxySpec) + case testCase.oneshot: + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftProxySpec, openshiftProxySpec) + default: + require.NoError(t, err) + require.NotNil(t, openshiftProxySpec) + require.Equal(t, *testCase.expectedOpenshiftProxySpec, *openshiftProxySpec) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/openshift_version_test.go b/controllers/clusterinfo/openshift_version_test.go new file mode 100644 index 000000000..47509b78e --- /dev/null +++ b/controllers/clusterinfo/openshift_version_test.go @@ -0,0 +1,274 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + ocpconfigv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestGetOpenshiftVersion(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedOpenshiftVersion string + expectsError bool + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + }{ + { + description: "completed history entry yields major.minor", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "two-component version is returned unchanged", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "single-component version yields the major only", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4"))), + }, + expectedOpenshiftVersion: "4", + }, + { + description: "pre-release version keeps only the first two segments", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10-rc.1"))), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "partial entries are skipped in favour of the completed one", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory(configv1.PartialUpdate, "4.15.0"), + updateHistory(configv1.CompletedUpdate, "4.14.10"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "an entry with an unset state is skipped in favour of the completed one", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory("", "4.15.0"), + updateHistory(configv1.CompletedUpdate, "4.14.10"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "the first completed entry wins, not the newest", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion( + updateHistory(configv1.CompletedUpdate, "4.14.10"), + updateHistory(configv1.CompletedUpdate, "4.12.1"), + )), + }, + expectedOpenshiftVersion: "4.14", + }, + { + description: "no completed entry in history", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.PartialUpdate, "4.15.0"))), + }, + expectsError: true, + expectedErrorContains: "failed to find Completed Cluster Version", + }, + { + description: "empty history", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, clusterVersion()), + }, + expectsError: true, + expectedErrorContains: "failed to find Completed Cluster Version", + }, + { + description: "no ClusterVersion resource means the cluster is not OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + }, + expectedOpenshiftVersion: "", + }, + { + description: "server error is reported, not treated as not-OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + }, + expectsError: true, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + }, + { + description: "forbidden is reported, not treated as not-OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondForbidden(), + }, + expectsError: true, + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + }, + { + description: "malformed response body", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithRawBody(http.StatusOK, "{"), + }, + expectsError: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + openshiftVersion, err := getOpenshiftVersion(ctx, apiServer.restConfig) + + if testCase.expectsError { + require.Error(t, err) + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + } + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + } + // NotFound is the one status this function must never surface: it is converted to + // the not-OpenShift sentinel, so seeing it here would mean the conversion was lost. + require.False(t, apierrors.IsNotFound(err)) + require.Empty(t, openshiftVersion) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftVersion, openshiftVersion) + } + + assertRequestedPathSequence(t, apiServer, []string{pathClusterVersion}) + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause is + // pinned by comparing against the error the same constructor produces for the same config. + t.Run("client construction failure returns the client's own error", func(t *testing.T) { + _, expectedClientError := ocpconfigv1.NewForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + openshiftVersion, err := getOpenshiftVersion(testContext(t), brokenRESTConfig()) + + require.EqualError(t, err, expectedClientError.Error()) + require.False(t, apierrors.IsNotFound(err)) + require.Empty(t, openshiftVersion) + }) +} + +func TestClusterInfoGetOpenshiftVersion(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedOpenshiftVersion string + handlers map[string]http.HandlerFunc + expectedOpenshiftVersion string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedRequestPaths []string + }{ + { + description: "oneshot returns the cached version without any API call", + oneshot: true, + cachedOpenshiftVersion: "4.14", + expectedOpenshiftVersion: "4.14", + }, + { + description: "oneshot returns an empty cached version", + oneshot: true, + cachedOpenshiftVersion: "", + expectedOpenshiftVersion: "", + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + }, + expectedOpenshiftVersion: "4.14", + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "non-oneshot reports a cluster that is not OpenShift as an empty version", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + }, + expectedOpenshiftVersion: "", + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "non-oneshot propagates errors", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondServerError(), + }, + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + openshiftVersion: testCase.cachedOpenshiftVersion, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + openshiftVersion, err := clusterInfoUnderTest.GetOpenshiftVersion() + + if testCase.expectedErrorReason != "" { + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Empty(t, openshiftVersion) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedOpenshiftVersion, openshiftVersion) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/runtime_test.go b/controllers/clusterinfo/runtime_test.go new file mode 100644 index 000000000..460418fe1 --- /dev/null +++ b/controllers/clusterinfo/runtime_test.go @@ -0,0 +1,351 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "net/http" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +func TestGetRuntimeString(t *testing.T) { + testCases := []struct { + description string + containerRuntimeVersion string + expectedContainerRuntime string + expectedErrorContains string + }{ + { + description: "docker runtime", + containerRuntimeVersion: "docker://20.10.23", + expectedContainerRuntime: consts.Docker, + }, + { + description: "containerd runtime", + containerRuntimeVersion: "containerd://1.7.11", + expectedContainerRuntime: consts.Containerd, + }, + { + description: "cri-o runtime maps to the crio constant", + containerRuntimeVersion: "cri-o://1.28.2", + expectedContainerRuntime: consts.CRIO, + }, + { + description: "unrecognised runtime", + containerRuntimeVersion: "rkt://1.0.0", + expectedErrorContains: "runtime not recognized: rkt://1.0.0", + }, + { + description: "node has not reported a runtime yet", + containerRuntimeVersion: "", + expectedErrorContains: "runtime not recognized: ", + }, + { + description: "bare prefix without a version suffix", + containerRuntimeVersion: "containerd", + expectedContainerRuntime: consts.Containerd, + }, + { + description: "runtime name appearing as a substring but not a prefix", + containerRuntimeVersion: "my-docker://1.0", + expectedErrorContains: "runtime not recognized: my-docker://1.0", + }, + { + description: "matching is case sensitive", + containerRuntimeVersion: "Docker://20.10", + expectedErrorContains: "runtime not recognized: Docker://20.10", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + node := corev1.Node{ + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: testCase.containerRuntimeVersion}, + }, + } + + containerRuntime, err := getRuntimeString(node) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.Empty(t, containerRuntime) + return + } + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + }) + } +} + +func TestGetContainerRuntime(t *testing.T) { + testCases := []struct { + description string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorContains string + expectedErrorReason metav1.StatusReason + expectedErrorCode int32 + expectedRequestPaths []string + }{ + { + description: "OpenShift short-circuits to crio without listing nodes", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion}, + }, + { + description: "single containerd node", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "single docker node", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "single cri-o node on a cluster that is not OpenShift", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("cri-o://1.28.2")), + }, + expectedContainerRuntime: consts.CRIO, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "containerd wins when it appears last", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "containerd wins when it appears first", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("containerd://1.7.11", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "no GPU nodes defaults to containerd", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList()), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "every node reports an unrecognised runtime, defaulting to containerd", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("rkt://1.0", "kata://3.2.0")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "an unrecognised node does not abort the scan", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("rkt://1.0", "docker://20.10.23")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + // Only this ordering catches a missing skip; in the case above the later node overwrites it anyway. + description: "an unrecognised node after a recognised one does not overwrite the result", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, + gpuNodeList("docker://20.10.23", "rkt://1.0")), + }, + expectedContainerRuntime: consts.Docker, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "node listing failure is fatal", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedErrorReason: metav1.StatusReasonInternalError, + expectedErrorCode: http.StatusInternalServerError, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "node listing denied by RBAC is fatal", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondForbidden(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedErrorReason: metav1.StatusReasonForbidden, + expectedErrorCode: http.StatusForbidden, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + ctx := testContext(t) + apiServer := newStubAPIServer(t, testCase.handlers) + + containerRuntime, err := getContainerRuntime(ctx, apiServer.restConfig) + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + requireAPIStatusError(t, err, testCase.expectedErrorReason, testCase.expectedErrorCode) + require.Empty(t, containerRuntime) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + } + + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + for _, request := range apiServer.requestsTo(pathNodes) { + assert.Equal(t, consts.GPUPresentLabel+"=true", request.query.Get("labelSelector")) + } + }) + } + + // client-go's TLS failure is a bare fmt.Errorf with no sentinel or type to match, so the cause is + // pinned by comparing against the error the same constructor produces for the same config. + t.Run("client construction failure returns the client's own error", func(t *testing.T) { + _, expectedClientError := corev1client.NewForConfig(brokenRESTConfig()) + require.Error(t, expectedClientError) + + containerRuntime, err := getContainerRuntime(testContext(t), brokenRESTConfig()) + + require.EqualError(t, err, expectedClientError.Error()) + require.Empty(t, containerRuntime) + }) +} + +func TestClusterInfoGetContainerRuntime(t *testing.T) { + testCases := []struct { + description string + oneshot bool + cachedContainerRuntime string + handlers map[string]http.HandlerFunc + expectedContainerRuntime string + expectedErrorContains string + expectedRequestPaths []string + }{ + { + // A nil rest.Config is the assertion: any API call would fail loudly. + description: "oneshot returns the cached runtime without any API call", + oneshot: true, + cachedContainerRuntime: consts.Containerd, + expectedContainerRuntime: consts.Containerd, + }, + { + description: "oneshot returns an empty cached runtime rather than re-fetching", + oneshot: true, + cachedContainerRuntime: "", + expectedContainerRuntime: "", + }, + { + description: "non-oneshot queries the cluster", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "non-oneshot propagates errors", + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondServerError(), + }, + expectedErrorContains: "unable to list nodes prior to checking container runtime", + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + { + description: "non-oneshot ignores the cached runtime", + cachedContainerRuntime: consts.Docker, + handlers: map[string]http.HandlerFunc{ + pathClusterVersion: respondNotFound(), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("containerd://1.7.11")), + }, + expectedContainerRuntime: consts.Containerd, + expectedRequestPaths: []string{pathClusterVersion, pathNodes}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.description, func(t *testing.T) { + clusterInfoUnderTest := &clusterInfo{ + ctx: testContext(t), + oneshot: testCase.oneshot, + containerRuntime: testCase.cachedContainerRuntime, + } + var apiServer *stubAPIServer + if testCase.handlers != nil { + apiServer = newStubAPIServer(t, testCase.handlers) + clusterInfoUnderTest.config = apiServer.restConfig + } + + containerRuntime, err := clusterInfoUnderTest.GetContainerRuntime() + + if testCase.expectedErrorContains != "" { + require.ErrorContains(t, err, testCase.expectedErrorContains) + require.Empty(t, containerRuntime) + } else { + require.NoError(t, err) + require.Equal(t, testCase.expectedContainerRuntime, containerRuntime) + } + + if apiServer != nil { + assertRequestedPathSequence(t, apiServer, testCase.expectedRequestPaths) + } + }) + } +} diff --git a/controllers/clusterinfo/stubs_test.go b/controllers/clusterinfo/stubs_test.go new file mode 100644 index 000000000..a236a3098 --- /dev/null +++ b/controllers/clusterinfo/stubs_test.go @@ -0,0 +1,488 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package clusterinfo + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "sync" + "testing" + + "github.com/go-logr/logr" + configv1 "github.com/openshift/api/config/v1" + imagev1 "github.com/openshift/api/image/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +const ( + pathNodes = "/api/v1/nodes" + pathClusterVersion = "/apis/config.openshift.io/v1/clusterversions/version" + pathProxy = "/apis/config.openshift.io/v1/proxies/cluster" + pathDriverToolkitImageStream = "/apis/image.openshift.io/v1/namespaces/openshift/imagestreams/driver-toolkit" + + // The core group's resource list is reached through the legacy /api prefix, not through /apis. + pathAPIVersions = "/api" + pathAPIGroups = "/apis" + pathCoreResources = "/api/v1" +) + +func pathGroupVersionResources(groupVersion string) string { + return "/apis/" + groupVersion +} + +type recordedRequest struct { + method string + path string + query url.Values +} + +// stubAPIServer keeps "a handler answered 404" distinct from "no handler was registered for this +// URL", so a wrong-path assumption cannot pass as a legitimate NotFound. +type stubAPIServer struct { + restConfig *rest.Config + + mu sync.Mutex + requests []recordedRequest + unhandledRequestPaths []string +} + +// An unregistered path fails the test during cleanup rather than at request time, so the case's +// own assertions report first. +func newStubAPIServer(t *testing.T, handlers map[string]http.HandlerFunc) *stubAPIServer { + t.Helper() + + apiServer := &stubAPIServer{} + mux := http.NewServeMux() + for path, handler := range handlers { + mux.HandleFunc(path, apiServer.recordingHandler(handler)) + } + mux.HandleFunc("/", apiServer.recordingHandler(func(w http.ResponseWriter, r *http.Request) { + apiServer.noteUnhandled(r.URL.Path) + respondNotFound()(w, r) + })) + + server := httptest.NewServer(mux) + apiServer.restConfig = &rest.Config{ + Host: server.URL, + // Pinning JSON makes rest.Request.UseProtobufAsDefault a no-op, so the core/v1 node + // client — which prefers protobuf — also talks JSON to this stub. + ContentConfig: rest.ContentConfig{ContentType: "application/json"}, + } + + t.Cleanup(func() { + require.Empty(t, apiServer.unhandledPaths(), "client requested paths that no stub handler was registered for") + }) + t.Cleanup(server.Close) + + return apiServer +} + +func (s *stubAPIServer) recordingHandler(handler http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests = append(s.requests, recordedRequest{method: r.Method, path: r.URL.Path, query: r.URL.Query()}) + s.mu.Unlock() + handler(w, r) + } +} + +func (s *stubAPIServer) noteUnhandled(path string) { + s.mu.Lock() + defer s.mu.Unlock() + s.unhandledRequestPaths = append(s.unhandledRequestPaths, path) +} + +func (s *stubAPIServer) unhandledPaths() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.unhandledRequestPaths) +} + +// recordedRequests is the only place the read side takes the mutex; the accessors below derive +// from its clone, so none of them can hand out a slice the recording handler still appends to. +func (s *stubAPIServer) recordedRequests() []recordedRequest { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.requests) +} + +func (s *stubAPIServer) requestedPaths() []string { + var paths []string + for _, request := range s.recordedRequests() { + paths = append(paths, request.path) + } + return paths +} + +func (s *stubAPIServer) requestsTo(path string) []recordedRequest { + var matchingRequests []recordedRequest + for _, request := range s.recordedRequests() { + if request.path == path { + matchingRequests = append(matchingRequests, request) + } + } + return matchingRequests +} + +func (s *stubAPIServer) distinctRequestedPaths() []string { + return slices.Compact(slices.Sorted(slices.Values(s.requestedPaths()))) +} + +func assertRequestedPathSequence(t *testing.T, apiServer *stubAPIServer, expectedPaths []string) { + t.Helper() + + assert.Equal(t, expectedPaths, apiServer.requestedPaths()) + assertEveryRequestWasARead(t, apiServer) +} + +// assertRequestedPathSet, like the sequence form, asserts every request was a read, but discovery +// fans out in parallel and repeats the walk on failure, so its order and repeat count are not pinned. +func assertRequestedPathSet(t *testing.T, apiServer *stubAPIServer, expectedPaths []string) { + t.Helper() + + assert.ElementsMatch(t, expectedPaths, apiServer.distinctRequestedPaths()) + assertEveryRequestWasARead(t, apiServer) +} + +func assertRequestedPathSetExcluding(t *testing.T, apiServer *stubAPIServer, expectedPaths []string, excludedPaths ...string) { + t.Helper() + + comparedPaths := slices.DeleteFunc(apiServer.distinctRequestedPaths(), func(path string) bool { + return slices.Contains(excludedPaths, path) + }) + assert.ElementsMatch(t, expectedPaths, comparedPaths) + assertEveryRequestWasARead(t, apiServer) +} + +func assertEveryRequestWasARead(t *testing.T, apiServer *stubAPIServer) { + t.Helper() + + for _, request := range apiServer.recordedRequests() { + assert.Equal(t, http.MethodGet, request.method, "unexpected %s to %s", request.method, request.path) + } +} + +// The Content-Type header is what client-go uses to choose a decoder; without it Go's +// DetectContentType labels a JSON body text/plain and decoding fails with "no serializer". +func respondWithJSON(statusCode int, responseObject any) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + body, err := json.Marshal(responseObject) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write(body) + } +} + +func respondWithRawBody(statusCode int, body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + } +} + +func respondWithFailureStatus(statusCode int32, reason metav1.StatusReason, message string) http.HandlerFunc { + return respondWithJSON(int(statusCode), &metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, + Message: message, + Reason: reason, + Code: statusCode, + }) +} + +func respondNotFound() http.HandlerFunc { + return respondWithFailureStatus(http.StatusNotFound, metav1.StatusReasonNotFound, + "the server could not find the requested resource") +} + +// respondServerError deliberately omits Retry-After. client-go only retries a 5xx that carries it, +// so the request fails immediately instead of sleeping through ten retries. +func respondServerError() http.HandlerFunc { + return respondWithFailureStatus(http.StatusInternalServerError, metav1.StatusReasonInternalError, + "the server encountered an internal error") +} + +func respondForbidden() http.HandlerFunc { + return respondWithFailureStatus(http.StatusForbidden, metav1.StatusReasonForbidden, + "forbidden: user cannot get the requested resource") +} + +// brokenRESTConfig makes every *client.NewForConfig call fail: transport.TLSConfigFor rejects a +// config that supplies CA data alongside Insecure. +func brokenRESTConfig() *rest.Config { + return &rest.Config{ + Host: "https://127.0.0.1:1", + TLSClientConfig: rest.TLSClientConfig{Insecure: true, CAData: []byte("not-a-cert")}, + } +} + +func testContext(t *testing.T) context.Context { + return log.IntoContext(t.Context(), logr.Discard()) +} + +// requireAPIStatusError discriminates on the typed error the client built rather than on the stubs' +// own message text, so it also fails if production code drops the %w that carries the cause. +func requireAPIStatusError(t *testing.T, err error, expectedReason metav1.StatusReason, expectedCode int32) { + t.Helper() + + var statusError *apierrors.StatusError + require.ErrorAs(t, err, &statusError) + require.Equal(t, expectedReason, statusError.Status().Reason) + require.Equal(t, expectedCode, statusError.Status().Code) +} + +func clusterVersion(history ...configv1.UpdateHistory) *configv1.ClusterVersion { + return &configv1.ClusterVersion{ + TypeMeta: metav1.TypeMeta{APIVersion: configv1.GroupVersion.String(), Kind: "ClusterVersion"}, + ObjectMeta: metav1.ObjectMeta{Name: "version"}, + Status: configv1.ClusterVersionStatus{History: history}, + } +} + +func updateHistory(state configv1.UpdateState, version string) configv1.UpdateHistory { + return configv1.UpdateHistory{State: state, Version: version} +} + +func clusterProxy(spec configv1.ProxySpec) *configv1.Proxy { + return &configv1.Proxy{ + TypeMeta: metav1.TypeMeta{APIVersion: configv1.GroupVersion.String(), Kind: "Proxy"}, + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Spec: spec, + } +} + +func driverToolkitImageStream(tags ...imagev1.TagReference) *imagev1.ImageStream { + return &imagev1.ImageStream{ + TypeMeta: metav1.TypeMeta{APIVersion: imagev1.GroupVersion.String(), Kind: "ImageStream"}, + ObjectMeta: metav1.ObjectMeta{Name: "driver-toolkit", Namespace: consts.OpenshiftNamespace}, + Spec: imagev1.ImageStreamSpec{Tags: tags}, + } +} + +func imageStreamTag(name, image string) imagev1.TagReference { + return imagev1.TagReference{ + Name: name, + From: &corev1.ObjectReference{Kind: "DockerImage", Name: image}, + } +} + +// Real ImageStreams carry both the ImageStreamTag and the DockerImage reference forms. +func imageStreamTagReferencingTag(name, referencedTag string) imagev1.TagReference { + return imagev1.TagReference{ + Name: name, + From: &corev1.ObjectReference{Kind: "ImageStreamTag", Name: referencedTag}, + } +} + +func imageStreamTagWithoutFrom(name string) imagev1.TagReference { + return imagev1.TagReference{Name: name} +} + +func gpuNodeList(containerRuntimeVersions ...string) *corev1.NodeList { + nodes := make([]corev1.Node, 0, len(containerRuntimeVersions)) + for index, containerRuntimeVersion := range containerRuntimeVersions { + nodes = append(nodes, corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("gpu-node-%d", index), + Labels: map[string]string{consts.GPUPresentLabel: "true"}, + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: containerRuntimeVersion}, + }, + }) + } + return &corev1.NodeList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "NodeList"}, + Items: nodes, + } +} + +// The first version listed is the group's preferred version, which is the one discovery keeps for +// a resource served by several. +type discoveredAPIGroup struct { + name string + versionsInPreferenceOrder []string + resources []metav1.APIResource + // Discovery deduplicates on group plus resource name, so only per-version resource sets let two + // versions of a resource both survive. + resourcesByVersion map[string][]metav1.APIResource + versionEndpointsFail bool +} + +func (group discoveredAPIGroup) resourcesServedAt(version string) []metav1.APIResource { + if resourcesForVersion, found := group.resourcesByVersion[version]; found { + return resourcesForVersion + } + return group.resources +} + +func draGVR(version string) schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "resource.k8s.io", Version: version, Resource: "deviceclasses"} +} + +// draAPIGroup lists the group's other resources ahead of deviceclasses, so the scan has to walk +// past non-matching entries rather than finding its match at index 0. +func draAPIGroup(versionsInPreferenceOrder ...string) discoveredAPIGroup { + return discoveredAPIGroup{ + name: "resource.k8s.io", + versionsInPreferenceOrder: versionsInPreferenceOrder, + resources: []metav1.APIResource{ + {Name: "resourceclaims", Kind: "ResourceClaim"}, + {Name: "resourceclaimtemplates", Kind: "ResourceClaimTemplate"}, + {Name: "resourceslices", Kind: "ResourceSlice"}, + deviceClassAPIResource(), + }, + } +} + +// A group that is advertised but whose version endpoints all fail is what drives +// ServerPreferredResources into its partial-failure path. +func unreachableDRAAPIGroup(versionsInPreferenceOrder ...string) discoveredAPIGroup { + draGroup := draAPIGroup(versionsInPreferenceOrder...) + draGroup.versionEndpointsFail = true + return draGroup +} + +func unrelatedDeviceClassAPIGroup() discoveredAPIGroup { + return discoveredAPIGroup{ + name: "devices.example.com", + versionsInPreferenceOrder: []string{"v1"}, + resources: []metav1.APIResource{deviceClassAPIResource()}, + } +} + +func deviceClassAPIResource() metav1.APIResource { + return metav1.APIResource{Name: "deviceclasses", Kind: "DeviceClass"} +} + +// ServerPreferredResources fetches every version of every group, not only each group's preferred +// version, so each listed version needs a handler. +func discoveryHandlers(groups ...discoveredAPIGroup) map[string]http.HandlerFunc { + handlers := map[string]http.HandlerFunc{ + pathAPIVersions: respondWithJSON(http.StatusOK, apiVersions("v1")), + pathAPIGroups: respondWithJSON(http.StatusOK, apiGroupList(groups...)), + pathCoreResources: respondWithJSON(http.StatusOK, + apiResourceList("v1", metav1.APIResource{Name: "nodes", Kind: "Node"})), + } + for _, group := range groups { + for _, version := range group.versionsInPreferenceOrder { + groupVersion := group.name + "/" + version + if group.versionEndpointsFail { + handlers[pathGroupVersionResources(groupVersion)] = respondServerError() + continue + } + handlers[pathGroupVersionResources(groupVersion)] = respondWithJSON(http.StatusOK, + apiResourceList(groupVersion, group.resourcesServedAt(version)...)) + } + } + return handlers +} + +// Must stay in sync with discoveryHandlers. +func discoveryPaths(groups ...discoveredAPIGroup) []string { + paths := []string{pathAPIVersions, pathAPIGroups, pathCoreResources} + for _, group := range groups { + for _, version := range group.versionsInPreferenceOrder { + paths = append(paths, pathGroupVersionResources(group.name+"/"+version)) + } + } + return paths +} + +func mergeHandlers(handlerMaps ...map[string]http.HandlerFunc) map[string]http.HandlerFunc { + mergedHandlers := map[string]http.HandlerFunc{} + for _, handlerMap := range handlerMaps { + maps.Copy(mergedHandlers, handlerMap) + } + return mergedHandlers +} + +var openshiftClusterDRAGroups = []discoveredAPIGroup{draAPIGroup("v1")} + +var openshiftClusterHandlers = mergeHandlers(discoveryHandlers(openshiftClusterDRAGroups...), + map[string]http.HandlerFunc{ + pathClusterVersion: respondWithJSON(http.StatusOK, + clusterVersion(updateHistory(configv1.CompletedUpdate, "4.14.10"))), + pathNodes: respondWithJSON(http.StatusOK, gpuNodeList("cri-o://1.28.2")), + pathDriverToolkitImageStream: respondWithJSON(http.StatusOK, driverToolkitImageStream( + imageStreamTag("410.84", "quay.io/openshift/driver-toolkit@sha256:aaa"), + imageStreamTag("412.86", "quay.io/openshift/driver-toolkit@sha256:bbb"), + )), + pathProxy: respondWithJSON(http.StatusOK, clusterProxy(configv1.ProxySpec{ + HTTPProxy: "http://proxy.example.com:3128", + })), + }) + +func apiVersions(versions ...string) *metav1.APIVersions { + return &metav1.APIVersions{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIVersions"}, + Versions: versions, + } +} + +func apiGroupList(groups ...discoveredAPIGroup) *metav1.APIGroupList { + groupList := &metav1.APIGroupList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIGroupList"}, + } + for _, group := range groups { + groupList.Groups = append(groupList.Groups, apiGroup(group)) + } + return groupList +} + +func apiGroup(group discoveredAPIGroup) metav1.APIGroup { + advertisedGroup := metav1.APIGroup{Name: group.name} + for _, version := range group.versionsInPreferenceOrder { + advertisedGroup.Versions = append(advertisedGroup.Versions, metav1.GroupVersionForDiscovery{ + GroupVersion: group.name + "/" + version, + Version: version, + }) + } + if len(advertisedGroup.Versions) == 0 { + panic(fmt.Sprintf("API group fixture %q lists no versions, so it has no preferred version", group.name)) + } + advertisedGroup.PreferredVersion = advertisedGroup.Versions[0] + return advertisedGroup +} + +func apiResourceList(groupVersion string, resources ...metav1.APIResource) *metav1.APIResourceList { + return &metav1.APIResourceList{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "APIResourceList"}, + GroupVersion: groupVersion, + APIResources: resources, + } +}