diff --git a/.tekton/integration-tests/integration-test-scenarios.yaml b/.tekton/integration-tests/integration-test-scenarios.yaml new file mode 100644 index 0000000..92ad55c --- /dev/null +++ b/.tekton/integration-tests/integration-test-scenarios.yaml @@ -0,0 +1,36 @@ +# IntegrationTestScenario for lightspeed-agentic-operator e2e. +# Triggers on lightspeed-agentic-operator component builds. +# Provisions an ephemeral Hypershift cluster, deploys operator from SNAPSHOT, +# runs make test-e2e with the fixed mock agent image. +# +# Apply to tenant namespace: kubectl apply -f .tekton/integration-tests/integration-test-scenarios.yaml +--- +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: agentic-operator-e2e-tests + namespace: crt-nshift-lightspeed-tenant + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: ols + contexts: + - description: Agentic operator e2e (bare-pod mode) + name: component_lightspeed-agentic-operator + params: + - name: test-name + value: agentic-e2e-tests + - name: namespace + value: openshift-lightspeed + - name: openshift-version-prefix + value: "4.19." + resolverRef: + resolver: git + resourceKind: pipeline + params: + - name: url + value: https://github.com/openshift/lightspeed-agentic-operator + - name: revision + value: main + - name: pathInRepo + value: .tekton/integration-tests/pipelines/agentic-operator-e2e-pipeline.yaml diff --git a/.tekton/integration-tests/pipelines/agentic-operator-e2e-pipeline.yaml b/.tekton/integration-tests/pipelines/agentic-operator-e2e-pipeline.yaml new file mode 100644 index 0000000..6635670 --- /dev/null +++ b/.tekton/integration-tests/pipelines/agentic-operator-e2e-pipeline.yaml @@ -0,0 +1,283 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: agentic-operator-e2e-pipeline +spec: + description: | + Runs lightspeed-agentic-operator e2e tests on a Konflux ephemeral OpenShift cluster (EaaS). + Provisions the cluster, deploys the operator from the SNAPSHOT image with a fixed mock agent, + runs make test-e2e, then deprovisions + params: + - name: SNAPSHOT + description: 'The JSON string representing the snapshot of the application under test.' + default: '{"components": [{"name":"test-app", "containerImage": "quay.io/example/repo:latest"}]}' + type: string + - name: test-name + description: 'The name of the test corresponding to a defined Konflux integration test.' + default: 'agentic-e2e-tests' + type: string + - name: namespace + description: 'Namespace to deploy the operator into' + default: 'openshift-lightspeed' + type: string + - name: openshift-version-prefix + description: 'Minor line prefix for eaas-get-latest-openshift-version-by-prefix (include trailing dot, e.g. 4.19.)' + default: '4.19.' + type: string + tasks: + - name: eaas-provision-space + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/eaas-provision-space/0.1/eaas-provision-space.yaml + params: + - name: ownerKind + value: PipelineRun + - name: ownerName + value: $(context.pipelineRun.name) + - name: ownerUid + value: $(context.pipelineRun.uid) + - name: provision-cluster + runAfter: + - eaas-provision-space + params: + - name: openshift-version-prefix + value: $(params.openshift-version-prefix) + taskSpec: + params: + - name: openshift-version-prefix + type: string + results: + - name: clusterName + value: "$(steps.create-cluster.results.clusterName)" + steps: + - name: pick-version + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-latest-openshift-version-by-prefix/0.1/eaas-get-latest-openshift-version-by-prefix.yaml + params: + - name: prefix + value: "$(params.openshift-version-prefix)" + - name: create-cluster + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.eaas-provision-space.results.secretRef) + - name: version + value: "$(steps.pick-version.results.version)" + - name: instanceType + value: "m5.large" + - name: install-and-test + description: Install operator from SNAPSHOT, run e2e tests. + runAfter: + - provision-cluster + taskSpec: + params: + - name: SNAPSHOT + type: string + - name: namespace + type: string + results: + - name: commit + value: "$(steps.install-operator.results.commit)" + volumes: + - name: credentials + emptyDir: {} + - name: ols-konflux-artifacts-bot-creds + secret: + secretName: ols-konflux-artifacts-bot + steps: + - name: get-kubeconfig + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + params: + - name: eaasSpaceSecretRef + value: $(tasks.eaas-provision-space.results.secretRef) + - name: clusterName + value: "$(tasks.provision-cluster.results.clusterName)" + - name: credentials + value: credentials + - name: install-operator + results: + - name: commit + type: string + volumeMounts: + - name: credentials + mountPath: /credentials + env: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + - name: KONFLUX_COMPONENT_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['appstudio.openshift.io/component'] + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: NAMESPACE + value: "$(params.namespace)" + - name: SANDBOX_IMAGE + value: "quay.io/openshift-lightspeed/ols-qe:lightspeed-mock-agent" + image: registry.redhat.io/openshift4/ose-cli:latest + script: | + set -euo pipefail + dnf -y install jq git make golang + OPERATOR_IMAGE="$(jq -r --arg component_name "${KONFLUX_COMPONENT_NAME}" '.components[] | select(.name == $component_name) | .containerImage' <<< "$SNAPSHOT")" + if [ -z "$OPERATOR_IMAGE" ] || [ "$OPERATOR_IMAGE" = "null" ]; then + echo "ERROR: could not extract operator image from SNAPSHOT" + exit 1 + fi + + COMMIT="$(jq -r --arg component_name "${KONFLUX_COMPONENT_NAME}" '.components[] | select(.name == $component_name) | .source.git.revision' <<< "$SNAPSHOT")" + if [ -z "$COMMIT" ] || [ "$COMMIT" = "null" ]; then + echo "ERROR: could not extract git revision from SNAPSHOT" + exit 1 + fi + echo -n "${COMMIT}" > $(step.results.commit.path) + + git clone --depth=1 https://github.com/openshift/lightspeed-agentic-operator.git /workspace/repo + cd /workspace/repo + git fetch --depth=1 origin "${COMMIT}" + git checkout "${COMMIT}" + + IMG="${OPERATOR_IMAGE}" \ + OPERATOR_NAMESPACE="${NAMESPACE}" \ + SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ + bash .tekton/integration-tests/scripts/install-operator.sh + - name: run-e2e-tests + resources: + requests: + memory: "8Gi" + limits: + memory: "8Gi" + onError: continue + volumeMounts: + - name: credentials + mountPath: /credentials + env: + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: TEST_NAMESPACE + value: "$(params.namespace)" + - name: OPERATOR_NAMESPACE + value: "$(params.namespace)" + image: registry.redhat.io/openshift4/ose-cli:latest + script: | + set -euo pipefail + dnf -y install git make golang + cd /workspace/repo + mkdir -p /workspace/konflux-artifacts + make test-e2e 2>&1 | tee /workspace/konflux-artifacts/e2e-test-output.log + - name: gather-cluster-resources + onError: continue + volumeMounts: + - name: credentials + mountPath: /credentials + env: + - name: KUBECONFIG + value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: ARTIFACT_DIR + value: "/workspace/konflux-artifacts" + - name: NAMESPACE + value: "$(params.namespace)" + image: quay.io/konflux-qe-incubator/konflux-qe-tools:latest + script: | + #!/bin/bash + set -x + cd "${ARTIFACT_DIR}" + oc get proposals -A -o yaml > proposals.yaml 2>/dev/null || true + oc get proposalapprovals -A -o yaml > proposalapprovals.yaml 2>/dev/null || true + oc get analysisresults -A -o yaml > analysisresults.yaml 2>/dev/null || true + oc get executionresults -A -o yaml > executionresults.yaml 2>/dev/null || true + oc get verificationresults -A -o yaml > verificationresults.yaml 2>/dev/null || true + oc get pods -n "${NAMESPACE}" -o yaml > pods.yaml 2>/dev/null || true + oc logs -n "${NAMESPACE}" deployment/controller-manager --tail=500 > operator-logs.txt 2>/dev/null || true + echo "Artifacts gathered:" + ls -la "${ARTIFACT_DIR}" + - name: list-artifacts + onError: continue + image: quay.io/konflux-qe-incubator/konflux-qe-tools:latest + workingDir: /workspace/konflux-artifacts + script: | + #!/bin/bash + echo "=== Artifacts to push ===" + ls -la /workspace/konflux-artifacts + - name: push-artifacts + onError: continue + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/tekton-integration-catalog.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/secure-push-oci/0.1/secure-push-oci.yaml + params: + - name: workdir-path + value: /workspace/konflux-artifacts + - name: oci-ref + value: "quay.io/openshift-lightspeed/ols-operator-artifacts:agentic-$(steps.install-operator.results.commit)" + - name: credentials-volume-name + value: ols-konflux-artifacts-bot-creds + - name: fail-if-any-step-failed + ref: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/tekton-integration-catalog.git + - name: revision + value: main + - name: pathInRepo + value: stepactions/fail-if-any-step-failed/0.1/fail-if-any-step-failed.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + - name: namespace + value: "$(params.namespace)" + finally: + - name: export-logs-for-retention + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/tekton-integration-catalog.git + - name: revision + value: main + - name: pathInRepo + value: tasks/export-logs/0.1/export-logs-to-quay.yaml + params: + - name: pipeline-run-name + value: $(context.pipelineRun.name) + - name: namespace + value: $(context.pipelineRun.namespace) + - name: quay-repo + value: "quay.io/openshift-lightspeed/ols-operator-artifacts" + - name: artifact-credentials-secret + value: ols-konflux-artifacts-bot diff --git a/.tekton/integration-tests/scripts/install-operator.sh b/.tekton/integration-tests/scripts/install-operator.sh new file mode 100755 index 0000000..d81f9cf --- /dev/null +++ b/.tekton/integration-tests/scripts/install-operator.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Konflux: install the agentic operator onto the ephemeral cluster using make targets. +# Run from the repo root after checkout. +# +# Required env: +# IMG — operator image (from SNAPSHOT) +# KUBECONFIG — path to kubeconfig +# +# Optional env: +# OPERATOR_NAMESPACE (default: openshift-lightspeed) +# SANDBOX_MODE (default: bare-pod) +# SANDBOX_IMAGE (default: quay.io/openshift-lightspeed/ols-qe:lightspeed-mock-agent) + +set -euo pipefail + +: "${IMG:?IMG must be set to the operator image}" +: "${KUBECONFIG:?KUBECONFIG must be set}" + +OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE:-openshift-lightspeed}" +SANDBOX_MODE="${SANDBOX_MODE:-bare-pod}" +SANDBOX_IMAGE="${SANDBOX_IMAGE:-quay.io/openshift-lightspeed/ols-qe:lightspeed-mock-agent}" + +echo "=== Agentic operator install ===" +echo " IMG: ${IMG}" +echo " OPERATOR_NAMESPACE: ${OPERATOR_NAMESPACE}" +echo " SANDBOX_MODE: ${SANDBOX_MODE}" +echo " SANDBOX_IMAGE: ${SANDBOX_IMAGE}" +echo "=================================" + +# Ensure namespace exists. +oc create namespace "${OPERATOR_NAMESPACE}" --dry-run=client -o yaml | oc apply -f - + +# Install CRDs. +echo "Installing CRDs..." +make install + +# Deploy operator (kustomize-based). +echo "Deploying operator..." +make deploy IMG="${IMG}" OPERATOR_NAMESPACE="${OPERATOR_NAMESPACE}" SANDBOX_MODE="${SANDBOX_MODE}" SANDBOX_IMAGE="${SANDBOX_IMAGE}" + +# Grant cluster-admin to operator SA (same as quickstart — covers escalation + SCC). +echo "Granting cluster-admin to operator SA..." +oc apply -f - </dev/null 2>&1 && $(KUBECTL) get crd "$$core_crd" >/dev/null 2>&1 && $(KUBECTL) get crd "$$st_crd" >/dev/null 2>&1; then \ - echo "Agent Sandbox already installed ($$ext_crd, $$core_crd, $$st_crd)."; \ + echo "Agent Sandbox CRDs present ($$ext_crd, $$core_crd, $$st_crd)."; \ else \ - v="$(AGENT_SANDBOX_VERSION)"; \ - base="$(AGENT_SANDBOX_RELEASE_BASE)"; \ - echo "Applying Agent Sandbox $$v ($$base/$$v/{manifest,extensions}.yaml) ..."; \ - $(KUBECTL) apply -f "$$base/$$v/manifest.yaml"; \ - $(KUBECTL) apply -f "$$base/$$v/extensions.yaml"; \ + echo "ERROR: Sandbox CRDs not found. Install the Sandbox operator before using sandbox-claim mode."; \ + echo " See: https://github.com/kubernetes-sigs/agent-sandbox/releases"; \ + exit 1; \ fi .PHONY: install @@ -146,12 +152,15 @@ uninstall: kustomize ## Remove CRDs from config/crd. Use ignore-not-found=true i $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: manifests install-agent-sandbox kustomize ## Pre-built image: apply CRDs+RBAC+Deployment (set IMG). For OpenShift dev build+push+integrated registry use deploy-local. +deploy: manifests validate-agent-sandbox kustomize ## Pre-built image: apply CRDs+RBAC+Deployment (set IMG). For OpenShift dev build+push+integrated registry use deploy-local. @tmpdir=$$(mktemp -d); \ trap 'rm -rf "$$tmpdir"' EXIT; \ cp -a config "$$tmpdir/"; \ for f in "$$tmpdir/config/manager/manager.yaml" "$$tmpdir/config/rbac/role_binding.yaml" "$$tmpdir/config/rbac/service_account.yaml" "$$tmpdir/config/default/kustomization.yaml"; do \ - sed -e 's|__OPERATOR_NAMESPACE__|$(OPERATOR_NAMESPACE)|g' "$$f" > "$$f.tmp" && mv "$$f.tmp" "$$f"; \ + sed -e 's|__OPERATOR_NAMESPACE__|$(OPERATOR_NAMESPACE)|g' \ + -e 's|__SANDBOX_MODE__|$(SANDBOX_MODE)|g' \ + -e 's|__SANDBOX_IMAGE__|$(SANDBOX_IMAGE)|g' \ + "$$f" > "$$f.tmp" && mv "$$f.tmp" "$$f"; \ done; \ cd "$$tmpdir/config/manager" && $(KUSTOMIZE) edit set image controller=$(IMG); \ $(KUSTOMIZE) build "$$tmpdir/config/default" | $(KUBECTL) apply -f - @@ -239,10 +248,11 @@ undeploy: kustomize ## Remove in-cluster operator (CRDs + RBAC + Deployment). Us $(KUSTOMIZE) build "$$tmpdir/config/default" | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - || true .PHONY: run -run: install install-agent-sandbox vet ## install + install-agent-sandbox (no-op if CRDs exist) + vet, then run the controller locally. - # Same kubeconfig discovery as other controller-runtime apps (see README.md). +run: install validate-agent-sandbox vet ## install + validate-agent-sandbox (no-op if bare-pod) + vet, then run the controller locally. go run ./cmd/main.go \ --namespace=$(OPERATOR_NAMESPACE) \ + --sandbox-mode=$(SANDBOX_MODE) \ + --agentic-sandbox-image=$(SANDBOX_IMAGE) \ --metrics-bind-address=$(METRICS_BIND_ADDRESS) \ --health-probe-bind-address=$(HEALTH_PROBE_BIND_ADDRESS) diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 18ff297..405ca68 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -30,6 +30,9 @@ spec: command: - /manager args: + - --namespace=__OPERATOR_NAMESPACE__ + - --sandbox-mode=__SANDBOX_MODE__ + - --agentic-sandbox-image=__SANDBOX_IMAGE__ - --metrics-bind-address=:8080 - --health-probe-bind-address=:8081 env: diff --git a/test/e2e/analysis_execution_test.go b/test/e2e/analysis_execution_test.go index 73f9822..b70d78b 100644 --- a/test/e2e/analysis_execution_test.go +++ b/test/e2e/analysis_execution_test.go @@ -32,16 +32,18 @@ import ( // 4. Assert: ProposalApproval exists, AnalysisResult exists, Proposal phase = Proposed // 5. Delete Proposal and verify sandbox released (finalizer completes) func TestAnalysisFlow_ProposalToProposed(t *testing.T) { + t.Log("=== TestAnalysisFlow_ProposalToProposed: validates Pending → Analyzing → Proposed ===") c := newClient(t) ctx := context.Background() + t.Log("Creating fixtures (LLMProvider, Agent, ApprovalPolicy, Secret)") createFixtures(t, c) prop := createProposal(t, c, "e2e-analysis-flow") + t.Logf("Proposal created: %s/%s", testNS, prop.Name) - // Wait for analysis to complete. + t.Log("Waiting for phase: Proposed (analysis complete)") updated := waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseProposed) - - // --- Verify outcomes --- + t.Log("Phase reached: Proposed") // Condition: Analyzed=True var analyzedFound bool @@ -56,6 +58,7 @@ func TestAnalysisFlow_ProposalToProposed(t *testing.T) { if !analyzedFound { t.Error("Analyzed condition not found on Proposal status") } + t.Log("Verified: Analyzed=True condition present") // ProposalApproval exists with owner reference. var approval agenticv1alpha1.ProposalApproval @@ -67,6 +70,7 @@ func TestAnalysisFlow_ProposalToProposed(t *testing.T) { } else if approval.OwnerReferences[0].Name != prop.Name { t.Errorf("ProposalApproval owner = %q, want %q", approval.OwnerReferences[0].Name, prop.Name) } + t.Log("Verified: ProposalApproval exists with correct owner reference") // AnalysisResult exists with owner reference and options. var analysisList agenticv1alpha1.AnalysisResultList @@ -93,11 +97,13 @@ func TestAnalysisFlow_ProposalToProposed(t *testing.T) { if opt.Proposal.Description == "" { t.Error("option proposal description is empty") } + t.Logf("Verified: AnalysisResult %s with %d option(s), title=%q", ar.Name, len(ar.Status.Options), opt.Title) // Sandbox info recorded. if updated.Status.Steps.Analysis.Sandbox.ClaimName == "" { t.Error("status.steps.analysis.sandbox.claimName is empty") } + t.Logf("Verified: sandbox info recorded, claimName=%s", updated.Status.Steps.Analysis.Sandbox.ClaimName) // Results tracked. if len(updated.Status.Steps.Analysis.Results) == 0 { @@ -106,14 +112,15 @@ func TestAnalysisFlow_ProposalToProposed(t *testing.T) { if updated.Status.Steps.Analysis.Results[0].Name == "" { t.Error("analysis result ref name is empty") } + t.Logf("Verified: analysis results tracked, ref=%s", updated.Status.Steps.Analysis.Results[0].Name) - // --- Delete Proposal and verify sandbox released --- + // Delete Proposal and verify sandbox released. claimName := updated.Status.Steps.Analysis.Sandbox.ClaimName + t.Log("Deleting Proposal — verifying finalizer cleanup") if err := c.Delete(ctx, prop); err != nil { t.Fatalf("delete Proposal: %v", err) } waitForDeletion(t, c, prop.Name) - - t.Logf("PASS: phase=Proposed, analysisResult=%s, sandbox=%s (released after deletion)", - ar.Name, claimName) + t.Logf("Verified: Proposal deleted, sandbox %s released", claimName) + t.Log("PASS: analysis complete, phase=Proposed, sandbox released") } diff --git a/test/e2e/denial_test.go b/test/e2e/denial_test.go index cac010d..1bde113 100644 --- a/test/e2e/denial_test.go +++ b/test/e2e/denial_test.go @@ -18,20 +18,25 @@ import ( // 3. Wait for phase = Denied (terminal) // 4. Assert: Denied condition present, sandboxes released on deletion func TestDenialFlow_ProposedToDenied(t *testing.T) { + t.Log("=== TestDenialFlow_ProposedToDenied: validates execution denial → Denied terminal ===") c := newClient(t) ctx := context.Background() + t.Log("Creating fixtures (LLMProvider, Agent, ApprovalPolicy, Secret)") createFixtures(t, c) prop := createProposal(t, c, "e2e-denial-flow") + t.Logf("Proposal created: %s/%s", testNS, prop.Name) - // Drive to Proposed (analysis complete). + t.Log("Waiting for phase: Proposed (analysis complete)") waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseProposed) + t.Log("Phase reached: Proposed") - // Deny execution. + t.Log("Denying execution stage") denyStage(t, c, prop.Name, agenticv1alpha1.ApprovalStageExecution) - // Wait for Denied (terminal). + t.Log("Waiting for phase: Denied (terminal)") updated := waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseDenied) + t.Log("Phase reached: Denied") // --- Verify: Denied condition --- var deniedFound bool @@ -46,12 +51,15 @@ func TestDenialFlow_ProposedToDenied(t *testing.T) { if !deniedFound { t.Error("Denied condition not found") } + t.Log("Verified: Denied=True condition present") // --- Cleanup --- + t.Log("Deleting Proposal — verifying finalizer cleanup") if err := c.Delete(ctx, prop); err != nil { t.Fatalf("delete Proposal: %v", err) } waitForDeletion(t, c, prop.Name) + t.Log("Verified: Proposal deleted successfully") t.Logf("PASS: execution denied, phase=Denied (terminal)") } diff --git a/test/e2e/execution_test.go b/test/e2e/execution_test.go index f6dd2da..3af6ff5 100644 --- a/test/e2e/execution_test.go +++ b/test/e2e/execution_test.go @@ -6,6 +6,7 @@ import ( "context" "testing" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -23,20 +24,25 @@ import ( // 5. Assert: ExecutionResult exists, Executed=True, sandbox info, RBAC annotation // 6. Delete Proposal, verify RBAC cleaned up func TestExecutionFlow_ProposedToVerifying(t *testing.T) { + t.Log("=== TestExecutionFlow_ProposedToVerifying: validates Proposed → Executing → Verifying with RBAC + SA ===") c := newClient(t) ctx := context.Background() + t.Log("Creating fixtures (LLMProvider, Agent, ApprovalPolicy, Secret)") createFixtures(t, c) prop := createProposal(t, c, "e2e-execution-flow") + t.Logf("Proposal created: %s/%s", testNS, prop.Name) - // Drive to Proposed (analysis complete). + t.Log("Waiting for phase: Proposed (analysis complete)") waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseProposed) + t.Log("Phase reached: Proposed") - // Approve execution with option 0. + t.Log("Approving execution with option 0") approveExecution(t, c, prop.Name, 0) - // Wait for Executing phase — RBAC should exist during this window (mock delays 60s). + t.Log("Waiting for phase: Executing") waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseExecuting) + t.Log("Phase reached: Executing — checking RBAC") // --- Verify: RBAC created --- roleName := "ls-exec-" + prop.Name @@ -50,6 +56,21 @@ func TestExecutionFlow_ProposedToVerifying(t *testing.T) { if err := c.Get(ctx, types.NamespacedName{Name: roleName, Namespace: "staging"}, &binding); err != nil { t.Fatalf("get RoleBinding %s in staging: %v", roleName, err) } + t.Logf("Verified: RoleBinding %s exists in staging", roleName) + + // Verify: per-proposal execution SA created. + saName := "ls-exec-" + testNS + "-" + prop.Name + var sa corev1.ServiceAccount + if err := c.Get(ctx, types.NamespacedName{Name: saName, Namespace: testNS}, &sa); err != nil { + t.Fatalf("get execution SA %s: %v", saName, err) + } + t.Logf("Execution SA %s exists", saName) + + // Verify: RoleBinding references the per-proposal SA. + if len(binding.Subjects) == 0 || binding.Subjects[0].Name != saName { + t.Errorf("RoleBinding subject = %v, want SA %s", binding.Subjects, saName) + } + t.Log("Verified: RoleBinding subjects correct SA") // Verify annotation on Proposal. var current agenticv1alpha1.Proposal @@ -59,9 +80,11 @@ func TestExecutionFlow_ProposedToVerifying(t *testing.T) { if current.Annotations["agentic.openshift.io/rbac-namespaces"] == "" { t.Error("rbac-namespaces annotation is empty") } + t.Log("Verified: RBAC annotation present on Proposal") - // Wait for execution to complete → Verifying phase. + t.Log("Waiting for phase: Verifying (execution complete)") updated := waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseVerifying) + t.Log("Phase reached: Verifying") // --- Verify: Executed condition --- var executedFound bool @@ -76,6 +99,7 @@ func TestExecutionFlow_ProposedToVerifying(t *testing.T) { if !executedFound { t.Error("Executed condition not found") } + t.Log("Verified: Executed=True condition present") // --- Verify: ExecutionResult exists --- var execList agenticv1alpha1.ExecutionResultList @@ -88,13 +112,16 @@ func TestExecutionFlow_ProposedToVerifying(t *testing.T) { if len(execList.Items[0].OwnerReferences) == 0 { t.Error("ExecutionResult has no owner references") } + t.Logf("Verified: ExecutionResult %s exists with owner reference", execList.Items[0].Name) // --- Verify: execution sandbox info --- if updated.Status.Steps.Execution.Sandbox.ClaimName == "" { t.Error("status.steps.execution.sandbox.claimName is empty") } + t.Logf("Verified: execution sandbox info recorded, claimName=%s", updated.Status.Steps.Execution.Sandbox.ClaimName) // --- Cleanup and verify RBAC removed --- + t.Log("Deleting Proposal — verifying RBAC + SA cleanup") if err := c.Delete(ctx, prop); err != nil { t.Fatalf("delete Proposal: %v", err) } @@ -103,6 +130,10 @@ func TestExecutionFlow_ProposedToVerifying(t *testing.T) { if err := c.Get(ctx, types.NamespacedName{Name: roleName, Namespace: "staging"}, &role); err == nil { t.Errorf("Role %s still exists after Proposal deletion — RBAC not cleaned up", roleName) } + if err := c.Get(ctx, types.NamespacedName{Name: saName, Namespace: testNS}, &sa); err == nil { + t.Errorf("SA %s still exists after Proposal deletion — not cleaned up", saName) + } + t.Log("Verified: RBAC Role + SA cleaned up after deletion") - t.Logf("PASS: execution complete, RBAC created and cleaned up") + t.Logf("PASS: execution complete, RBAC + SA created and cleaned up") } diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index e62d711..b44e16d 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -23,9 +23,15 @@ import ( const ( pollInterval = 2 * time.Second pollTimeout = 10 * time.Minute - testNS = "openshift-lightspeed" ) +var testNS = func() string { + if ns := os.Getenv("TEST_NAMESPACE"); ns != "" { + return ns + } + return "openshift-lightspeed" +}() + // --- Client --- func newClient(t *testing.T) client.Client { @@ -109,6 +115,13 @@ func deleteSandboxClaim(t *testing.T, c client.Client, name, namespace string) { _ = c.Delete(context.Background(), obj) } +// deleteBarePod removes a bare pod by name from the operator namespace (same as testNS). +func deleteBarePod(t *testing.T, c client.Client, name string) { + t.Helper() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNS}} + _ = c.Delete(context.Background(), pod) +} + // --- Fixture builders --- // e2eFixtures holds the prerequisite CRs needed for any proposal flow. @@ -150,7 +163,6 @@ func createFixtures(t *testing.T, c client.Client) *e2eFixtures { Spec: agenticv1alpha1.ApprovalPolicySpec{ Stages: []agenticv1alpha1.ApprovalPolicyStage{ {Name: agenticv1alpha1.SandboxStepAnalysis, Approval: agenticv1alpha1.ApprovalModeAutomatic}, - {Name: agenticv1alpha1.SandboxStepVerification, Approval: agenticv1alpha1.ApprovalModeAutomatic}, }, }, }, @@ -195,12 +207,15 @@ func createProposal(t *testing.T, c client.Client, name string) *agenticv1alpha1 }, } - // Clean leftovers. + // Clean leftovers from previous runs (proposals, approvals, sandbox claims, bare pods). cleanup(t, c, &agenticv1alpha1.Proposal{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNS}}) cleanup(t, c, &agenticv1alpha1.ProposalApproval{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNS}}) deleteSandboxClaim(t, c, "ls-analysis-"+name, testNS) deleteSandboxClaim(t, c, "ls-execution-"+name, testNS) deleteSandboxClaim(t, c, "ls-verification-"+name, testNS) + deleteBarePod(t, c, "ls-analysis-"+name) + deleteBarePod(t, c, "ls-execution-"+name) + deleteBarePod(t, c, "ls-verification-"+name) if err := c.Create(ctx, prop); err != nil { t.Fatalf("create Proposal: %v", err) @@ -321,3 +336,34 @@ func approveExecution(t *testing.T, c client.Client, name string, optionIdx int3 } t.Logf("approved execution with option %d", optionIdx) } + +// approveVerification patches the ProposalApproval to approve verification. +func approveVerification(t *testing.T, c client.Client, name string) { + t.Helper() + ctx := context.Background() + + var approval agenticv1alpha1.ProposalApproval + if err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: testNS}, &approval); err != nil { + t.Fatalf("get ProposalApproval for verification approval: %v", err) + } + + base := approval.DeepCopy() + found := false + for i, s := range approval.Spec.Stages { + if s.Type == agenticv1alpha1.ApprovalStageVerification { + approval.Spec.Stages[i].Verification = agenticv1alpha1.VerificationApproval{Agent: "e2e-agent"} + found = true + break + } + } + if !found { + approval.Spec.Stages = append(approval.Spec.Stages, agenticv1alpha1.ApprovalStage{ + Type: agenticv1alpha1.ApprovalStageVerification, + Verification: agenticv1alpha1.VerificationApproval{Agent: "e2e-agent"}, + }) + } + if err := c.Patch(ctx, &approval, client.MergeFrom(base)); err != nil { + t.Fatalf("approve verification: %v", err) + } + t.Logf("approved verification") +} diff --git a/test/e2e/verification_test.go b/test/e2e/verification_test.go index 1058b83..6a48453 100644 --- a/test/e2e/verification_test.go +++ b/test/e2e/verification_test.go @@ -21,20 +21,32 @@ import ( // 3. Assert: VerificationResult exists, Verified=True, terminal state // 4. Delete Proposal, verify RBAC cleaned up func TestVerificationFlow_VerifyingToCompleted(t *testing.T) { + t.Log("=== TestVerificationFlow_VerifyingToCompleted: validates full lifecycle → Completed ===") c := newClient(t) ctx := context.Background() + t.Log("Creating fixtures (LLMProvider, Agent, ApprovalPolicy, Secret)") createFixtures(t, c) prop := createProposal(t, c, "e2e-verification-flow") + t.Logf("Proposal created: %s/%s", testNS, prop.Name) - // Drive to Proposed. + t.Log("Waiting for phase: Proposed (analysis complete)") waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseProposed) + t.Log("Phase reached: Proposed") - // Approve execution. + t.Log("Approving execution with option 0") approveExecution(t, c, prop.Name, 0) - // Wait for Completed (execution + verification both finish). + t.Log("Waiting for phase: Verifying (execution complete)") + waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseVerifying) + t.Log("Phase reached: Verifying") + + t.Log("Approving verification") + approveVerification(t, c, prop.Name) + + t.Log("Waiting for phase: Completed (verification complete)") updated := waitForPhase(t, c, prop.Name, agenticv1alpha1.ProposalPhaseCompleted) + t.Log("Phase reached: Completed") // --- Verify: Verified condition --- var verifiedFound bool @@ -49,6 +61,7 @@ func TestVerificationFlow_VerifyingToCompleted(t *testing.T) { if !verifiedFound { t.Error("Verified condition not found") } + t.Log("Verified: Verified=True condition present") // --- Verify: VerificationResult exists --- var verifyList agenticv1alpha1.VerificationResultList @@ -61,14 +74,17 @@ func TestVerificationFlow_VerifyingToCompleted(t *testing.T) { if len(verifyList.Items[0].OwnerReferences) == 0 { t.Error("VerificationResult has no owner references") } + t.Logf("Verified: VerificationResult %s exists with owner reference", verifyList.Items[0].Name) // --- Verify: verification sandbox info --- if updated.Status.Steps.Verification.Sandbox.ClaimName == "" { t.Error("status.steps.verification.sandbox.claimName is empty") } + t.Logf("Verified: verification sandbox info recorded, claimName=%s", updated.Status.Steps.Verification.Sandbox.ClaimName) // --- Cleanup and verify RBAC removed --- roleName := "ls-exec-" + prop.Name + t.Log("Deleting Proposal — verifying RBAC cleanup") if err := c.Delete(ctx, prop); err != nil { t.Fatalf("delete Proposal: %v", err) } @@ -78,6 +94,6 @@ func TestVerificationFlow_VerifyingToCompleted(t *testing.T) { if err := c.Get(ctx, types.NamespacedName{Name: roleName, Namespace: "staging"}, &role); err == nil { t.Errorf("Role %s still exists after deletion — RBAC not cleaned up", roleName) } - - t.Logf("PASS: verification complete, phase=Completed, RBAC cleaned after deletion") + t.Log("Verified: RBAC cleaned up after deletion") + t.Log("PASS: verification complete, phase=Completed, RBAC cleaned") }