Merge pull request #1512 from hj-johannes-lee/PR-2023-027

e2e: make running subsets of e2e tests more organized

Author: mythi

.github/workflows/lib-e2e.yaml

@@ -34,9 +34,11 @@ jobs:
             runner: simics-gnr
             images: intel-iaa-plugin intel-idxd-config-initcontainer accel-config-demo intel-deviceplugin-operator
           - name: e2e-qat
+            targetjob: e2e-qat FOCUS=Resource:generic
             runner: qat
             images: intel-qat-plugin intel-qat-initcontainer crypto-perf
           - name: e2e-qat4
+            targetjob: e2e-qat FOCUS="Mode:dpdk.*Resource:(cy|dc)" SKIP=App:crypto-perf
             runner: simics-spr
             images: intel-qat-plugin intel-qat-initcontainer openssl-qat-engine
           - name: e2e-sgx

DEVEL.md

@@ -183,34 +183,107 @@ container images with the executables under test must be available in the
 cluster. If these two conditions are satisfied, run the tests with:
 
 ```bash
-$ go test -v ./test/e2e/...
+# Run all e2e tests in this repository
+go test -v ./test/e2e/...
 ```
 
-In case you want to run only certain tests, e.g., QAT ones, run:
+If you need to specify paths to your custom `kubeconfig` containing
+embedded authentication info then add the `-kubeconfig` argument:
 
 ```bash
-$ go test -v ./test/e2e/... -args -ginkgo.focus "QAT"
+go test -v ./test/e2e/... -args -kubeconfig /path/to/kubeconfig
 ```
 
-If you need to specify paths to your custom `kubeconfig` containing
-embedded authentication info then add the `-kubeconfig` argument:
+The full list of available options can be obtained with:
 
 ```bash
-$ go test -v ./test/e2e/... -args -kubeconfig /path/to/kubeconfig
+go test ./test/e2e/... -args -help
 ```
 
-The full list of available options can be obtained with:
+In most cases, it would not be possible to run all E2E tests in one system.
+For running a subset of tests, there are labels that you can use to pick out specific parts.
+You can run the tests with:
+```bash
+# Run a subset of tests
+go test -v ./test/e2e/... -args -ginkgo.focus <labels in regex> -ginkgo.skip <labels in regex>
+```
+
+#### Table of Labels
+
+| Device | Mode             | Resource    | App                            |
+|:-------|:-----------------|:------------|:-------------------------------|
+| `dlb`  |-                 | `pf`, `vf`  | `libdlb`                       |
+| `dsa`  |-                 | `dedicated` | `accel-config`                 |
+| `fpga` | `af`, `region`   |             | `opae-nlb-demo`                |
+| `gpu`  |-                 | `i915`      | `busybox`, `tensorflow`        |
+| `iaa`  |-                 | `dedicated` | `accel-config`                 |
+| `qat`  | `dpdk`           | `dc`        | `openssl`                      |
+| `qat`  | `dpdk`           | `cy`        | `openssl`, `crypto-perf`       |
+| `qat`  | `dpdk`           | `generic`   | `crypto-perf`, `compress-perf` |
+| `qat`  | `kernel`         | `cy1_dc0`   | `busybox`                      |
+| `sgx`  |-                 |             | `sgx-sdk-demo`                 |
+
+#### Examples
+
+```bash
+# DLB for VF resource without any app running
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:dlb.*Resource:vf.*App:noapp"
+
+# FPGA with af mode with opae-nlb-demo app running
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:fpga.*Mode:af.*App:opae-nlb-demo"
+
+# GPU with running only tensorflow app
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:gpu.*App:tensorflow"
+#or
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:gpu" -ginkgo.skip "App:busybox"
+
+# QAT for qat4 cy resource with openssl app running
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:cy.*App:openssl"
+
+# QAT with dpdk mode for qat2 generic resource with all apps running
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:qat.*Resource:generic.*App:(crypto-perf|compress-perf)"
+
+# SGX without running sgx-sdk-demo app
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:sgx" -ginkgo.skip "App:sgx-sdk-demo"
+
+# All of Sapphire Rapids device plugins
+go test -v ./test/e2e/... -args -ginkgo.focus "Device:(dlb|dsa|iaa|qat|sgx)"
+```
+
+## Predefined E2E Tests
+
+It is possible to run predefined e2e tests with:
+```
+make e2e-<device> [E2E_LEVEL={basic|full}] [FOCUS=<labels in regex>] [SKIP=<labels in regex>]
+```  
+
+| `E2E_LEVEL`   | Equivalent `FOCUS` or `SKIP` | Explanation                                                                                      |
+:-------------- |:---------------------------- |:------------------------------------------------------------------------------------------------ |
+| `basic`       | `FOCUS=App:noapp`            | `basic` does not run any app pod, but checks if the plugin works and the resources are available |
+| `full`        | `SKIP=App:noapp`             | `full` checks all resources, runs all apps except the spec kept for no app running               | 
+
+### Examples
 
 ```bash
-$ go test ./test/e2e/... -args -help
+# DLB for both of pf and vf resources with running libdlb app
+make e2e-dlb E2E_LEVEL=full
+
+# QAT for cy resource with running only openssl app
+make e2e-qat FOCUS=Resource:cy.*App:openssl
+
+# QAT for dc resource without running any app
+make e2e-qat E2E_LEVEL=basic FOCUS=Resource:dc
+
+# GPU without running tensorflow app
+make e2e-gpu E2E_LEVEL=full SKIP=tensorflow
 ```
 
 It is also possible to run the tests which don't depend on hardware
 without a pre-configured Kubernetes cluster. Just make sure you have
 [Kind](https://kind.sigs.k8s.io/) installed on your host and run:
 
 ```
-$ make test-with-kind
+make test-with-kind
 ```
 
 ### Run Controller Tests with a Local Control Plane

Makefile

@@ -140,29 +140,36 @@ REG?=$(ORG)/
 TAG?=devel
 export TAG
 
+ifeq ($(E2E_LEVEL), $(filter $(E2E_LEVEL), full))
+  GENERATED_SKIP_OPT=-ginkgo.skip "App:noapp"
+else ifeq ($(E2E_LEVEL),basic)
+  ADDITIONAL_FOCUS_REGEX=App:noapp
+else
+  $(error Unsupported E2E_LEVEL value: $(E2E_LEVEL). Possible options: full, basic)
+endif
+GENERATED_SKIP_OPT += $(if $(SKIP),-ginkgo.skip "$(SKIP)")
+ADDITIONAL_FOCUS_REGEX := $(if $(FOCUS),$(FOCUS).*$(ADDITIONAL_FOCUS_REGEX),$(ADDITIONAL_FOCUS_REGEX))
+
 e2e-fpga:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "FPGA" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:fpga.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-qat:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "QAT Gen2" -delete-namespace-on-failure=false
-
-e2e-qat4:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "QAT Gen4" -ginkgo.skip "dpdk crypto-perf" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:qat.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-sgx:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "SGX" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:sgx.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-gpu:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "GPU" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:gpu.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-dsa:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "DSA" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dsa.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-iaa:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "IAA" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:iaa.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 e2e-dlb:
-	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "DLB" -delete-namespace-on-failure=false
+	@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dlb.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
 
 terrascan:
 	@ls deployments/*/kustomization.yaml | while read f ; \

test/e2e/dlb/dlb.go

@@ -39,7 +39,7 @@ const (
 )
 
 func init() {
-	ginkgo.Describe("DLB plugin", describe)
+	ginkgo.Describe("DLB plugin [Device:dlb]", describe)
 }
 
 func describe() {
@@ -81,30 +81,38 @@ func describe() {
 		}
 	})
 
-	ginkgo.Context("When PF resources are available", func() {
+	ginkgo.Context("When PF resources are available [Resource:pf]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			resource := v1.ResourceName("dlb.intel.com/pf")
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resource, 30*time.Second); err != nil {
 				framework.Failf("unable to wait for nodes to have positive allocatable resource %s: %v", resource, err)
 			}
 		})
 
-		ginkgo.It("can run demo app", func(ctx context.Context) {
+		ginkgo.It("can run demo app [App:libdlb]", func(ctx context.Context) {
 			runDemoApp(ctx, "PF", demoPFYaml, f)
 		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
 	})
 
-	ginkgo.Context("When VF resources are available", func() {
+	ginkgo.Context("When VF resources are available [Resource:vf]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			resource := v1.ResourceName("dlb.intel.com/vf")
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resource, 30*time.Second); err != nil {
 				framework.Failf("unable to wait for nodes to have positive allocatable resource %s: %v", resource, err)
 			}
 		})
 
-		ginkgo.It("can run demo app", func(ctx context.Context) {
+		ginkgo.It("can run demo app [App:libdlb]", func(ctx context.Context) {
 			runDemoApp(ctx, "VF", demoVFYaml, f)
 		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
 	})
 }
 

test/e2e/dsa/dsa.go

@@ -40,7 +40,7 @@ const (
 )
 
 func init() {
-	ginkgo.Describe("DSA plugin", describe)
+	ginkgo.Describe("DSA plugin [Device:dsa]", describe)
 }
 
 func describe() {
@@ -94,20 +94,24 @@ func describe() {
 		}
 	})
 
-	ginkgo.Context("When DSA resources are available", func() {
+	ginkgo.Context("When DSA resources are available [Resource:dedicated]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			ginkgo.By("checking if the resource is allocatable")
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "dsa.intel.com/wq-user-dedicated", 300*time.Second); err != nil {
 				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
 			}
 		})
 
-		ginkgo.It("deploys a demo app", func(ctx context.Context) {
+		ginkgo.It("deploys a demo app [App:accel-config]", func(ctx context.Context) {
 			e2ekubectl.RunKubectlOrDie(f.Namespace.Name, "apply", "-f", demoPath)
 
 			ginkgo.By("waiting for the DSA demo to succeed")
 			err := e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, podName, f.Namespace.Name, 200*time.Second)
 			gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, podName, podName))
 		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
 	})
 }

test/e2e/fpga/fpga.go

@@ -47,7 +47,7 @@ const (
 )
 
 func init() {
-	ginkgo.Describe("FPGA Plugin", describe)
+	ginkgo.Describe("FPGA Plugin [Device:fpga]", describe)
 }
 
 func describe() {
@@ -64,23 +64,43 @@ func describe() {
 	fmw := framework.NewDefaultFramework("fpgaplugin-e2e")
 	fmw.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
 
-	ginkgo.Context("When FPGA plugin is running in region mode", func() {
+	ginkgo.Context("When FPGA plugin is running in region mode [Mode:region]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, arria10NodeResource, "region")
 		})
-		ginkgo.It("runs an opae-nlb-demo pod two times", func(ctx context.Context) {
+		ginkgo.It("runs an opae-nlb-demo pod two times [App:opae-nlb-demo]", func(ctx context.Context) {
 			runTestCase(ctx, fmw, "region", nlb3PodResource, "nlb3", "nlb0")
 			runTestCase(ctx, fmw, "region", nlb0PodResource, "nlb0", "nlb3")
 		})
 	})
 
-	ginkgo.Context("When FPGA plugin is running in af mode", func() {
+	ginkgo.Context("When FPGA plugin is running in af mode [Mode:af]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, nlb0NodeResource, "af")
 		})
-		ginkgo.It("runs an opae-nlb-demo pod", func(ctx context.Context) {
+		ginkgo.It("runs an opae-nlb-demo pod [App:opae-nlb-demo]", func(ctx context.Context) {
 			runTestCase(ctx, fmw, "af", nlb0PodResourceAF, "nlb0", "nlb3")
 		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
+	})
+
+	ginkgo.Context("When FPGA plugin is running in region mode [Mode:region]", func() {
+		ginkgo.BeforeEach(func(ctx context.Context) {
+			runDevicePlugin(ctx, fmw, pluginKustomizationPath, mappingsCollectionPath, arria10NodeResource, "region")
+		})
+		ginkgo.It("runs [App:opae-nlb-demo]", func(ctx context.Context) {
+			runTestCase(ctx, fmw, "region", nlb3PodResource, "nlb3", "nlb0")
+		})
+		ginkgo.It("runs an opae-nlb-demo pod [App:opae-nlb-demo]", func(ctx context.Context) {
+			runTestCase(ctx, fmw, "region", nlb0PodResource, "nlb0", "nlb3")
+		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
 	})
 }
 

test/e2e/gpu/gpu.go

@@ -43,7 +43,7 @@ const (
 )
 
 func init() {
-	ginkgo.Describe("GPU plugin", describe)
+	ginkgo.Describe("GPU plugin [Device:gpu]", describe)
 }
 
 func describe() {
@@ -74,14 +74,14 @@ func describe() {
 		}
 	})
 
-	ginkgo.Context("When GPU resources are available", func() {
+	ginkgo.Context("When GPU resources are available [Resource:i915]", func() {
 		ginkgo.BeforeEach(func(ctx context.Context) {
 			ginkgo.By("checking if the resource is allocatable")
 			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "gpu.intel.com/i915", 30*time.Second); err != nil {
 				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
 			}
 		})
-		ginkgo.It("checks availability of GPU resources", func(ctx context.Context) {
+		ginkgo.It("checks availability of GPU resources [App:busybox]", func(ctx context.Context) {
 			ginkgo.By("submitting a pod requesting GPU resources")
 			podSpec := &v1.Pod{
 				ObjectMeta: metav1.ObjectMeta{Name: "gpuplugin-tester"},
@@ -122,7 +122,7 @@ func describe() {
 			framework.Logf("found card and renderD from the log")
 		})
 
-		ginkgo.It("run a small workload on the GPU", func(ctx context.Context) {
+		ginkgo.It("run a small workload on the GPU [App:tensorflow]", func(ctx context.Context) {
 			kustomYaml, err := utils.LocateRepoFile(tfKustomizationYaml)
 			if err != nil {
 				framework.Failf("unable to locate %q: %v", tfKustomizationYaml, err)
@@ -139,5 +139,9 @@ func describe() {
 
 			framework.Logf("tensorflow execution succeeded!")
 		})
+
+		ginkgo.When("there is no app to run [App:noapp]", func() {
+			ginkgo.It("does nothing", func() {})
+		})
 	})
 }