qat,e2e: add heartbeat and auto-reset validation

Signed-off-by: Hyeongju Johannes Lee <[email protected]>

Author: hj-johannes-lee

test/e2e/qat/qatplugin_dpdk.go

@@ -16,6 +16,8 @@ package qat
 
 import (
 	"context"
+	"fmt"
+	"os"
 	"path/filepath"
 	"strconv"
 	"time"
@@ -38,6 +40,8 @@ const (
 	qatPluginKustomizationYaml = "deployments/qat_plugin/overlays/e2e/kustomization.yaml"
 	cryptoTestYaml             = "deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml"
 	compressTestYaml           = "deployments/qat_dpdk_app/compress-perf/compress-perf-dpdk-pod-requesting-qat-dc.yaml"
+	autoResetPattern           = "/sys/bus/pci/drivers/*/*/qat/auto-reset"
+	injectErrorPattern         = "/sys/kernel/debug/qat_*/heartbeat/inject_error"
 )
 
 const (
@@ -137,6 +141,27 @@ func describeQatDpdkPlugin() {
 		ginkgo.When("there is no app to run [App:noapp]", func() {
 			ginkgo.It("does nothing", func() {})
 		})
+
+		ginkgo.It("checks auto-reset functionality [Functionality: auto-reset]", func(ctx context.Context) {
+			ginkgo.By("checking if heartbeat status is read correctly")
+			if err := setQatAutoReset("off"); err != nil {
+				framework.Failf("unable to set auto reset in this system: %v", err)
+			}
+			if err := injectError(); err != nil {
+				framework.Failf("unable to test error injection in this system: %v", err)
+			}
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+				framework.Failf("unable to wait for nodes to have zero resource: %v", err)
+			}
+
+			ginkgo.By("checking if auto-reset works to solve injected errors")
+			if err := setQatAutoReset("on"); err != nil {
+				framework.Failf("unable to set auto reset in this system: %v", err)
+			}
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForPositiveResource); err != nil {
+				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+			}
+		})
 	})
 
 	ginkgo.Context("When QAT resources are available with compress (dc) services enabled [Resource:dc]", func() {
@@ -164,6 +189,27 @@ func describeQatDpdkPlugin() {
 		ginkgo.When("there is no app to run [App:noapp]", func() {
 			ginkgo.It("does nothing", func() {})
 		})
+
+		ginkgo.It("checks auto-reset functionality [Functionality: auto-reset]", func(ctx context.Context) {
+			ginkgo.By("checking if heartbeat status is read correctly")
+			if err := setQatAutoReset("off"); err != nil {
+				framework.Failf("unable to set auto reset in this system: %v", err)
+			}
+			if err := injectError(); err != nil {
+				framework.Failf("unable to test error injection in this system: %v", err)
+			}
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+				framework.Failf("unable to wait for nodes to have zero resource: %v", err)
+			}
+
+			ginkgo.By("checking if auto-reset works to solve injected errors")
+			if err := setQatAutoReset("on"); err != nil {
+				framework.Failf("unable to set auto reset in this system: %v", err)
+			}
+			if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForPositiveResource); err != nil {
+				framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+			}
+		})
 	})
 }
 
@@ -199,3 +245,38 @@ func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int,
 	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
 	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
 }
+
+func setQatAutoReset(cfgVal string) error {
+	matches, err := filepath.Glob(autoResetPattern)
+	if err != nil {
+		return err
+	}
+
+	for _, filePath := range matches {
+		err = os.WriteFile(filePath, []byte(cfgVal), 0600)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func injectError() error {
+	matches, err := filepath.Glob(injectErrorPattern)
+	if err != nil {
+		return err
+	}
+
+	for _, filePath := range matches {
+		file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_TRUNC, 0600)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		os.Stdout = file
+		fmt.Println("1")
+	}
+
+	return nil
+}