qat,e2e: add heartbeat and auto-reset validations

Signed-off-by: Hyeongju Johannes Lee <[email protected]>

Author: hj-johannes-lee

demo/qat-auto-reset/Dockerfile

@@ -0,0 +1,4 @@
+FROM busybox
+COPY qat-auto-reset-test.sh /usr/bin/
+
+CMD ["/usr/bin/qat-auto-reset-test.sh"]

demo/qat-auto-reset/qat-auto-reset-test.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942 4944 4946}
+DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done)
+
+set_auto_reset() {
+  for dev in $DEVS; do
+    auto_reset_path="/sys/bus/pci/devices/0000:$dev/qat/auto_reset"
+    echo "Setting $dev\'s auto-reset $1" # on/off
+    echo $1 > "$auto_reset_path"
+  done
+}
+
+inject_error() {
+  inject_error_paths=$(find /sys/kernel/debug/qat_*/heartbeat/ -name inject_error)
+
+  for path in $inject_error_paths; do
+    echo "Injecting error into: $path"
+    echo 1 > $path
+  done
+}
+
+case "$1" in
+  inject_error)
+    inject_error
+    ;;
+  set_auto_reset)
+    if [ -z "$2" ]; then
+      echo "Usage: $0 set_auto_reset <on|off>"
+      exit 1
+    fi
+    set_auto_reset "$2"
+    ;;
+  *)
+    echo "Usage: $0 {inject_error|set_auto_reset <on|off>}"
+    exit 1
+    ;;
+esac

test/e2e/qat/qatplugin_dpdk.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/test/e2e/utils"
@@ -137,6 +138,10 @@ func describeQatDpdkPlugin() {
 		ginkgo.When("there is no app to run [App:noapp]", func() {
 			ginkgo.It("does nothing", func() {})
 		})
+
+		ginkgo.It("checks heartbeat detection and auto-reset functionalities [Functionality:auto-reset]", func(ctx context.Context) {
+			checkAutoResetFunctionality(ctx, f, symmetric, resourceName)
+		})
 	})
 
 	ginkgo.Context("When QAT resources are available with compress (dc) services enabled [Resource:dc]", func() {
@@ -164,6 +169,10 @@ func describeQatDpdkPlugin() {
 		ginkgo.When("there is no app to run [App:noapp]", func() {
 			ginkgo.It("does nothing", func() {})
 		})
+
+		ginkgo.It("checks heartbeat detection and auto-reset functionalities [Functionality:auto-reset]", func(ctx context.Context) {
+			checkAutoResetFunctionality(ctx, f, compression, resourceName)
+		})
 	})
 }
 
@@ -199,3 +208,72 @@ func runCpaSampleCode(ctx context.Context, f *framework.Framework, runTests int,
 	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
 	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
 }
+
+func checkAutoResetFunctionality(ctx context.Context, f *framework.Framework, runTests int, resourceName v1.ResourceName) {
+	ginkgo.By("checking if heartbeat status is read correctly")
+	setQatAutoReset("off", ctx, f)
+	injectError(ctx, f)
+	if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 100*time.Second, utils.WaitForZeroResource); err != nil {
+		framework.Failf("unable to wait for nodes to have zero resource: %v", err)
+	}
+
+	ginkgo.By("checking if auto-reset works to solve injected errors")
+	setQatAutoReset("on", ctx, f)
+	if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, resourceName, 300*time.Second, utils.WaitForPositiveResource); err != nil {
+		framework.Failf("unable to wait for nodes to have positive allocatable resource: %v", err)
+	}
+
+	ginkgo.By("checking if deployed pod runs successfully")
+	runCpaSampleCode(ctx, f, runTests, resourceName)
+}
+
+func setQatAutoReset(status string, ctx context.Context, f *framework.Framework) {
+	runQatAutoResetPod([]string{"set_auto_reset", status}, ctx, f)
+}
+
+func injectError(ctx context.Context, f *framework.Framework) {
+	runQatAutoResetPod([]string{"inject_error"}, ctx, f)
+}
+
+func runQatAutoResetPod(command []string, ctx context.Context, f *framework.Framework) {
+	ginkgo.By("submitting a pod that sets auto-reset function of QAT devices")
+	yes := true
+	podSpec := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: "qat-auto-reset-" + strings.ReplaceAll(strings.Join(command, "-"), "_", "-")},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:            "qat-auto-reset",
+					Image:           "intel/qat-auto-reset:devel",
+					ImagePullPolicy: "IfNotPresent",
+					Command:         append([]string{"qat-auto-reset-test.sh"}, command...),
+					SecurityContext: &v1.SecurityContext{
+						Privileged: &yes,
+					},
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "debugfs",
+							MountPath: "/sys/kernel/debug/",
+						},
+					},
+				},
+			},
+			Volumes: []v1.Volume{
+				{
+					Name: "debugfs",
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: "/sys/kernel/debug/",
+						},
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+		},
+	}
+	pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, podSpec, metav1.CreateOptions{})
+	framework.ExpectNoError(err, "pod Create API error")
+
+	err = e2epod.WaitForPodSuccessInNamespaceTimeout(ctx, f.ClientSet, pod.ObjectMeta.Name, f.Namespace.Name, 300*time.Second)
+	gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, pod.Spec.Containers[0].Name))
+}