gpu: refactor nfdhook functionality to plugin

NFD v0.14+ doesn't support binary NFD hooks by default, so there is
a need to move the label creation away from the GPU nfdhook.

Move extended resource label creation to plugin, and drop labels that were
already marked deprecated (platform_gen, media_version etc.).

Drop init-container from deployment files and operator. It is still possible
to use an initcontainer, but the default deployments do not support it.

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/gpu_nfdhook/README.md

@@ -11,6 +11,8 @@ Table of Contents
 
 ## Introduction
 
+***NOTE:*** NFD's binary hook support will be turned off by default in the 0.14 release. The functionality in the GPU NFD hook is moved into a new NFD rule and into GPU plugin, and the capability labels are being removed completely. The GPU plugin deployment doesn't anymore support using init container. This directory will be removed in the future.
+
 This is the [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery)
 binary hook implementation for the Intel GPUs. The intel-gpu-initcontainer (which
 is built with the other images) can be used as part of the gpu-plugin deployment

cmd/gpu_nfdhook/main.go

@@ -15,25 +15,14 @@
 package main
 
 import (
-	"os"
-
-	"k8s.io/klog/v2"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 )
 
 const (
-	sysfsDirectory      = "/host-sys"
-	sysfsDRMDirectory   = sysfsDirectory + "/class/drm"
-	debugfsDRIDirectory = sysfsDirectory + "/kernel/debug/dri"
+	sysfsDirectory    = "/host-sys"
+	sysfsDRMDirectory = sysfsDirectory + "/class/drm"
 )
 
 func main() {
-	l := newLabeler(sysfsDRMDirectory, debugfsDRIDirectory)
-
-	err := l.createLabels()
-	if err != nil {
-		klog.Errorf("%+v", err)
-		os.Exit(1)
-	}
-
-	l.printLabels()
+	labeler.CreateAndPrintLabels(sysfsDRMDirectory)
 }

cmd/gpu_plugin/gpu_plugin.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2022 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,13 +31,16 @@ import (
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
 
 const (
 	sysfsDrmDirectory = "/sys/class/drm"
 	devfsDriDirectory = "/dev/dri"
+	nfdFeatureDir     = "/etc/kubernetes/node-feature-discovery/features.d"
+	resourceFilename  = "intel-gpu-resources.txt"
 	gpuDeviceRE       = `^card[0-9]+$`
 	controlDeviceRE   = `^controlD[0-9]+$`
 	pciAddressRE      = "^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9a-f]{1}$"
@@ -53,6 +56,9 @@ const (
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
+
+	// Labeler's max update interval, 5min.
+	labelerMaxInterval = 5 * 60 * time.Second
 )
 
 type cliOptions struct {
@@ -242,8 +248,9 @@ type devicePlugin struct {
 	controlDeviceReg *regexp.Regexp
 	pciAddressReg    *regexp.Regexp
 
-	scanTicker *time.Ticker
-	scanDone   chan bool
+	scanTicker    *time.Ticker
+	scanDone      chan bool
+	scanResources chan bool
 
 	resMan rm.ResourceManager
 
@@ -270,6 +277,7 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi
 		scanTicker:       time.NewTicker(scanPeriod),
 		scanDone:         make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
 		bypathFound:      true,
+		scanResources:    make(chan bool, 1),
 	}
 
 	if options.resourceManagement {
@@ -347,17 +355,26 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 			klog.Warning("Failed to scan: ", err)
 		}
 
+		countChanged := false
+
 		for name, prev := range previousCount {
 			count := devTree.DeviceTypeCount(name)
 			if count != prev {
 				klog.V(1).Infof("GPU scan update: %d->%d '%s' resources found", prev, count, name)
 
 				previousCount[name] = count
+
+				countChanged = true
 			}
 		}
 
 		notifier.Notify(devTree)
 
+		// Trigger resource scan if it's enabled.
+		if dp.resMan != nil && countChanged {
+			dp.scanResources <- true
+		}
+
 		select {
 		case <-dp.scanDone:
 			return nil
@@ -515,6 +532,18 @@ func main() {
 	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
 
 	plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts)
+
+	if plugin.options.resourceManagement {
+		// Start labeler to export labels file for NFD.
+		nfdFeatureFile := path.Join(nfdFeatureDir, resourceFilename)
+
+		klog.V(2).Infof("NFD feature file location: %s", nfdFeatureFile)
+
+		// Labeler catches OS signals and calls os.Exit() after receiving any.
+		go labeler.Run(prefix+sysfsDrmDirectory, nfdFeatureFile,
+			labelerMaxInterval, plugin.scanResources)
+	}
+
 	manager := dpapi.NewManager(namespace, plugin)
 	manager.Run()
 }

cmd/gpu_plugin/gpu_plugin_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/render-device.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2021 Intel Corporation.
+# Copyright 2021-2023 Intel Corporation.
 #
 # SPDX-License-Identifier: Apache-2.0
 #

cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.