qat: 4xxx: split generic resource to compression and crypto

Signed-off-by: Mikko Ylinen <mikko.ylinen@intel.com>

Author: mythi

README.md

@@ -218,7 +218,7 @@ The summary of resources available via plugins in this repository is given in a
 | `fpga.intel.com` | custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)| [intelfpga-job.yaml](demo/intelfpga-job.yaml) |
 | `gpu.intel.com`  | `i915`                       | [intelgpu-job.yaml](demo/intelgpu-job.yaml) |
 | `iaa.intel.com`  | `wq-user-[shared or dedicated]` | [iaa-qpl-demo-pod.yaml](demo/iaa-qpl-demo-pod.yaml) |
-| `qat.intel.com`  | `generic`                    | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
+| `qat.intel.com`  | `generic` or `cy`/`dc`/`sym`/`asym` | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
 | `sgx.intel.com`  | `epc`                        | [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml) |
 | `vpu.intel.com`  | `hddl`                       | [intelvpu-job.yaml](demo/intelvpu-job.yaml) |
 

cmd/qat_plugin/dpdkdrv/dpdkdrv.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-ini/ini"
 	"github.com/pkg/errors"
 
 	"k8s.io/klog/v2"
@@ -51,6 +52,9 @@ const (
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
+
+	// Resource name to use when device capabilities are not available.
+	defaultCapabilities = "generic"
 )
 
 // QAT PCI VF Device ID -> kernel QAT VF device driver mappings.
@@ -352,6 +356,57 @@ func (dp *DevicePlugin) getDpdkMounts(dpdkDeviceName string) []pluginapi.Mount {
 	}
 }
 
+func getDeviceCapabilities(device string) (string, error) {
+	devID, err := getDeviceID(device)
+	if err != nil {
+		return "", errors.Wrapf(err, "cannot determine device capabilities")
+	}
+
+	devicesWithCapabilities := map[string]struct{}{
+		"4941": {}, // Check QAT Gen4 (4xxx) VF PCI ID only
+	}
+
+	if _, ok := devicesWithCapabilities[devID]; !ok {
+		return defaultCapabilities, nil
+	}
+
+	pfDev, err := filepath.EvalSymlinks(filepath.Join(device, "physfn"))
+	if err != nil {
+		klog.Warningf("failed to get PF device ID for %s: %q", filepath.Base(device), err)
+		return defaultCapabilities, nil
+	}
+
+	// TODO: check the sysfs state entry first when it lands.
+
+	lOpts := ini.LoadOptions{
+		IgnoreInlineComment: true,
+	}
+
+	devCfgPath := filepath.Join(filepath.Dir(filepath.Join(pfDev, "../../")), "kernel/debug",
+		fmt.Sprintf("qat_4xxx_%s/dev_cfg", filepath.Base(pfDev)))
+
+	devCfg, err := ini.LoadSources(lOpts, devCfgPath)
+	if err != nil {
+		klog.Warningf("failed to read dev_cfg for %s: %q", filepath.Base(pfDev), err)
+		return defaultCapabilities, nil
+	}
+
+	switch devCfg.Section("GENERAL").Key("ServicesEnabled").String() {
+	case "sym;asym":
+		return "cy", nil
+	case "asym;sym":
+		return "cy", nil
+	case "dc":
+		return "dc", nil
+	case "sym":
+		return "sym", nil
+	case "asym":
+		return "asym", nil
+	default:
+		return defaultCapabilities, nil
+	}
+}
+
 func getDeviceID(device string) (string, error) {
 	devID, err := os.ReadFile(filepath.Join(device, "device"))
 	if err != nil {
@@ -526,7 +581,12 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
 			return nil, err
 		}
 
-		klog.V(1).Infof("Device %s found", vfBdf)
+		cap, err := getDeviceCapabilities(vfDevice)
+		if err != nil {
+			return nil, err
+		}
+
+		klog.V(1).Infof("Device %s with %s capabilities found", vfBdf, cap)
 
 		n = n + 1
 		envs := map[string]string{
@@ -535,7 +595,7 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
 
 		devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs, nil)
 
-		devTree.AddDevice("generic", vfBdf, devinfo)
+		devTree.AddDevice(cap, vfBdf, devinfo)
 	}
 
 	return devTree, nil

cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Intel Corporation. All Rights Reserved.
+// Copyright 2018-2022 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -428,6 +428,69 @@ func TestScan(t *testing.T) {
 			maxDevNum:      1,
 			expectedDevNum: 1,
 		},
+		{
+			name:            "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), PF with dc capabilities",
+			dpdkDriver:      "vfio-pci",
+			kernelVfDrivers: []string{"4xxxvf"},
+			dirs: []string{
+				"sys/bus/pci/drivers/4xxx",
+				"sys/bus/pci/drivers/vfio-pci",
+				"sys/devices/pci0000:02/0000:02:00.0",
+				"sys/kernel/debug/qat_4xxx_0000:02:00.0",
+				"sys/bus/pci/devices/0000:02:01.0",
+			},
+			files: map[string][]byte{
+				"sys/devices/pci0000:02/0000:02:00.0/device":     []byte("0x4940"),
+				"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
+				"sys/bus/pci/devices/0000:02:01.0/device":        []byte("0x4941"),
+			},
+			symlinks: map[string]string{
+				"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
+				"sys/bus/pci/devices/0000:02:01.0/physfn":      "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/bus/pci/drivers/4xxx/0000:02:00.0":        "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/bus/pci/devices/0000:02:00.0":             "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/devices/pci0000:02/0000:02:00.0/virtfn0":  "sys/bus/pci/devices/0000:02:01.0",
+			},
+			maxDevNum:      1,
+			expectedDevNum: 1,
+		},
+		{
+			name:            "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), two PFs with dc and cy capabilities",
+			dpdkDriver:      "vfio-pci",
+			kernelVfDrivers: []string{"4xxxvf"},
+			dirs: []string{
+				"sys/bus/pci/drivers/4xxx",
+				"sys/bus/pci/drivers/vfio-pci",
+				"sys/devices/pci0000:02/0000:02:00.0",
+				"sys/devices/pci0000:03/0000:03:00.0",
+				"sys/kernel/debug/qat_4xxx_0000:02:00.0",
+				"sys/kernel/debug/qat_4xxx_0000:03:00.0",
+				"sys/bus/pci/devices/0000:02:01.0",
+				"sys/bus/pci/devices/0000:03:01.0",
+			},
+			files: map[string][]byte{
+				"sys/devices/pci0000:02/0000:02:00.0/device":     []byte("0x4940"),
+				"sys/devices/pci0000:03/0000:03:00.0/device":     []byte("0x4940"),
+				"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
+				"sys/kernel/debug/qat_4xxx_0000:03:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = sym;asym"),
+				"sys/bus/pci/devices/0000:02:01.0/device":        []byte("0x4941"),
+				"sys/bus/pci/devices/0000:03:01.0/device":        []byte("0x4941"),
+			},
+			symlinks: map[string]string{
+				"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
+				"sys/bus/pci/devices/0000:03:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile2",
+				"sys/bus/pci/devices/0000:02:01.0/physfn":      "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/bus/pci/devices/0000:03:01.0/physfn":      "sys/devices/pci0000:03/0000:03:00.0",
+				"sys/bus/pci/drivers/4xxx/0000:02:00.0":        "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/bus/pci/drivers/4xxx/0000:03:00.0":        "sys/devices/pci0000:03/0000:03:00.0",
+				"sys/bus/pci/devices/0000:02:00.0":             "sys/devices/pci0000:02/0000:02:00.0",
+				"sys/bus/pci/devices/0000:03:00.0":             "sys/devices/pci0000:03/0000:03:00.0",
+				"sys/devices/pci0000:02/0000:02:00.0/virtfn0":  "sys/bus/pci/devices/0000:02:01.0",
+				"sys/devices/pci0000:03/0000:03:00.0/virtfn0":  "sys/bus/pci/devices/0000:03:01.0",
+			},
+			maxDevNum:      2,
+			expectedDevNum: 2,
+		},
 	}
 	for _, tt := range tcases {
 		t.Run(tt.name, func(t *testing.T) {
@@ -461,8 +524,12 @@ func TestScan(t *testing.T) {
 			if !tt.expectedErr && err != nil {
 				t.Errorf("got unexpected error: %+v", err)
 			}
-			if len(fN.tree["generic"]) != tt.expectedDevNum {
-				t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, len(fN.tree["generic"]))
+			devNum := 0
+			for _, resource := range fN.tree {
+				devNum = devNum + len(resource)
+			}
+			if devNum != tt.expectedDevNum {
+				t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, devNum)
 			}
 
 			if err = os.RemoveAll(tmpdir); err != nil {

deployments/qat_plugin/base/intel-qat-plugin.yaml

@@ -24,6 +24,9 @@ spec:
         - name: devdir
           mountPath: /dev/vfio
           readOnly: true
+        - name: debugfsdir
+          mountPath: /sys/kernel/debug
+          readOnly: true
         - name: pcidir
           mountPath: /sys/bus/pci
         - name: kubeletsockets
@@ -32,6 +35,9 @@ spec:
       - name: devdir
         hostPath:
           path: /dev/vfio
+      - name: debugfsdir
+        hostPath:
+          path: /sys/kernel/debug
       - name: pcidir
         hostPath:
           path: /sys/bus/pci

pkg/controllers/qat/controller_test.go

@@ -81,6 +81,11 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 									MountPath: "/dev/vfio",
 									ReadOnly:  true,
 								},
+								{
+									Name:      "debugfsdir",
+									MountPath: "/sys/kernel/debug",
+									ReadOnly:  true,
+								},
 								{
 									Name:      "pcidir",
 									MountPath: "/sys/bus/pci",
@@ -102,6 +107,14 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 								},
 							},
 						},
+						{
+							Name: "debugfsdir",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/sys/kernel/debug",
+								},
+							},
+						},
 						{
 							Name: "pcidir",
 							VolumeSource: v1.VolumeSource{