add resource pool client chain element

pkg/networkservice/common/resourcepool/client.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2021 Nordix Foundation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resourcepool
+
+import (
+	"context"
+	"sync"
+
+	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/common"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/tools/log"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+)
+
+type resourcePoolClient struct {
+	resourcePool *resourcePoolConfig
+}
+
+// NewClient returns a new resource pool client chain element
+func NewClient(
+	driverType sriov.DriverType,
+	resourceLock sync.Locker,
+	pciPool PCIPool,
+	resourcePool ResourcePool,
+	cfg *config.Config,
+) networkservice.NetworkServiceClient {
+	return &resourcePoolClient{resourcePool: &resourcePoolConfig{
+		driverType:   driverType,
+		resourceLock: resourceLock,
+		pciPool:      pciPool,
+		resourcePool: resourcePool,
+		config:       cfg,
+		selectedVFs:  map[string]string{},
+	}}
+}
+
+func (i *resourcePoolClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
+	logger := log.FromContext(ctx).WithField("resourcePoolClient", "Request")
+
+	oldPCIAddress := request.GetConnection().GetMechanism().GetParameters()[common.PCIAddressKey]
+	oldTokenID := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey]
+
+	conn, err := next.Client(ctx).Request(ctx, request, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	tokenID, ok := conn.GetMechanism().GetParameters()[TokenIDKey]
+	if !ok {
+		logger.Infof("no token id present for endpoint connection %v", conn)
+		return conn, nil
+	}
+
+	err = assignVF(ctx, logger, conn, tokenID, i.resourcePool)
+	if err != nil {
+		_ = i.resourcePool.close(conn)
+		if _, closeErr := next.Client(ctx).Close(ctx, conn, opts...); closeErr != nil {
+			logger.Errorf("failed to close failed connection: %s %s", conn.GetId(), closeErr.Error())
+		}
+		return nil, err
+	}
+
+	// Don't make second request if PCI address, token id weren't changed
+	if conn.GetMechanism().GetParameters()[common.PCIAddressKey] == oldPCIAddress && oldTokenID == tokenID {
+		return conn, nil
+	}
+
+	// communicate assigned VF's pci address to endpoint by making another Request.
+	// this would also need subsequent chain elements to ignore handling of response
+	// for 2nd Request.
+	request.Connection = conn.Clone()
+	if conn, err = next.Client(ctx).Request(ctx, request); err != nil {
+		// Perform local cleanup in case of second Request failed
+		_ = i.resourcePool.close(request.Connection)
+	}
+
+	return conn, err
+}
+
+func (i *resourcePoolClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
+	rv, err := next.Client(ctx).Close(ctx, conn, opts...)
+	closeErr := i.resourcePool.close(conn)
+
+	if err != nil && closeErr != nil {
+		return nil, errors.Wrapf(err, "failed to free VF: %v", closeErr)
+	}
+	if closeErr != nil {
+		return nil, errors.Wrap(closeErr, "failed to free VF")
+	}
+	return rv, err
+}

pkg/networkservice/common/resourcepool/common.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2021 Nordix Foundation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package resourcepool
+
+import (
+	"context"
+	"sync"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/common"
+	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio"
+	"github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig"
+	"github.com/networkservicemesh/sdk/pkg/tools/log"
+	"github.com/pkg/errors"
+
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov"
+	"github.com/networkservicemesh/sdk-sriov/pkg/sriov/config"
+)
+
+const (
+	// TokenIDKey is a token ID mechanism parameter key
+	TokenIDKey = "tokenID" // TODO: move to api
+)
+
+// PCIPool is a pci.Pool interface
+type PCIPool interface {
+	GetPCIFunction(pciAddr string) (sriov.PCIFunction, error)
+	BindDriver(ctx context.Context, iommuGroup uint, driverType sriov.DriverType) error
+}
+
+// ResourcePool is a resource.Pool interface
+type ResourcePool interface {
+	Select(tokenID string, driverType sriov.DriverType) (string, error)
+	Free(vfPCIAddr string) error
+}
+
+type resourcePoolConfig struct {
+	driverType   sriov.DriverType
+	resourceLock sync.Locker
+	pciPool      PCIPool
+	resourcePool ResourcePool
+	config       *config.Config
+	selectedVFs  map[string]string
+}
+
+func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) {
+	vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType)
+	}
+	s.selectedVFs[connID] = vfPCIAddr
+
+	for pfPCIAddr, pfCfg := range s.config.PhysicalFunctions {
+		for i, vfCfg := range pfCfg.VirtualFunctions {
+			if vfCfg.Address != vfPCIAddr {
+				continue
+			}
+
+			pf, err := s.pciPool.GetPCIFunction(pfPCIAddr)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr)
+			}
+			vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName()
+			if err != nil {
+				return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr)
+			}
+
+			vf, err := s.pciPool.GetPCIFunction(vfPCIAddr)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr)
+			}
+
+			vfConfig.VFNum = i
+
+			return vf, err
+		}
+	}
+
+	return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID])
+}
+
+func (s *resourcePoolConfig) close(conn *networkservice.Connection) error {
+	vfPCIAddr, ok := s.selectedVFs[conn.GetId()]
+	if !ok {
+		return nil
+	}
+	delete(s.selectedVFs, conn.GetId())
+
+	s.resourceLock.Lock()
+	defer s.resourceLock.Unlock()
+
+	return s.resourcePool.Free(vfPCIAddr)
+}
+
+func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Connection, tokenID string, resourcePool *resourcePoolConfig) error {
+	vfConfig := vfconfig.Config(ctx)
+	resourcePool.resourceLock.Lock()
+	defer resourcePool.resourceLock.Unlock()
+
+	logger.Infof("trying to select VF for %v", resourcePool.driverType)
+	vf, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID)
+	if err != nil {
+		return err
+	}
+	logger.Infof("selected VF: %+v", vf)
+
+	iommuGroup, err := vf.GetIOMMUGroup()
+	if err != nil {
+		return errors.Wrapf(err, "failed to get VF IOMMU group: %v", vf.GetPCIAddress())
+	}
+
+	if err = resourcePool.pciPool.BindDriver(ctx, iommuGroup, resourcePool.driverType); err != nil {
+		return err
+	}
+
+	switch resourcePool.driverType {
+	case sriov.KernelDriver:
+		vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName()
+		if err != nil {
+			return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress())
+		}
+	case sriov.VFIOPCIDriver:
+		vfio.ToMechanism(conn.GetMechanism()).SetIommuGroup(iommuGroup)
+	}
+	conn.GetMechanism().GetParameters()[common.PCIAddressKey] = vf.GetPCIAddress()
+
+	return nil
+}