From d73424a770ad9dc00b09ce6bc5832ebf2f84f220 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> Date: Tue, 18 May 2021 11:59:39 +0200 Subject: [PATCH 1/5] add resource pool client chain element Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> --- .../common/resourcepool/client.go | 90 +++++++++++ .../common/resourcepool/common.go | 140 ++++++++++++++++++ .../common/resourcepool/server.go | 129 ++-------------- 3 files changed, 243 insertions(+), 116 deletions(-) create mode 100644 pkg/networkservice/common/resourcepool/client.go create mode 100644 pkg/networkservice/common/resourcepool/common.go diff --git a/pkg/networkservice/common/resourcepool/client.go b/pkg/networkservice/common/resourcepool/client.go new file mode 100644 index 00000000..e6b0968f --- /dev/null +++ b/pkg/networkservice/common/resourcepool/client.go @@ -0,0 +1,90 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcepool + +import ( + "context" + "sync" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/sdk-sriov/pkg/sriov" + "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" +) + +type resourcePoolClient struct { + resourcePool *resourcePoolConfig +} + +// NewClient returns a new resource pool client chain element +func NewClient( + driverType sriov.DriverType, + resourceLock sync.Locker, + pciPool PCIPool, + resourcePool ResourcePool, + cfg *config.Config, +) networkservice.NetworkServiceClient { + return &resourcePoolClient{resourcePool: &resourcePoolConfig{ + driverType: driverType, + resourceLock: resourceLock, + pciPool: pciPool, + resourcePool: resourcePool, + config: cfg, + selectedVFs: map[string]string{}, + }} +} + +func (i *resourcePoolClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + logger := log.FromContext(ctx).WithField("resourcePoolClient", "Request") + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + tokenID, ok := conn.GetMechanism().GetParameters()[TokenIDKey] + if !ok { + logger.Infof("no token id present for endpoint connection %v", conn) + return conn, nil + } + + err = assignVF(ctx, logger, conn, tokenID, i.resourcePool) + if err != nil { + _ = i.resourcePool.close(conn) + return nil, err + } + + return conn, nil +} + +func (i *resourcePoolClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + rv, err := next.Client(ctx).Close(ctx, conn, opts...) + closeErr := i.resourcePool.close(conn) + + if err != nil && closeErr != nil { + return nil, errors.Wrapf(err, "failed to free VF: %v", closeErr) + } + if closeErr != nil { + return nil, errors.Wrap(closeErr, "failed to free VF") + } + return rv, err +} diff --git a/pkg/networkservice/common/resourcepool/common.go b/pkg/networkservice/common/resourcepool/common.go new file mode 100644 index 00000000..89ee8b07 --- /dev/null +++ b/pkg/networkservice/common/resourcepool/common.go @@ -0,0 +1,140 @@ +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resourcepool + +import ( + "context" + "sync" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio" + "github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/pkg/errors" + + "github.com/networkservicemesh/sdk-sriov/pkg/sriov" + "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" +) + +const ( + // TokenIDKey is a token ID mechanism parameter key + TokenIDKey = "tokenID" // TODO: move to api +) + +// PCIPool is a pci.Pool interface +type PCIPool interface { + GetPCIFunction(pciAddr string) (sriov.PCIFunction, error) + BindDriver(ctx context.Context, iommuGroup uint, driverType sriov.DriverType) error +} + +// ResourcePool is a resource.Pool interface +type ResourcePool interface { + Select(tokenID string, driverType sriov.DriverType) (string, error) + Free(vfPCIAddr string) error +} + +type resourcePoolConfig struct { + driverType sriov.DriverType + resourceLock sync.Locker + pciPool PCIPool + resourcePool ResourcePool + config *config.Config + selectedVFs map[string]string +} + +func (s *resourcePoolConfig) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) { + vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType) + if err != nil { + return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType) + } + s.selectedVFs[connID] = vfPCIAddr + + for pfPCIAddr, pfCfg := range s.config.PhysicalFunctions { + for i, vfCfg := range pfCfg.VirtualFunctions { + if vfCfg.Address != vfPCIAddr { + continue + } + + pf, err := s.pciPool.GetPCIFunction(pfPCIAddr) + if err != nil { + return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr) + } + vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName() + if err != nil { + return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr) + } + + vf, err := s.pciPool.GetPCIFunction(vfPCIAddr) + if err != nil { + return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr) + } + + vfConfig.VFNum = i + + return vf, err + } + } + + return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID]) +} + +func (s *resourcePoolConfig) close(conn *networkservice.Connection) error { + vfPCIAddr, ok := s.selectedVFs[conn.GetId()] + if !ok { + return nil + } + delete(s.selectedVFs, conn.GetId()) + + s.resourceLock.Lock() + defer s.resourceLock.Unlock() + + return s.resourcePool.Free(vfPCIAddr) +} + +func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Connection, tokenID string, resourcePool *resourcePoolConfig) error { + vfConfig := vfconfig.Config(ctx) + resourcePool.resourceLock.Lock() + defer resourcePool.resourceLock.Unlock() + + logger.Infof("trying to select VF for %v", resourcePool.driverType) + vf, err := resourcePool.selectVF(conn.GetId(), vfConfig, tokenID) + if err != nil { + return err + } + logger.Infof("selected VF: %+v", vf) + + iommuGroup, err := vf.GetIOMMUGroup() + if err != nil { + return errors.Wrapf(err, "failed to get VF IOMMU group: %v", vf.GetPCIAddress()) + } + + if err = resourcePool.pciPool.BindDriver(ctx, iommuGroup, resourcePool.driverType); err != nil { + return err + } + + switch resourcePool.driverType { + case sriov.KernelDriver: + vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName() + if err != nil { + return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress()) + } + case sriov.VFIOPCIDriver: + vfio.ToMechanism(conn.GetMechanism()).SetIommuGroup(iommuGroup) + } + + return nil +} diff --git a/pkg/networkservice/common/resourcepool/server.go b/pkg/networkservice/common/resourcepool/server.go index 9780a367..0180f1ca 100644 --- a/pkg/networkservice/common/resourcepool/server.go +++ b/pkg/networkservice/common/resourcepool/server.go @@ -25,8 +25,6 @@ import ( "github.com/pkg/errors" "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio" - "github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "github.com/networkservicemesh/sdk/pkg/tools/log" @@ -34,30 +32,8 @@ import ( "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" ) -const ( - // TokenIDKey is a token ID mechanism parameter key - TokenIDKey = "tokenID" // TODO: move to api -) - -// PCIPool is a pci.Pool interface -type PCIPool interface { - GetPCIFunction(pciAddr string) (sriov.PCIFunction, error) - BindDriver(ctx context.Context, iommuGroup uint, driverType sriov.DriverType) error -} - -// ResourcePool is a resource.Pool interface -type ResourcePool interface { - Select(tokenID string, driverType sriov.DriverType) (string, error) - Free(vfPCIAddr string) error -} - type resourcePoolServer struct { - driverType sriov.DriverType - resourceLock sync.Locker - pciPool PCIPool - resourcePool ResourcePool - config *config.Config - selectedVFs map[string]string + resourcePool *resourcePoolConfig } // NewServer returns a new resource pool server chain element @@ -68,109 +44,43 @@ func NewServer( resourcePool ResourcePool, cfg *config.Config, ) networkservice.NetworkServiceServer { - return &resourcePoolServer{ + return &resourcePoolServer{resourcePool: &resourcePoolConfig{ driverType: driverType, resourceLock: resourceLock, pciPool: pciPool, resourcePool: resourcePool, config: cfg, selectedVFs: map[string]string{}, - } + }} } func (s *resourcePoolServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { logger := log.FromContext(ctx).WithField("resourcePoolServer", "Request") - - tokenID, ok := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey] + conn := request.GetConnection() + tokenID, ok := conn.GetMechanism().GetParameters()[TokenIDKey] if !ok { - return nil, errors.New("no token ID provided") + logger.Infof("no token id present for client connection %v", conn) + return next.Server(ctx).Request(ctx, request) } - vfConfig := vfconfig.Config(ctx) - if err := func() error { - s.resourceLock.Lock() - defer s.resourceLock.Unlock() - - logger.Infof("trying to select VF for %v", s.driverType) - vf, err := s.selectVF(request.GetConnection().GetId(), vfConfig, tokenID) - if err != nil { - return err - } - logger.Infof("selected VF: %+v", vf) - - iommuGroup, err := vf.GetIOMMUGroup() - if err != nil { - return errors.Wrapf(err, "failed to get VF IOMMU group: %v", vf.GetPCIAddress()) - } - - if err = s.pciPool.BindDriver(ctx, iommuGroup, s.driverType); err != nil { - return err - } - - switch s.driverType { - case sriov.KernelDriver: - vfConfig.VFInterfaceName, err = vf.GetNetInterfaceName() - if err != nil { - return errors.Wrapf(err, "failed to get VF net interface name: %v", vf.GetPCIAddress()) - } - case sriov.VFIOPCIDriver: - vfio.ToMechanism(request.GetConnection().GetMechanism()).SetIommuGroup(iommuGroup) - } - - return nil - }(); err != nil { - _ = s.close(request.GetConnection()) + err := assignVF(ctx, logger, conn, tokenID, s.resourcePool) + if err != nil { + _ = s.resourcePool.close(conn) return nil, err } - conn, err := next.Server(ctx).Request(ctx, request) + conn, err = next.Server(ctx).Request(ctx, request) if err != nil { - _ = s.close(request.GetConnection()) + _ = s.resourcePool.close(request.GetConnection()) } return conn, err } -func (s *resourcePoolServer) selectVF(connID string, vfConfig *vfconfig.VFConfig, tokenID string) (vf sriov.PCIFunction, err error) { - vfPCIAddr, err := s.resourcePool.Select(tokenID, s.driverType) - if err != nil { - return nil, errors.Wrapf(err, "failed to select VF for: %v", s.driverType) - } - s.selectedVFs[connID] = vfPCIAddr - - for pfPCIAddr, pfCfg := range s.config.PhysicalFunctions { - for i, vfCfg := range pfCfg.VirtualFunctions { - if vfCfg.Address != vfPCIAddr { - continue - } - - pf, err := s.pciPool.GetPCIFunction(pfPCIAddr) - if err != nil { - return nil, errors.Wrapf(err, "failed to get PF: %v", pfPCIAddr) - } - vfConfig.PFInterfaceName, err = pf.GetNetInterfaceName() - if err != nil { - return nil, errors.Errorf("failed to get PF net interface name: %v", pfPCIAddr) - } - - vf, err := s.pciPool.GetPCIFunction(vfPCIAddr) - if err != nil { - return nil, errors.Wrapf(err, "failed to get VF: %v", vfPCIAddr) - } - - vfConfig.VFNum = i - - return vf, err - } - } - - return nil, errors.Errorf("no VF with selected PCI address exists: %v", s.selectedVFs[connID]) -} - func (s *resourcePoolServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { _, err := next.Server(ctx).Close(ctx, conn) - closeErr := s.close(conn) + closeErr := s.resourcePool.close(conn) if err != nil && closeErr != nil { return nil, errors.Wrapf(err, "failed to free VF: %v", closeErr) @@ -180,16 +90,3 @@ func (s *resourcePoolServer) Close(ctx context.Context, conn *networkservice.Con } return &empty.Empty{}, err } - -func (s *resourcePoolServer) close(conn *networkservice.Connection) error { - vfPCIAddr, ok := s.selectedVFs[conn.GetId()] - if !ok { - return nil - } - delete(s.selectedVFs, conn.GetId()) - - s.resourceLock.Lock() - defer s.resourceLock.Unlock() - - return s.resourcePool.Free(vfPCIAddr) -} From d74674305432acdd7bfd8395b6c7c5378e313905 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> Date: Thu, 29 Jul 2021 10:29:36 +0200 Subject: [PATCH 2/5] set pci address into mechanism parameter Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> --- pkg/networkservice/common/resourcepool/client.go | 3 +++ pkg/networkservice/common/resourcepool/common.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/pkg/networkservice/common/resourcepool/client.go b/pkg/networkservice/common/resourcepool/client.go index e6b0968f..07a4755e 100644 --- a/pkg/networkservice/common/resourcepool/client.go +++ b/pkg/networkservice/common/resourcepool/client.go @@ -70,6 +70,9 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic err = assignVF(ctx, logger, conn, tokenID, i.resourcePool) if err != nil { _ = i.resourcePool.close(conn) + if _, closeErr := next.Client(ctx).Close(ctx, conn, opts...); closeErr != nil { + logger.Errorf("failed to close failed connection: %s %s", conn.GetId(), closeErr.Error()) + } return nil, err } diff --git a/pkg/networkservice/common/resourcepool/common.go b/pkg/networkservice/common/resourcepool/common.go index 89ee8b07..462b8230 100644 --- a/pkg/networkservice/common/resourcepool/common.go +++ b/pkg/networkservice/common/resourcepool/common.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/common" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/vfio" "github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig" "github.com/networkservicemesh/sdk/pkg/tools/log" @@ -135,6 +136,7 @@ func assignVF(ctx context.Context, logger log.Logger, conn *networkservice.Conne case sriov.VFIOPCIDriver: vfio.ToMechanism(conn.GetMechanism()).SetIommuGroup(iommuGroup) } + conn.GetMechanism().GetParameters()[common.PCIAddressKey] = vf.GetPCIAddress() return nil } From 1f3d1c96eedec9105c5c6e9dc2e88890a66903f8 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> Date: Thu, 29 Jul 2021 12:07:52 +0200 Subject: [PATCH 3/5] communicate assigned vf pci address to endpoint Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> --- pkg/networkservice/common/resourcepool/client.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/networkservice/common/resourcepool/client.go b/pkg/networkservice/common/resourcepool/client.go index 07a4755e..ef7202b0 100644 --- a/pkg/networkservice/common/resourcepool/client.go +++ b/pkg/networkservice/common/resourcepool/client.go @@ -76,6 +76,18 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic return nil, err } + // communicate assigned VF's pci address to endpoint by making another Request and ignore + // returned connection. this would also need subsequent chain elements to ignore + // handling of response for 2nd Request. + request.Connection = conn.Clone() + if _, err = next.Client(ctx).Request(ctx, request); err != nil { + // Perform local cleanup in case of second Request failed + _ = i.resourcePool.close(conn) + if _, closeErr := next.Client(ctx).Close(ctx, conn, opts...); closeErr != nil { + logger.Errorf("failed to close failed connection: %s %s", conn.GetId(), closeErr.Error()) + } + } + return conn, nil } From 046599550e44061bb09c53d1411ef5eabf8652f9 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> Date: Thu, 29 Jul 2021 15:20:08 +0200 Subject: [PATCH 4/5] make appropriate second request for sending pci address Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> --- .../common/resourcepool/client.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/networkservice/common/resourcepool/client.go b/pkg/networkservice/common/resourcepool/client.go index ef7202b0..212fe0e9 100644 --- a/pkg/networkservice/common/resourcepool/client.go +++ b/pkg/networkservice/common/resourcepool/client.go @@ -22,6 +22,7 @@ import ( "github.com/golang/protobuf/ptypes/empty" "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/common" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/pkg/errors" @@ -56,6 +57,9 @@ func NewClient( func (i *resourcePoolClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { logger := log.FromContext(ctx).WithField("resourcePoolClient", "Request") + oldPCIAddress := request.GetConnection().GetMechanism().GetParameters()[common.PCIAddressKey] + oldTokenIDKey := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey] + conn, err := next.Client(ctx).Request(ctx, request, opts...) if err != nil { return nil, err @@ -76,13 +80,18 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic return nil, err } - // communicate assigned VF's pci address to endpoint by making another Request and ignore - // returned connection. this would also need subsequent chain elements to ignore - // handling of response for 2nd Request. + // Don't make second request if PCI address, token id weren't changed + if conn.GetMechanism().GetParameters()[common.PCIAddressKey] == oldPCIAddress && oldTokenIDKey == tokenID { + return conn, nil + } + + // communicate assigned VF's pci address to endpoint by making another Request. + // this would also need subsequent chain elements to ignore handling of response + // for 2nd Request. request.Connection = conn.Clone() - if _, err = next.Client(ctx).Request(ctx, request); err != nil { + if conn, err = next.Client(ctx).Request(ctx, request); err != nil { // Perform local cleanup in case of second Request failed - _ = i.resourcePool.close(conn) + _ = i.resourcePool.close(request.Connection) if _, closeErr := next.Client(ctx).Close(ctx, conn, opts...); closeErr != nil { logger.Errorf("failed to close failed connection: %s %s", conn.GetId(), closeErr.Error()) } From 66634f482dec05bc8c3e4f36d5a107b2fc290db3 Mon Sep 17 00:00:00 2001 From: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> Date: Thu, 29 Jul 2021 17:05:07 +0200 Subject: [PATCH 5/5] fix review comments Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@est.tech> --- pkg/networkservice/common/resourcepool/client.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/networkservice/common/resourcepool/client.go b/pkg/networkservice/common/resourcepool/client.go index 212fe0e9..fcedb129 100644 --- a/pkg/networkservice/common/resourcepool/client.go +++ b/pkg/networkservice/common/resourcepool/client.go @@ -58,7 +58,7 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic logger := log.FromContext(ctx).WithField("resourcePoolClient", "Request") oldPCIAddress := request.GetConnection().GetMechanism().GetParameters()[common.PCIAddressKey] - oldTokenIDKey := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey] + oldTokenID := request.GetConnection().GetMechanism().GetParameters()[TokenIDKey] conn, err := next.Client(ctx).Request(ctx, request, opts...) if err != nil { @@ -81,7 +81,7 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic } // Don't make second request if PCI address, token id weren't changed - if conn.GetMechanism().GetParameters()[common.PCIAddressKey] == oldPCIAddress && oldTokenIDKey == tokenID { + if conn.GetMechanism().GetParameters()[common.PCIAddressKey] == oldPCIAddress && oldTokenID == tokenID { return conn, nil } @@ -92,12 +92,9 @@ func (i *resourcePoolClient) Request(ctx context.Context, request *networkservic if conn, err = next.Client(ctx).Request(ctx, request); err != nil { // Perform local cleanup in case of second Request failed _ = i.resourcePool.close(request.Connection) - if _, closeErr := next.Client(ctx).Close(ctx, conn, opts...); closeErr != nil { - logger.Errorf("failed to close failed connection: %s %s", conn.GetId(), closeErr.Error()) - } } - return conn, nil + return conn, err } func (i *resourcePoolClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {