Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: use wi for e2e test #1556

Closed
wants to merge 3 commits into from
Closed

test: use wi for e2e test #1556

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7deb843
test: use wi for e2e test
cvvz Aug 19, 2024
870dc9c
bypass e2etest
cvvz Aug 19, 2024
d15338f
fix e2e
umagnus Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ require (
github.com/onsi/ginkgo/v2 v2.19.1
github.com/onsi/gomega v1.34.1
github.com/pborman/uuid v1.2.1
github.com/pelletier/go-toml v1.9.5
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0
github.com/stretchr/testify v1.9.0
Expand Down
2 changes: 0 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,8 +220,6 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL
github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand Down
3 changes: 3 additions & 0 deletions hack/verify-examples.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ rollout_and_wait() {
}

echo "begin to create deployment examples ..."

kubectl config set-context --current --namespace=default

if [ -v EXTERNAL_E2E_TEST_BLOBFUSE_v2 ]; then
echo "create blobfuse2 storage class ..."
kubectl apply -f deploy/example/storageclass-blobfuse2.yaml
Expand Down
4 changes: 4 additions & 0 deletions pkg/util/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,10 @@ func parseAzcopyJobShow(jobshow string) (AzcopyJobState, string, error) {
func GetKubeClient(kubeconfig string, kubeAPIQPS float64, kubeAPIBurst int, userAgent string) (kubernetes.Interface, error) {
var err error
var kubeCfg *rest.Config
if kubeconfig == "no-need-kubeconfig" {
klog.V(2).Infof("kubeconfig is set as no-need-kubeconfig, kubeClient will be nil")
return nil, nil
}
if kubeCfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
return nil, err
}
Expand Down
24 changes: 18 additions & 6 deletions test/e2e/dynamic_provisioning_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -965,6 +965,8 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
"skuName": "Premium_LRS",
"protocol": "nfs",
"mountPermissions": "0755",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
}
test.Run(ctx, cs, ns)
Expand Down Expand Up @@ -997,6 +999,8 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
"skuName": "Premium_LRS",
"protocol": "nfs",
"mountPermissions": "0755",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
}
test.Run(ctx, cs, ns)
Expand Down Expand Up @@ -1027,8 +1031,10 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
Pod: pod,
PodWithClonedVolume: podWithClonedVolume,
StorageClassParameters: map[string]string{
"skuName": "Standard_LRS",
"protocol": "fuse2",
"skuName": "Standard_LRS",
"protocol": "fuse2",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
}
test.Run(ctx, cs, ns)
Expand Down Expand Up @@ -1059,8 +1065,10 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
Pod: pod,
PodWithClonedVolume: podWithClonedVolume,
StorageClassParameters: map[string]string{
"skuName": "Standard_LRS",
"protocol": "fuse2",
"skuName": "Standard_LRS",
"protocol": "fuse2",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
}
test.Run(ctx, cs, ns)
Expand Down Expand Up @@ -1094,6 +1102,8 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
"protocol": "nfs",
"mountPermissions": "0755",
"allowsharedkeyaccess": "true",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
ClonedStorageClassParameters: map[string]string{
"skuName": "Standard_LRS",
Expand Down Expand Up @@ -1130,8 +1140,10 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
Pod: pod,
PodWithClonedVolume: podWithClonedVolume,
StorageClassParameters: map[string]string{
"skuName": "Standard_LRS",
"protocol": "fuse2",
"skuName": "Standard_LRS",
"protocol": "fuse2",
"secretNamespace": "kube-system",
"secretName": fmt.Sprintf("secret-%d", time.Now().Unix()),
},
ClonedStorageClassParameters: map[string]string{
"skuName": "Premium_LRS",
Expand Down
36 changes: 2 additions & 34 deletions test/e2e/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package e2e

import (
"context"
"encoding/json"
"flag"
"fmt"
"log"
Expand All @@ -42,7 +41,6 @@ import (
"sigs.k8s.io/blob-csi-driver/pkg/util"
"sigs.k8s.io/blob-csi-driver/test/utils/azure"
"sigs.k8s.io/blob-csi-driver/test/utils/credentials"
"sigs.k8s.io/blob-csi-driver/test/utils/testutil"
)

const (
Expand Down Expand Up @@ -90,24 +88,11 @@ func TestE2E(t *testing.T) {
var _ = ginkgo.SynchronizedBeforeSuite(func(ctx ginkgo.SpecContext) []byte {
creds, err := credentials.CreateAzureCredentialFile()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
_, err = azureClient.EnsureResourceGroup(ctx, creds.ResourceGroup, creds.Location, nil)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

if testutil.IsRunningInProw() {
// Need to login to ACR using SP credential if we are running in Prow so we can push test images.
// If running locally, user should run 'docker login' before running E2E tests
registry := os.Getenv("REGISTRY")
gomega.Expect(registry).NotTo(gomega.Equal(""))

log.Println("Attempting docker login with Azure service principal")
cmd := exec.Command("docker", "login", fmt.Sprintf("--username=%s", creds.AADClientID), fmt.Sprintf("--password=%s", creds.AADClientSecret), registry)
err = cmd.Run()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
log.Println("docker login is successful")
}

// Install Azure Blob Storage CSI driver on cluster from project root
e2eBootstrap := testCmd{
command: "make",
Expand All @@ -123,25 +108,8 @@ var _ = ginkgo.SynchronizedBeforeSuite(func(ctx ginkgo.SpecContext) []byte {
endLog: "metrics service created",
}
execTestCmd([]testCmd{e2eBootstrap, createMetricsSVC})

if testutil.IsRunningInProw() {
data, err := json.Marshal(creds)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
return data
}

return nil
}, func(ctx ginkgo.SpecContext, data []byte) {
if testutil.IsRunningInProw() {
creds := &credentials.Credentials{}
err := json.Unmarshal(data, creds)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
// set env for azidentity.EnvironmentCredential
os.Setenv("AZURE_TENANT_ID", creds.TenantID)
os.Setenv("AZURE_CLIENT_ID", creds.AADClientID)
os.Setenv("AZURE_CLIENT_SECRET", creds.AADClientSecret)
}

// k8s.io/kubernetes/test/e2e/framework requires env KUBECONFIG to be set
// it does not fall back to defaults
if os.Getenv(kubeconfigEnvVar) == "" {
Expand Down Expand Up @@ -230,7 +198,7 @@ func execTestCmd(cmds []testCmd) {
func checkAccountCreationLeak(ctx context.Context) {
creds, err := credentials.CreateAzureCredentialFile()
gomega.Expect(err).NotTo(gomega.HaveOccurred())
azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

accountNum, err := azureClient.GetAccountNumByResourceGroup(ctx, creds.ResourceGroup)
Expand Down
137 changes: 69 additions & 68 deletions test/e2e/testsuites/pre_provisioned_provided_credentials_tester.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ package testsuites
import (
"context"
"fmt"
"strings"

"github.com/onsi/ginkgo/v2"

"sigs.k8s.io/blob-csi-driver/pkg/blob"
"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
"sigs.k8s.io/blob-csi-driver/test/utils/azure"

//"sigs.k8s.io/blob-csi-driver/test/utils/azure"

v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
Expand All @@ -41,11 +41,11 @@ type PreProvisionedProvidedCredentiasTest struct {
}

func (t *PreProvisionedProvidedCredentiasTest) Run(ctx context.Context, client clientset.Interface, namespace *v1.Namespace) {
kvClient, err := azure.NewKeyVaultClient()
framework.ExpectNoError(err)
//kvClient, err := azure.NewKeyVaultClient()
//framework.ExpectNoError(err)

authClient, err := azure.NewAuthorizationClient()
framework.ExpectNoError(err)
//authClient, err := azure.NewAuthorizationClient()
//framework.ExpectNoError(err)

for _, pod := range t.Pods {
for n, volume := range pod.Volumes {
Expand Down Expand Up @@ -98,79 +98,80 @@ func (t *PreProvisionedProvidedCredentiasTest) Run(ctx context.Context, client c
"azurestorageaccountsastoken": sasToken,
}
run()
/*
// test for service principal
ginkgo.By("Run for service principal")
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "SPN",
"azurestoragespnclientid": kvClient.Cred.AADClientID,
"azurestoragespntenantid": kvClient.Cred.TenantID,
}
secretData = map[string]string{
"azurestorageaccountname": accountName,
"azurestoragespnclientsecret": kvClient.Cred.AADClientSecret,
}

// test for service principal
ginkgo.By("Run for service principal")
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "SPN",
"azurestoragespnclientid": kvClient.Cred.AADClientID,
"azurestoragespntenantid": kvClient.Cred.TenantID,
}
secretData = map[string]string{
"azurestorageaccountname": accountName,
"azurestoragespnclientsecret": kvClient.Cred.AADClientSecret,
}

// assign role to service principal
objectID, err := kvClient.GetServicePrincipalObjectID(ctx, kvClient.Cred.AADClientID)
framework.ExpectNoError(err, fmt.Sprintf("Error GetServicePrincipalObjectID from clientID(%s): %v", kvClient.Cred.AADClientID, err))
// assign role to service principal
objectID, err := kvClient.GetServicePrincipalObjectID(ctx, kvClient.Cred.AADClientID)
framework.ExpectNoError(err, fmt.Sprintf("Error GetServicePrincipalObjectID from clientID(%s): %v", kvClient.Cred.AADClientID, err))

resourceID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", kvClient.Cred.SubscriptionID, kvClient.Cred.ResourceGroup, accountName)
resourceID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", kvClient.Cred.SubscriptionID, kvClient.Cred.ResourceGroup, accountName)

ginkgo.By(fmt.Sprintf("assign Storage Blob Data Contributor role to the service principal, objectID:%s", objectID))
roleDef, err := authClient.GetRoleDefinition(ctx, resourceID, "Storage Blob Data Contributor")
framework.ExpectNoError(err, fmt.Sprintf("Error GetRoleDefinition from resourceID(%s): %v", resourceID, err))
ginkgo.By(fmt.Sprintf("assign Storage Blob Data Contributor role to the service principal, objectID:%s", objectID))
roleDef, err := authClient.GetRoleDefinition(ctx, resourceID, "Storage Blob Data Contributor")
framework.ExpectNoError(err, fmt.Sprintf("Error GetRoleDefinition from resourceID(%s): %v", resourceID, err))

roleDefID := *roleDef.ID
_, err = authClient.AssignRole(ctx, resourceID, objectID, roleDefID)
if err != nil && strings.Contains(err.Error(), "The role assignment already exists") {
err = nil
}
framework.ExpectNoError(err, fmt.Sprintf("Error AssignRole (roleDefID(%s)) to objectID(%s) to access resource (resourceID(%s)), error: %v", roleDefID, objectID, resourceID, err))
roleDefID := *roleDef.ID
_, err = authClient.AssignRole(ctx, resourceID, objectID, roleDefID)
if err != nil && strings.Contains(err.Error(), "The role assignment already exists") {
err = nil
}
framework.ExpectNoError(err, fmt.Sprintf("Error AssignRole (roleDefID(%s)) to objectID(%s) to access resource (resourceID(%s)), error: %v", roleDefID, objectID, resourceID, err))

run()
run()

// test for managed identity(objectID)
objectID, err = kvClient.GetMSIObjectID(ctx, "blobfuse-csi-driver-e2e-test-id")
if err != nil {
// only e2e-vmss test job will use msi blobfuse-csi-driver-e2e-test-id, other jobs use service principal, so skip here
return
}
// test for managed identity(objectID)
objectID, err = kvClient.GetMSIObjectID(ctx, "blobfuse-csi-driver-e2e-test-id")
if err != nil {
// only e2e-vmss test job will use msi blobfuse-csi-driver-e2e-test-id, other jobs use service principal, so skip here
return
}

ginkgo.By(fmt.Sprintf("Run for managed identity (objectID %s)", objectID))
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "MSI",
"azurestorageidentityobjectid": objectID,
}
ginkgo.By(fmt.Sprintf("Run for managed identity (objectID %s)", objectID))
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "MSI",
"azurestorageidentityobjectid": objectID,
}

secretData = map[string]string{
"azurestorageaccountname": accountName,
}
ginkgo.By(fmt.Sprintf("assign Storage Blob Data Contributor role to the managed identity, objectID:%s", objectID))
_, err = authClient.AssignRole(ctx, resourceID, objectID, roleDefID)
if err != nil && strings.Contains(err.Error(), "The role assignment already exists") {
err = nil
}
framework.ExpectNoError(err, fmt.Sprintf("Error AssignRole (roleDefID(%s)) to objectID(%s) to access resource (resourceID(%s)), error: %v", roleDefID, objectID, resourceID, err))
secretData = map[string]string{
"azurestorageaccountname": accountName,
}
ginkgo.By(fmt.Sprintf("assign Storage Blob Data Contributor role to the managed identity, objectID:%s", objectID))
_, err = authClient.AssignRole(ctx, resourceID, objectID, roleDefID)
if err != nil && strings.Contains(err.Error(), "The role assignment already exists") {
err = nil
}
framework.ExpectNoError(err, fmt.Sprintf("Error AssignRole (roleDefID(%s)) to objectID(%s) to access resource (resourceID(%s)), error: %v", roleDefID, objectID, resourceID, err))

run()
run()

// test for managed identity(resourceID)
resourceID, err = kvClient.GetMSIResourceID(ctx, "blobfuse-csi-driver-e2e-test-id")
if err != nil {
// only e2e-vmss test job will use msi blobfuse-csi-driver-e2e-test-id, other jobs use service principal, so skip here
return
}
ginkgo.By(fmt.Sprintf("Run for managed identity (resourceID %s)", resourceID))
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "MSI",
"azurestorageidentityresourceid": resourceID,
}
secretData = map[string]string{
"azurestorageaccountname": accountName,
}
// test for managed identity(resourceID)
resourceID, err = kvClient.GetMSIResourceID(ctx, "blobfuse-csi-driver-e2e-test-id")
if err != nil {
// only e2e-vmss test job will use msi blobfuse-csi-driver-e2e-test-id, other jobs use service principal, so skip here
return
}
ginkgo.By(fmt.Sprintf("Run for managed identity (resourceID %s)", resourceID))
pod.Volumes[n].Attrib = map[string]string{
"azurestorageauthtype": "MSI",
"azurestorageidentityresourceid": resourceID,
}
secretData = map[string]string{
"azurestorageaccountname": accountName,
}

run()
run()
*/
}
}
}
4 changes: 2 additions & 2 deletions test/sanity/run-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ if [ ! -f "$azcopyPath" ]; then
chmod +x /usr/local/bin/azcopy
fi

_output/amd64/blobplugin --endpoint "$controllerendpoint" -v=5 &
_output/amd64/blobplugin --endpoint "$nodeendpoint" --nodeid "$nodeid" --enable-blob-mock-mount -v=5 &
_output/amd64/blobplugin --endpoint "$controllerendpoint" -v=5 --kubeconfig "no-need-kubeconfig" &
_output/amd64/blobplugin --endpoint "$nodeendpoint" --nodeid "$nodeid" --enable-blob-mock-mount -v=5 --kubeconfig "no-need-kubeconfig" &

echo "Begin to run sanity test..."
readonly CSI_SANITY_BIN='csi-sanity'
Expand Down
2 changes: 1 addition & 1 deletion test/sanity/sanity_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestSanity(t *testing.T) {

os.Setenv("AZURE_CREDENTIAL_FILE", credentials.TempAzureCredentialFilePath)

azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret)
azureClient, err := azure.GetClient(creds.Cloud, creds.SubscriptionID, creds.AADClientID, creds.TenantID, creds.AADClientSecret, creds.AADFederatedTokenFile)
assert.NoError(t, err)

ctx := context.Background()
Expand Down
12 changes: 9 additions & 3 deletions test/utils/azure/azure_helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,20 @@ type Client struct {
accountsClient accountclient.Interface
}

func GetClient(cloud, subscriptionID, clientID, tenantID, clientSecret string) (*Client, error) {
func GetClient(cloud, subscriptionID, clientID, tenantID, clientSecret string, aadFederatedTokenFile string) (*Client, error) {
armConfig := &azclient.ARMClientConfig{
Cloud: cloud,
TenantID: tenantID,
}
useFederatedWorkloadIdentityExtension := false
if aadFederatedTokenFile != "" {
useFederatedWorkloadIdentityExtension = true
}
credProvider, err := azclient.NewAuthProvider(armConfig, &azclient.AzureAuthConfig{
AADClientID: clientID,
AADClientSecret: clientSecret,
AADClientID: clientID,
AADClientSecret: clientSecret,
AADFederatedTokenFile: aadFederatedTokenFile,
UseFederatedWorkloadIdentityExtension: useFederatedWorkloadIdentityExtension,
})
if err != nil {
return nil, err
Expand Down
Loading
Loading