diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 239af0ec508..ea5403686f1 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -127,6 +127,7 @@ This reference uses `.` to denote the nesting of values. * `blockstore.azure.pre_signed_expiry` `(time duration : "15m")` - Expiry of pre-signed URL. * `blockstore.azure.disable_pre_signed` `(bool : false)` - Disable use of pre-signed URL. * `blockstore.azure.disable_pre_signed_ui` `(bool : true)` - Disable use of pre-signed URL in the UI. +* `blockstore.azure.china_cloud` `(bool : false)` - Enable for using lakeFS on Azure China Cloud. * `blockstore.s3.region` `(string : "us-east-1")` - Default region for lakeFS to use when interacting with S3. * `blockstore.s3.profile` `(string : )` - If specified, will be used as a [named credentials profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-using-profiles) * `blockstore.s3.credentials_file` `(string : )` - If specified, will be used as a [credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) diff --git a/pkg/block/azure/adapter.go b/pkg/block/azure/adapter.go index 104cc47d4f8..cafe8b99405 100644 --- a/pkg/block/azure/adapter.go +++ b/pkg/block/azure/adapter.go @@ -29,7 +29,8 @@ const ( // more the 5000 different accounts, which is highly unlikely udcCacheSize = 5000 - BlobEndpointFormat = "https://%s.blob.core.windows.net/" + BlobEndpointGlobalFormat = "https://%s.blob.core.windows.net/" + BlobEndpointChinaCloudFormat = "https://%s.blob.core.chinacloudapi.cn/" ) type Adapter struct { @@ -37,6 +38,7 @@ type Adapter struct { preSignedExpiry time.Duration disablePreSigned bool disablePreSignedUI bool + chinaCloud bool } func NewAdapter(ctx context.Context, params params.Azure) (*Adapter, error) { @@ -54,6 +56,7 @@ func NewAdapter(ctx context.Context, params params.Azure) (*Adapter, error) { preSignedExpiry: preSignedExpiry, disablePreSigned: params.DisablePreSigned, disablePreSignedUI: params.DisablePreSignedUI, + chinaCloud: params.ChinaCloud, }, nil } @@ -62,6 +65,7 @@ type BlobURLInfo struct { ContainerURL string ContainerName string BlobURL string + Host string } type PrefixURLInfo struct { @@ -106,6 +110,7 @@ func ResolveBlobURLInfoFromURL(pathURL *url.URL) (BlobURLInfo, error) { ContainerURL: fmt.Sprintf("%s://%s/%s", pathURL.Scheme, pathURL.Host, pathParts[0]), ContainerName: pathParts[0], BlobURL: strings.Join(pathParts[1:], "/"), + Host: pathURL.Host, }, nil } @@ -131,6 +136,7 @@ func resolveBlobURLInfo(obj block.ObjectPointer) (BlobURLInfo, error) { ContainerURL: qp.ContainerURL, ContainerName: qp.ContainerName, BlobURL: qp.BlobURL + "/" + key, + Host: parsedNamespace.Host, } if qp.BlobURL == "" { info.BlobURL = key @@ -266,7 +272,7 @@ func (a *Adapter) getPreSignedURL(ctx context.Context, obj block.ObjectPointer, } // format blob URL with signed SAS query params - accountEndpoint := fmt.Sprintf(BlobEndpointFormat, qualifiedKey.StorageAccountName) + accountEndpoint := buildAccountEndpoint(qualifiedKey.StorageAccountName, a.chinaCloud) u, err := url.JoinPath(accountEndpoint, qualifiedKey.ContainerName, qualifiedKey.BlobURL) if err != nil { return "", err @@ -564,6 +570,12 @@ func (a *Adapter) GetStorageNamespaceInfo() block.StorageNamespaceInfo { info := block.DefaultStorageNamespaceInfo(block.BlockstoreTypeAzure) info.ImportValidityRegex = `^https?://[a-z0-9_-]+\.(blob|adls)\.core\.windows\.net` // added adls for import hint validation in UI info.ValidityRegex = `^https?://[a-z0-9_-]+\.blob\.core\.windows\.net` + + if a.chinaCloud { + info.ImportValidityRegex = `^https?://[a-z0-9_-]+\.(blob|adls)\.core\.chinacloudapi\.cn` + info.ValidityRegex = `^https?://[a-z0-9_-]+\.blob\.core\.chinacloudapi\.cn` + } + info.Example = "https://mystorageaccount.blob.core.windows.net/mycontainer/" if a.disablePreSigned { info.PreSignSupport = false diff --git a/pkg/block/azure/adapter_test.go b/pkg/block/azure/adapter_test.go index c38e9920764..d594f5707b0 100644 --- a/pkg/block/azure/adapter_test.go +++ b/pkg/block/azure/adapter_test.go @@ -30,26 +30,34 @@ func TestAzureAdapter(t *testing.T) { } func TestAdapterNamespace(t *testing.T) { - adapter, err := azure.NewAdapter(context.Background(), params.Azure{ - StorageAccount: accountName, - StorageAccessKey: accountKey, - TestEndpointURL: blockURL, - }) - require.NoError(t, err, "create new adapter") - - expr, err := regexp.Compile(adapter.GetStorageNamespaceInfo().ValidityRegex) - require.NoError(t, err) - tests := []struct { Name string Namespace string Success bool + China bool }{ { Name: "valid_https", Namespace: "https://test.blob.core.windows.net/container1/repo1", Success: true, }, + { + Name: "valid_https", + Namespace: "https://test.blob.core.windows.net/container1/repo1", + Success: false, + China: true, + }, + { + Name: "valid_https_china", + Namespace: "https://test.blob.core.chinacloudapi.cn/container1/repo1", + Success: false, + }, + { + Name: "valid_https_china", + Namespace: "https://test.blob.core.chinacloudapi.cn/container1/repo1", + Success: true, + China: true, + }, { Name: "valid_http", Namespace: "http://test.blob.core.windows.net/container1/repo1", @@ -75,9 +83,30 @@ func TestAdapterNamespace(t *testing.T) { Namespace: "this is a bad string", Success: false, }, + { + Name: "invalid_https_china_mix_1", + Namespace: "https://test.blob.core.chinacloudapi.net/container1/repo1", + Success: false, + }, + { + Name: "invalid_https_china_mix_2", + Namespace: "https://test.blob.core.windows.cn/container1/repo1", + Success: false, + }, } for _, tt := range tests { t.Run(tt.Name, func(t *testing.T) { + adapter, err := azure.NewAdapter(context.Background(), params.Azure{ + StorageAccount: accountName, + StorageAccessKey: accountKey, + TestEndpointURL: blockURL, + ChinaCloud: tt.China, + }) + require.NoError(t, err, "create new adapter") + + expr, err := regexp.Compile(adapter.GetStorageNamespaceInfo().ValidityRegex) + require.NoError(t, err) + require.Equal(t, tt.Success, expr.MatchString(tt.Namespace)) }) } diff --git a/pkg/block/azure/client_cache.go b/pkg/block/azure/client_cache.go index 2d54908812a..a4525654606 100644 --- a/pkg/block/azure/client_cache.go +++ b/pkg/block/azure/client_cache.go @@ -124,7 +124,7 @@ func BuildAzureServiceClient(params params.Azure) (*service.Client, error) { if params.TestEndpointURL != "" { // For testing purposes - override default url template url = params.TestEndpointURL } else { - url = fmt.Sprintf(BlobEndpointFormat, params.StorageAccount) + url = buildAccountEndpoint(params.StorageAccount, params.ChinaCloud) } options := service.ClientOptions{ClientOptions: azcore.ClientOptions{Retry: policy.RetryOptions{TryTimeout: params.TryTimeout}}} @@ -142,3 +142,11 @@ func BuildAzureServiceClient(params params.Azure) (*service.Client, error) { } return service.NewClient(url, defaultCreds, &options) } + +func buildAccountEndpoint(storageAccount string, chinaCloud bool) string { + format := BlobEndpointGlobalFormat + if chinaCloud { + format = BlobEndpointChinaCloudFormat + } + return fmt.Sprintf(format, storageAccount) +} diff --git a/pkg/block/params/block.go b/pkg/block/params/block.go index ffab817c3f0..47e10a9d047 100644 --- a/pkg/block/params/block.go +++ b/pkg/block/params/block.go @@ -77,6 +77,8 @@ type Azure struct { PreSignedExpiry time.Duration DisablePreSigned bool DisablePreSignedUI bool + // Azure China Cloud has different endpoints, different services and it's isolated from Azure Global Cloud + ChinaCloud bool // TestEndpointURL - For testing purposes, provide a custom URL to override the default URL template TestEndpointURL string } diff --git a/pkg/config/config.go b/pkg/config/config.go index 23a19697377..db29d6e7905 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -248,6 +248,7 @@ type Config struct { PreSignedExpiry time.Duration `mapstructure:"pre_signed_expiry"` DisablePreSigned bool `mapstructure:"disable_pre_signed"` DisablePreSignedUI bool `mapstructure:"disable_pre_signed_ui"` + ChinaCloud bool `mapstructure:"china_cloud"` // TestEndpointURL for testing purposes TestEndpointURL string `mapstructure:"test_endpoint_url"` } `mapstructure:"azure"` @@ -495,6 +496,9 @@ func (c *Config) BlockstoreAzureParams() (blockparams.Azure, error) { if c.Blockstore.Azure.AuthMethod != "" { logging.ContextUnavailable().Warn("blockstore.azure.auth_method is deprecated. Value is no longer used.") } + if c.Blockstore.Azure.ChinaCloud { + logging.ContextUnavailable().Warn("blockstore.azure.china_cloud is enabled. lakeFS will only function on Azure China Cloud") + } return blockparams.Azure{ StorageAccount: c.Blockstore.Azure.StorageAccount, StorageAccessKey: c.Blockstore.Azure.StorageAccessKey, @@ -503,6 +507,7 @@ func (c *Config) BlockstoreAzureParams() (blockparams.Azure, error) { TestEndpointURL: c.Blockstore.Azure.TestEndpointURL, DisablePreSigned: c.Blockstore.Azure.DisablePreSigned, DisablePreSignedUI: c.Blockstore.Azure.DisablePreSignedUI, + ChinaCloud: c.Blockstore.Azure.ChinaCloud, }, nil }