Skip to content
This repository was archived by the owner on Oct 12, 2023. It is now read-only.

Commit cea0550

Browse files
zfengmszfengms
authored
Feature/nationalcloud (#239)
* support national cloud * fix hardcoded domain name in createOutputFile * update rAzureBatch version etc * auto discovery of storage account endpoint suffix * styling fix * fix test failure * add back endpointSuffix for storage account * add storage account endpoint suffix to downloadBlob call * update docs
1 parent 3d84350 commit cea0550

File tree

8 files changed

+87
-22
lines changed

8 files changed

+87
-22
lines changed

‎DESCRIPTION

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ Depends:
1717
foreach (>= 1.4.3),
1818
iterators (>= 1.0.8)
1919
Imports:
20-
rAzureBatch (>= 0.5.3),
20+
rAzureBatch (>= 0.5.7),
2121
jsonlite,
2222
rjson,
2323
xml2,
@@ -27,5 +27,5 @@ Suggests:
2727
caret,
2828
plyr,
2929
lintr
30-
Remotes: Azure/rAzureBatch@v0.5.6
30+
Remotes: Azure/rAzureBatch@v0.5.7
3131
RoxygenNote: 6.0.1

‎R/credentials.R

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@
2222
#' generateCredentialsConfig("test_config.json")
2323
#' generateCredentialsConfig("test_config.json", batchAccount = "testbatchaccount",
2424
#' batchKey = "test_batch_account_key", batchUrl = "http://testbatchaccount.azure.com",
25-
#' storageAccount = "teststorageaccount", storageKey = "test_storage_account_key")
25+
#' storageAccount = "teststorageaccount", storageKey = "test_storage_account_key",
26+
#' storageEndpointSuffix = "core.windows.net")
27+
#' supported storage account endpoint suffix: core.windows.net (default),
28+
#' core.chinacloudapi.cn, core.cloudapi.de, core.usgovcloudapi.net, etc.
2629
#' }
2730
#' @export
2831
generateCredentialsConfig <- function(fileName, ...) {
@@ -46,6 +49,11 @@ generateCredentialsConfig <- function(fileName, ...) {
4649
"storage_account_key",
4750
args$storageKey)
4851

52+
storageSuffix <-
53+
ifelse(is.null(args$storageEndpointSuffix),
54+
"core.windows.net",
55+
args$storageEndpointSuffix)
56+
4957
githubAuthenticationToken <-
5058
ifelse(is.null(args$githubAuthenticationToken),
5159
"",
@@ -77,7 +85,8 @@ generateCredentialsConfig <- function(fileName, ...) {
7785
key = batchKey,
7886
url = batchUrl),
7987
storageAccount = list(name = storageName,
80-
key = storageKey),
88+
key = storageKey,
89+
endpointSuffix = storageEndpointSuffix),
8190
githubAuthenticationToken = githubAuthenticationToken,
8291
dockerAuthentication = list(username = dockerUsername,
8392
password = dockerPassword,

‎R/doAzureParallel.R

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,8 @@ setHttpTraffic <- function(value = FALSE) {
272272
assign(
273273
"inputs",
274274
list(name = storageCredentials$name,
275-
sasToken = sasToken),
275+
sasToken = sasToken,
276+
endpointSuffix = storageCredentials$endpointSuffix),
276277
.doAzureBatchGlobals
277278
)
278279
}
@@ -417,20 +418,33 @@ setHttpTraffic <- function(value = FALSE) {
417418
# Creating read-only SAS token blob resource file urls
418419
sasToken <- rAzureBatch::createSasToken("r", "c", id)
419420
workerScriptUrl <-
420-
rAzureBatch::createBlobUrl(storageCredentials$name, id, "worker.R", sasToken)
421+
rAzureBatch::createBlobUrl(storageCredentials$name, id, "worker.R", sasToken, storageCredentials$endpointSuffix)
421422
mergerScriptUrl <-
422-
rAzureBatch::createBlobUrl(storageCredentials$name, id, "merger.R", sasToken)
423+
rAzureBatch::createBlobUrl(storageCredentials$name, id, "merger.R", sasToken, storageCredentials$endpointSuffix)
423424
installGithubScriptUrl <-
424425
rAzureBatch::createBlobUrl(storageCredentials$name,
425426
id,
426427
"install_github.R",
427-
sasToken)
428+
sasToken,
429+
storageCredentials$endpointSuffix)
428430
installCranScriptUrl <-
429-
rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_cran.R", sasToken)
431+
rAzureBatch::createBlobUrl(storageCredentials$name,
432+
id,
433+
"install_cran.R",
434+
sasToken,
435+
storageCredentials$endpointSuffix)
430436
installBioConductorScriptUrl <-
431-
rAzureBatch::createBlobUrl(storageCredentials$name, id, "install_bioconductor.R", sasToken)
437+
rAzureBatch::createBlobUrl(storageCredentials$name,
438+
id,
439+
"install_bioconductor.R",
440+
sasToken,
441+
storageCredentials$endpointSuffix)
432442
jobCommonFileUrl <-
433-
rAzureBatch::createBlobUrl(storageCredentials$name, id, jobFileName, sasToken)
443+
rAzureBatch::createBlobUrl(storageCredentials$name,
444+
id,
445+
jobFileName,
446+
sasToken,
447+
storageCredentials$endpointSuffix)
434448

435449
requiredJobResourceFiles <- list(
436450
rAzureBatch::createResourceFile(url = workerScriptUrl, fileName = "worker.R"),
@@ -608,6 +622,7 @@ setHttpTraffic <- function(value = FALSE) {
608622
paste0("result/", "merge-result.rds"),
609623
sasToken = sasToken,
610624
accountName = storageCredentials$name,
625+
endpointSuffix = storageCredentials$endpointSuffix,
611626
downloadPath = tempFile,
612627
overwrite = TRUE
613628
)
@@ -702,7 +717,7 @@ setHttpTraffic <- function(value = FALSE) {
702717
azureStorageUrl <-
703718
paste0("http://",
704719
storageCredentials$name,
705-
".blob.core.windows.net/",
720+
sprintf(".blob.%s/", storageCredentials$endpointSuffix),
706721
id)
707722

708723
staticHtml <- "<h1>Errors:</h1>"

‎R/helpers.R

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626

2727
readToken <- rAzureBatch::createSasToken("r", "c", jobId)
2828
envFileUrl <-
29-
rAzureBatch::createBlobUrl(storageCredentials$name, jobId, envFile, readToken)
29+
rAzureBatch::createBlobUrl(storageCredentials$name, jobId, envFile, readToken, storageCredentials$endpointSuffix)
3030
resourceFiles <-
3131
list(rAzureBatch::createResourceFile(url = envFileUrl, fileName = envFile))
3232
}
@@ -38,10 +38,11 @@
3838
if (!is.null(cloudCombine)) {
3939
assign("cloudCombine", cloudCombine, .doAzureBatchGlobals)
4040
copyCommand <- sprintf(
41-
"%s %s %s --download --saskey $BLOBXFER_SASKEY --remoteresource . --include result/*.rds",
41+
"%s %s %s --endpoint %s --download --saskey $BLOBXFER_SASKEY --remoteresource . --include result/*.rds",
4242
accountName,
4343
jobId,
44-
"$AZ_BATCH_TASK_WORKING_DIR"
44+
"$AZ_BATCH_TASK_WORKING_DIR",
45+
storageCredentials$endpointSuffix
4546
)
4647

4748
downloadCommand <-
@@ -61,7 +62,8 @@
6162
rAzureBatch::createBlobUrl(
6263
storageAccount = storageCredentials$name,
6364
containerName = jobId,
64-
sasToken = rAzureBatch::createSasToken("w", "c", jobId)
65+
sasToken = rAzureBatch::createSasToken("w", "c", jobId),
66+
storageEndpointSuffix = storageCredentials$endpointSuffix
6567
)
6668

6769
outputFiles <- list(

‎R/utility.R

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -170,13 +170,17 @@ createOutputFile <- function(filePattern, url) {
170170
)
171171

172172
# Parsing url to obtain container's virtual directory path
173-
azureDomain <- "blob.core.windows.net"
174-
parsedValue <- strsplit(url, azureDomain)[[1]]
173+
# sample url: "https://accountname.blob.core.windows.net/outputs?se=2017-07-31&sr=c&st=2017-07-12"
174+
# after split by "/"
175+
# parsedValue[1] is "https"
176+
# parsedValue[2] is ""
177+
# parsedValue[3] is "accountname.blob.core.windows.net"
178+
# parsedValue[4] is "outputs?se=2017-07-31&sr=c&st=2017-07-12"
175179

176-
accountName <- parsedValue[1]
177-
urlPath <- parsedValue[2]
180+
parsedValue <- strsplit(url, "/")[[1]]
178181

179-
baseUrl <- paste0(accountName, azureDomain)
182+
baseUrl <- paste0(parsedValue[1], "//", parsedValue[3])
183+
urlPath <- sub(baseUrl, "", url)
180184
parsedUrlPath <- strsplit(urlPath, "?", fixed = TRUE)[[1]]
181185

182186
storageContainerPath <- parsedUrlPath[1]

‎docs/33-programmatically-generate-config.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ You can generate credentials by creating a R object as shown below:
1515
),
1616
"storageAccount" = list(
1717
"name" = "storageaccountname",
18-
"key" = "storageaccountkey"
18+
"key" = "storageaccountkey",
19+
"endpointSuffix" = "core.windows.net"
1920
),
2021
"githubAuthenticationToken" = "",
2122
"dockerAuthentication" = list("username" = "",

‎docs/34-national-clouds.md

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Configuration for national clouds
2+
3+
doAzureParallel is configured to run in public Azure cloud by default. To run workloads in national clouds, configure endpoint suffix for storage account in the cluster config which tells doAzureParallel which national cloud environment the storage account resides.
4+
5+
EndpointSuffix is the last part of the connection string shown in the Storage Account Access keys blade from Azure portal. The possible values usually are:
6+
7+
Azure public cloud: core.windows.net
8+
Azure China cloud: core.chinacloudapi.cn
9+
Azure US government cloud: core.usgovcloudapi.net
10+
Azure German cloud: core.cloudapi.de
11+
12+
The value may be different if a DNS redirect is used, so it is better to double check its value on Storage Account Access keys blade.
13+
14+
Below is a sample of credential config with endpoint suffix specified:
15+
16+
```R
17+
{
18+
"batchAccount": {
19+
"name": <Azure Batch Account Name>,
20+
"key": <Azure Batch Account Key>,
21+
"url": <Azure Batch Account URL>
22+
},
23+
"storageAccount": {
24+
"name": <Azure Storage Account Name>,
25+
"key": <Azure Storage Account Key>,
26+
"endpointSuffix": <Azure Storage Account Endpoint Suffix>
27+
},
28+
"githubAuthenticationToken": {}
29+
}
30+
```

‎docs/README.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ This section will provide information about how Azure works, how best to take ad
4545

4646
Generate credentials and cluster config at runtime programmatically
4747

48+
11. **National Cloud configuration" [(link)](/.34-national-clouds.md)
49+
50+
How to run workload in Azure national clouds
51+
4852
## Additional Documentation
4953
Take a look at our [**Troubleshooting Guide**](./40-troubleshooting.md) for information on how to diagnose common issues.
5054

0 commit comments

Comments
 (0)