Skip to content
This repository was archived by the owner on Oct 12, 2023. It is now read-only.

Commit 84aa7c9

Browse files
authored
Fix/storage management (#267)
* Removed rAzureBatch from storage api calls * Fixed quota documentation * Added job and core quota limits
1 parent 6582af4 commit 84aa7c9

File tree

5 files changed

+61
-34
lines changed

5 files changed

+61
-34
lines changed

R/autoscale.R

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,10 @@ resizeCluster <- function(cluster,
9797
algorithm = "QUEUE",
9898
timeInterval = "PT5M") {
9999
config <- getOption("az_config")
100-
pool <- config$batchClient$poolOperations$getPool(
100+
cluster <- config$batchClient$poolOperations$getPool(
101101
cluster$poolId)
102102

103-
rAzureBatch::resizePool(
103+
config$batchClient$poolOperations$resizePool(
104104
cluster$poolId,
105105
autoscaleFormula = getAutoscaleFormula(
106106
algorithm,

R/cluster.R

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,8 @@ makeCluster <-
380380
#' }
381381
#' @export
382382
getCluster <- function(clusterName, verbose = TRUE) {
383-
config <- getOption("az_config")
383+
config <- getConfiguration()
384+
384385
pool <- config$batchClient$poolOperations$getPool(
385386
clusterName)
386387

@@ -476,14 +477,14 @@ getClusterList <- function(filter = NULL) {
476477
}
477478
}
478479

479-
pools <-
480-
rAzureBatch::listPools(
481-
query = list(
482-
"$filter" = filterClause,
483-
"$select" = paste0("id,state,allocationState,vmSize,currentDedicatedNodes,",
484-
"targetDedicatedNodes,currentLowPriorityNodes,targetLowPriorityNodes")
485-
)
480+
config <- getOption("az_config")
481+
pools <- config$batchClient$poolOperations$listPools(
482+
query = list(
483+
"$filter" = filterClause,
484+
"$select" = paste0("id,state,allocationState,vmSize,currentDedicatedNodes,",
485+
"targetDedicatedNodes,currentLowPriorityNodes,targetLowPriorityNodes")
486486
)
487+
)
487488

488489
count <- length(pools$value)
489490
id <- character(count)

R/logging.R

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ getClusterFile <-
3434

3535
filePath <- sprintf(prefixfilePath, filePath)
3636

37-
nodeFileContent <- rAzureBatch::getNodeFile(
37+
config <- getConfiguration()
38+
batchClient <- config$batchClient
39+
40+
nodeFileContent <- batchClient$fileOperations$getNodeFile(
3841
cluster$poolId,
3942
nodeId,
4043
filePath,
@@ -76,15 +79,17 @@ getJobFile <-
7679
filePath <- substring(filePath, 2)
7780
}
7881

79-
jobFileContent <-
80-
rAzureBatch::getTaskFile(
81-
jobId,
82-
taskId,
83-
filePath,
84-
downloadPath = downloadPath,
85-
overwrite = overwrite,
86-
progress = TRUE
87-
)
82+
config <- getConfiguration()
83+
batchClient <- config$batchClient
84+
85+
jobFileContent <- batchClient$fileOperations$getTaskFile(
86+
jobId,
87+
taskId,
88+
filePath,
89+
downloadPath = downloadPath,
90+
overwrite = overwrite,
91+
progress = TRUE
92+
)
8893

8994
jobFileContent
9095
}

R/storage_management.R

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@
1010
#' }
1111
#' @export
1212
listStorageContainers <- function(prefix = "") {
13+
config <- getConfiguration()
14+
storageClient <- config$storageClient
15+
1316
xmlResponse <-
14-
rAzureBatch::listContainers(prefix, content = "parsed")
17+
storageClient$containerOperations$deleteContainer$listContainers(
18+
prefix, content = "parsed")
1519

1620
name <- getXmlValues(xmlResponse, ".//Container/Name")
1721
lastModified <-
@@ -74,8 +78,14 @@ deleteStorageContainer <- function(container, verbose = TRUE) {
7478
#' }
7579
#' @export
7680
listStorageFiles <- function(container, prefix = "", ...) {
77-
xmlResponse <-
78-
rAzureBatch::listBlobs(container, prefix, content = "parsed", ...)
81+
config <- getConfiguration()
82+
storageClient <- config$storageClient
83+
84+
xmlResponse <- storageClient$blobOperations$listBlobs(
85+
container,
86+
prefix,
87+
content = "parsed",
88+
...)
7989

8090
filePath <- getXmlValues(xmlResponse, ".//Blob/Name")
8191

@@ -126,14 +136,18 @@ getStorageFile <-
126136
overwrite = FALSE,
127137
verbose = TRUE,
128138
...) {
129-
jobFileContent <- rAzureBatch::downloadBlob(
130-
container,
131-
blobPath,
132-
downloadPath = downloadPath,
133-
overwrite = overwrite,
134-
progress = TRUE,
135-
...
136-
)
139+
config <- getConfiguration()
140+
storageClient <- config$storageClient
141+
142+
jobFileContent <-
143+
storageClient$blobOperations$downloadBlob(
144+
container,
145+
blobPath,
146+
downloadPath = downloadPath,
147+
overwrite = overwrite,
148+
progress = TRUE,
149+
...
150+
)
137151

138152
jobFileContent
139153
}
@@ -145,8 +159,15 @@ getStorageFile <-
145159
#'
146160
#' @export
147161
deleteStorageFile <- function(container, blobPath, ...) {
162+
config <- getConfiguration()
163+
storageClient <- config$storageClient
164+
148165
response <-
149-
rAzureBatch::deleteBlob(container, blobPath, content = "response", ...)
166+
storageClient$blobOperations$deleteBlob(
167+
container,
168+
blobPath,
169+
content = "response",
170+
...)
150171

151172
if (response$status_code == 202) {
152173
cat(

docs/12-quota-limitations.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ Our default VM size selection is the **"Standard_F2"** that has 2 core per VM. W
1010

1111
## Number of *foreach* Loops
1212

13-
By default, doAzureParallel users are limited to running 20 *foreach* loops in Azure at a time. This is because each *foreach* loops generates a *job*, of which users are by default limited to 20. To go beyond that, users need to wait for their *jobs* to complete.
13+
By default, doAzureParallel users are limited to running 20 *foreach* loops in Azure at a time. This is because each *foreach* loops generates a *job*, of which users are by default limited to 20.
1414

15-
## Increasing Your Quota
15+
## Increasing Your Core and Job Quota
1616

1717
To increase your default quota limitations, please visit [this page](https://docs.microsoft.com/en-us/azure/batch/batch-quota-limit#increase-a-quota) for instructions.
1818

0 commit comments

Comments
 (0)