Fix/storage management (#267)

* Removed rAzureBatch from storage api calls * Fixed quota documentation * Added job and core quota limits
Azure · May 7, 2018 · 84aa7c9 · 84aa7c9
1 parent 6582af4
commit 84aa7c9
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 34 deletions.
diff --git a/R/autoscale.R b/R/autoscale.R
@@ -97,10 +97,10 @@ resizeCluster <- function(cluster,
                           algorithm = "QUEUE",
                           timeInterval = "PT5M") {
   config <- getOption("az_config")
-  pool <- config$batchClient$poolOperations$getPool(
+  cluster <- config$batchClient$poolOperations$getPool(
     cluster$poolId)
 
-  rAzureBatch::resizePool(
+  config$batchClient$poolOperations$resizePool(
     cluster$poolId,
     autoscaleFormula = getAutoscaleFormula(
       algorithm,

diff --git a/R/cluster.R b/R/cluster.R
@@ -380,7 +380,8 @@ makeCluster <-
 #' }
 #' @export
 getCluster <- function(clusterName, verbose = TRUE) {
-  config <- getOption("az_config")
+  config <- getConfiguration()
+
   pool <- config$batchClient$poolOperations$getPool(
     clusterName)
 
@@ -476,14 +477,14 @@ getClusterList <- function(filter = NULL) {
     }
   }
 
-  pools <-
-    rAzureBatch::listPools(
-      query = list(
-        "$filter" = filterClause,
-        "$select" = paste0("id,state,allocationState,vmSize,currentDedicatedNodes,",
-                    "targetDedicatedNodes,currentLowPriorityNodes,targetLowPriorityNodes")
-      )
+  config <- getOption("az_config")
+  pools <- config$batchClient$poolOperations$listPools(
+    query = list(
+      "$filter" = filterClause,
+      "$select" = paste0("id,state,allocationState,vmSize,currentDedicatedNodes,",
+                         "targetDedicatedNodes,currentLowPriorityNodes,targetLowPriorityNodes")
     )
+  )
 
   count <- length(pools$value)
   id <- character(count)

diff --git a/R/logging.R b/R/logging.R
@@ -34,7 +34,10 @@ getClusterFile <-
 
     filePath <- sprintf(prefixfilePath, filePath)
 
-    nodeFileContent <- rAzureBatch::getNodeFile(
+    config <- getConfiguration()
+    batchClient <- config$batchClient
+
+    nodeFileContent <- batchClient$fileOperations$getNodeFile(
       cluster$poolId,
       nodeId,
       filePath,
@@ -76,15 +79,17 @@ getJobFile <-
       filePath <- substring(filePath, 2)
     }
 
-    jobFileContent <-
-      rAzureBatch::getTaskFile(
-        jobId,
-        taskId,
-        filePath,
-        downloadPath = downloadPath,
-        overwrite = overwrite,
-        progress = TRUE
-      )
+    config <- getConfiguration()
+    batchClient <- config$batchClient
+
+    jobFileContent <- batchClient$fileOperations$getTaskFile(
+      jobId,
+      taskId,
+      filePath,
+      downloadPath = downloadPath,
+      overwrite = overwrite,
+      progress = TRUE
+    )
 
     jobFileContent
   }
diff --git a/R/storage_management.R b/R/storage_management.R
@@ -10,8 +10,12 @@
 #' }
 #' @export
 listStorageContainers <- function(prefix = "") {
+  config <- getConfiguration()
+  storageClient <- config$storageClient
+
   xmlResponse <-
-    rAzureBatch::listContainers(prefix, content = "parsed")
+    storageClient$containerOperations$deleteContainer$listContainers(
+      prefix, content = "parsed")
 
   name <- getXmlValues(xmlResponse, ".//Container/Name")
   lastModified <-
@@ -74,8 +78,14 @@ deleteStorageContainer <- function(container, verbose = TRUE) {
 #' }
 #' @export
 listStorageFiles <- function(container, prefix = "", ...) {
-  xmlResponse <-
-    rAzureBatch::listBlobs(container, prefix, content = "parsed", ...)
+  config <- getConfiguration()
+  storageClient <- config$storageClient
+
+  xmlResponse <- storageClient$blobOperations$listBlobs(
+    container,
+    prefix,
+    content = "parsed",
+    ...)
 
   filePath <- getXmlValues(xmlResponse, ".//Blob/Name")
 
@@ -126,14 +136,18 @@ getStorageFile <-
            overwrite = FALSE,
            verbose = TRUE,
            ...) {
-    jobFileContent <- rAzureBatch::downloadBlob(
-      container,
-      blobPath,
-      downloadPath = downloadPath,
-      overwrite = overwrite,
-      progress = TRUE,
-      ...
-    )
+    config <- getConfiguration()
+    storageClient <- config$storageClient
+
+    jobFileContent <-
+      storageClient$blobOperations$downloadBlob(
+        container,
+        blobPath,
+        downloadPath = downloadPath,
+        overwrite = overwrite,
+        progress = TRUE,
+        ...
+      )
 
     jobFileContent
   }
@@ -145,8 +159,15 @@ getStorageFile <-
 #'
 #' @export
 deleteStorageFile <- function(container, blobPath, ...) {
+  config <- getConfiguration()
+  storageClient <- config$storageClient
+
   response <-
-    rAzureBatch::deleteBlob(container, blobPath, content = "response", ...)
+    storageClient$blobOperations$deleteBlob(
+      container,
+      blobPath,
+      content = "response",
+      ...)
 
   if (response$status_code == 202) {
     cat(

diff --git a/docs/12-quota-limitations.md b/docs/12-quota-limitations.md
@@ -10,9 +10,9 @@ Our default VM size selection is the **"Standard_F2"** that has 2 core per VM. W
 
 ## Number of *foreach* Loops
 
-By default, doAzureParallel users are limited to running 20 *foreach* loops in Azure at a time. This is because each *foreach* loops generates a *job*, of which users are by default limited to 20. To go beyond that, users need to wait for their *jobs* to complete.
+By default, doAzureParallel users are limited to running 20 *foreach* loops in Azure at a time. This is because each *foreach* loops generates a *job*, of which users are by default limited to 20.
 
-## Increasing Your Quota
+## Increasing Your Core and Job Quota
 
 To increase your default quota limitations, please visit [this page](https://docs.microsoft.com/en-us/azure/batch/batch-quota-limit#increase-a-quota) for instructions.