Update Azure Batch .Net SDK to meet API version 2018-03-01.6.1. (#4151)

* Regenerate protocol based on 2018-02 Swagger spec

* Add object model layer for new APIs

* Update MSBuild version for ProxyLayerParser

* Add tests for new APIs

* Update Batch version to 8.1.0

 - Updated the changelog as well.

* Update to latest swagger spec.

* Add AzSdk.RP.props to source.

* Move LogUpload test case to IaaS pool.

src/SDKs/Batch/DataPlane/AzSdk.RP.props

@@ -0,0 +1,7 @@
+<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <!--This file and it's contents are updated at build time moving or editing might result in build failure. Take due deligence while editing this file-->
+  <PropertyGroup>
+    <AzureApiTag>BatchService_2018-03-01.6.1;</AzureApiTag>
+    <PackageTags>$(PackageTags);$(CommonTags);$(AzureApiTag);</PackageTags>
+  </PropertyGroup>
+</Project>

src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/CloudPoolIntegrationTests.cs

@@ -1000,6 +1000,38 @@ await pool.ResizeAsync(
             await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
         }
 
+        [Fact]
+        [Trait(TestTraits.Duration.TraitName, TestTraits.Duration.Values.ShortDuration)]
+        public void PoolStateCount_IsReturnedFromServer()
+        {
+            Action test = () =>
+            {
+                using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
+                {
+                    var nodeCounts = batchCli.PoolOperations.ListPoolNodeCounts();
+
+                    Assert.NotEmpty(nodeCounts);
+                    var poolId = nodeCounts.First().PoolId;
+
+                    foreach (var poolNodeCount in nodeCounts)
+                    {
+                        Assert.NotEmpty(poolNodeCount.PoolId);
+                        Assert.NotNull(poolNodeCount.Dedicated);
+                        Assert.NotNull(poolNodeCount.LowPriority);
+
+                        // Check a few properties at random
+                        Assert.Equal(0, poolNodeCount.LowPriority.Unusable);
+                        Assert.Equal(0, poolNodeCount.LowPriority.Offline);
+                    }
+
+                    var filteredNodeCounts = batchCli.PoolOperations.ListPoolNodeCounts(new ODATADetailLevel(filterClause: $"poolId eq '{poolId}'")).ToList();
+                    Assert.Single(filteredNodeCounts);
+                }
+            };
+
+            SynchronizationContextHelper.RunTest(test, LongTestTimeout);
+        }
+
         #region Test helpers
 
         /// <summary>

src/SDKs/Batch/DataPlane/Azure.Batch.IntegrationTests/ComputeNodeIntegrationTests.cs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
-namespace BatchClientIntegrationTests
+namespace BatchClientIntegrationTests
 {
     using System;
     using System.Collections.Generic;
@@ -13,6 +13,9 @@
     using Fixtures;
     using Microsoft.Azure.Batch;
     using Microsoft.Azure.Batch.Common;
+    using Microsoft.WindowsAzure.Storage;
+    using Microsoft.WindowsAzure.Storage.Auth;
+    using Microsoft.WindowsAzure.Storage.Blob;
     using IntegrationTestUtilities;
     using Microsoft.Azure.Batch.Protocol.BatchRequests;
     using Microsoft.Rest.Azure;
@@ -774,6 +777,77 @@ public void TestComputeNodeUserIaas()
             SynchronizationContextHelper.RunTest(test, TestTimeout);
         }
 
+        [Fact]
+        [Trait(TestTraits.Duration.TraitName, TestTraits.Duration.Values.ShortDuration)]
+        public void ComputeNodeUploadLogs()
+        {
+            Action test = () =>
+            {
+                using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
+                {
+                    var node = batchCli.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).First();
+
+                    // Generate a storage container URL
+                    StagingStorageAccount storageAccount = TestUtilities.GetStorageCredentialsFromEnvironment();
+                    CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
+                        new StorageCredentials(storageAccount.StorageAccount, storageAccount.StorageAccountKey),
+                        blobEndpoint: storageAccount.BlobUri,
+                        queueEndpoint: null,
+                        tableEndpoint: null,
+                        fileEndpoint: null);
+                    CloudBlobClient blobClient = cloudStorageAccount.CreateCloudBlobClient();
+                    const string containerName = "computenodelogscontainer";
+                    var container = blobClient.GetContainerReference(containerName);
+
+                    try
+                    {
+                        container.CreateIfNotExists();
+
+                        // Ensure that there are no items in the container to begin with
+                        var blobs = container.ListBlobs();
+                        Assert.Empty(blobs);
+
+                        var sas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
+                        {
+                            Permissions = SharedAccessBlobPermissions.Write,
+                            SharedAccessExpiryTime = DateTime.UtcNow.AddDays(1)
+                        });
+                        var fullSas = container.Uri + sas;
+
+                        var startTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5));
+
+                        var result = batchCli.PoolOperations.UploadComputeNodeBatchServiceLogs(
+                            this.poolFixture.PoolId,
+                            node.Id,
+                            fullSas,
+                            startTime);
+
+                        Assert.NotEqual(0, result.NumberOfFilesUploaded);
+                        Assert.NotEmpty(result.VirtualDirectoryName);
+
+                        // Allow up to 2m for files to get uploaded
+                        DateTime timeoutAt = DateTime.UtcNow.AddMinutes(2);
+                        while (DateTime.UtcNow < timeoutAt)
+                        {
+                            blobs = container.ListBlobs();
+                            if (blobs.Any())
+                            {
+                                break;
+                            }
+                        }
+
+                        Assert.NotEmpty(blobs);
+                    }
+                    finally
+                    {
+                        container.DeleteIfExists();
+                    }
+                }
+            };
+
+            SynchronizationContextHelper.RunTest(test, TestTimeout);
+        }
+
         [Fact]
         [Trait(TestTraits.Duration.TraitName, TestTraits.Duration.Values.ShortDuration)]
         public void TestGetRemoteLoginSettings()

src/SDKs/Batch/DataPlane/Azure.Batch/AssemblyAttributes.cs

@@ -10,7 +10,7 @@
 [assembly: AssemblyDescription("Client library for interacting with the Azure Batch service.")]
 
 [assembly: AssemblyVersion("8.0.0.0")]
-[assembly: AssemblyFileVersion("8.0.1.0")]
+[assembly: AssemblyFileVersion("8.1.0.0")]
 [assembly: AssemblyCompany("Microsoft Corporation")]
 [assembly: AssemblyProduct("Microsoft Azure")]
 [assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")]

src/SDKs/Batch/DataPlane/Azure.Batch/AsyncListPoolNodeCounts.cs

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.Batch
+{
+    using System.Diagnostics;
+    using System.Linq;
+    using System.Threading;
+    using Models = Microsoft.Azure.Batch.Protocol.Models;
+
+    internal class AsyncListPoolNodeCountsEnumerator : PagedEnumeratorBase<PoolNodeCounts>
+    {
+        private readonly PoolOperations _parentPoolOperations;
+        private readonly BehaviorManager _behaviorMgr;
+        private readonly DetailLevel _detailLevel;
+
+#region // constructors
+
+        internal AsyncListPoolNodeCountsEnumerator(
+                PoolOperations parentPoolOperations,
+                BehaviorManager behaviorMgr,
+                DetailLevel detailLevel)
+        {
+            _parentPoolOperations = parentPoolOperations;
+            _behaviorMgr = behaviorMgr;
+            _detailLevel = detailLevel;
+        }
+
+#endregion // constructors
+
+        public override PoolNodeCounts Current  // for IPagedEnumerator<T> and IEnumerator<T>
+        {
+            get
+            {
+                // start with the current object off of base
+                object curObj = base._currentBatch[base._currentIndex];
+
+                // it must be a protocol object from previous call
+                Models.PoolNodeCounts protocolObj = curObj as Models.PoolNodeCounts;
+
+                Debug.Assert(null != protocolObj);
+
+                // wrap protocol object
+                PoolNodeCounts wrapped = new PoolNodeCounts(protocolObj);
+
+                return wrapped;
+            }
+        }
+
+        /// <summary>
+        /// fetch another batch of objects from the server
+        /// </summary>
+        protected override async System.Threading.Tasks.Task GetNextBatchFromServerAsync(SkipTokenHandler skipHandler, CancellationToken cancellationToken)
+        {
+            do
+            {
+                // start the protocol layer call
+                var asyncTask = _parentPoolOperations.ParentBatchClient.ProtocolLayer.ListPoolNodeCounts(
+                    skipHandler.SkipToken,
+                    _behaviorMgr,
+                    _detailLevel,
+                    cancellationToken);
+
+                var response = await asyncTask.ConfigureAwait(continueOnCapturedContext: false);
+
+                // remember any skiptoken returned.  This also sets the bool
+                skipHandler.SkipToken = response.Body.NextPageLink;
+
+                // remember the protocol tasks returned
+                base._currentBatch = null;
+
+                if (null != response.Body.GetEnumerator())
+                {
+                    base._currentBatch = response.Body.ToArray();
+                }
+            }
+            // it is possible for there to be no results so we keep trying
+            while (skipHandler.ThereIsMoreData && ((null == _currentBatch) || _currentBatch.Length <= 0));
+        }
+    }
+}

src/SDKs/Batch/DataPlane/Azure.Batch/Azure.Batch.csproj

@@ -10,7 +10,7 @@
     <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
     <PackageId>Azure.Batch</PackageId>
     <Description>This client library provides access to the Microsoft Azure Batch service.</Description>
-    <VersionPrefix>8.0.1</VersionPrefix>
+    <VersionPrefix>8.1.0</VersionPrefix>
     <DefineConstants>$(DefineConstants);CODESIGN</DefineConstants>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <AssemblyName>Microsoft.Azure.Batch</AssemblyName>

src/SDKs/Batch/DataPlane/Azure.Batch/ComputeNode.cs

@@ -375,9 +375,84 @@ public void DisableScheduling(
             asyncTask.WaitAndUnaggregateException(this.CustomBehaviors, additionalBehaviors);
         }
 
-#endregion ComputeNode
+        /// <summary>
+        /// Upload Azure Batch service log files from the compute node.
+        /// </summary>
+        /// <param name="containerUrl">
+        /// The URL of the container within Azure Blob Storage to which to upload the Batch Service log file(s). The URL must include a Shared Access Signature (SAS) granting write permissions to the container.
+        /// </param>
+        /// <param name="startTime">
+        /// The start of the time range from which to upload Batch Service log file(s). Any log file containing a log message in the time range will be uploaded.
+        /// This means that the operation might retrieve more logs than have been requested since the entire log file is always uploaded.
+        /// </param>
+        /// <param name="endTime">
+        /// The end of the time range from which to upload Batch Service log file(s). Any log file containing a log message in the time range will be uploaded.
+        /// This means that the operation might retrieve more logs than have been requested since the entire log file is always uploaded. If this is omitted, the default is the current time.
+        /// </param>
+        /// <param name="additionalBehaviors">A collection of <see cref="BatchClientBehavior"/> instances that are applied to the Batch service request after the <see cref="CustomBehaviors"/>.</param>
+        /// <param name="cancellationToken">A <see cref="CancellationToken"/> for controlling the lifetime of the asynchronous operation.</param>
+        /// <returns>A <see cref="System.Threading.Tasks.Task"/> that represents the asynchronous operation.</returns>
+        /// <remarks>
+        /// This is for gathering Azure Batch service log files in an automated fashion from nodes if you are experiencing an error and wish to escalate to Azure support.
+        /// The Azure Batch service log files should be shared with Azure support to aid in debugging issues with the Batch service.
+        /// </remarks>
+        public System.Threading.Tasks.Task<UploadBatchServiceLogsResult> UploadComputeNodeBatchServiceLogsAsync(
+            string containerUrl,
+            DateTime startTime,
+            DateTime? endTime = null,
+            IEnumerable<BatchClientBehavior> additionalBehaviors = null,
+            CancellationToken cancellationToken = default(CancellationToken))
+        {
+            // craft the behavior manager for this call
+            BehaviorManager bhMgr = new BehaviorManager(this.CustomBehaviors, additionalBehaviors);
+
+            return this.parentBatchClient.PoolOperations.UploadComputeNodeBatchServiceLogsAsyncImpl(
+                this.parentPoolId,
+                this.Id,
+                containerUrl,
+                startTime,
+                endTime,
+                bhMgr,
+                cancellationToken);
+        }
+
+        /// <summary>
+        /// Upload Azure Batch service log files from the specified compute node.
+        /// </summary>
+        /// <param name="containerUrl">
+        /// The URL of the container within Azure Blob Storage to which to upload the Batch Service log file(s). The URL must include a Shared Access Signature (SAS) granting write permissions to the container.
+        /// </param>
+        /// <param name="startTime">
+        /// The start of the time range from which to upload Batch Service log file(s). Any log file containing a log message in the time range will be uploaded.
+        /// This means that the operation might retrieve more logs than have been requested since the entire log file is always uploaded.
+        /// </param>
+        /// <param name="endTime">
+        /// The end of the time range from which to upload Batch Service log file(s). Any log file containing a log message in the time range will be uploaded.
+        /// This means that the operation might retrieve more logs than have been requested since the entire log file is always uploaded. If this is omitted, the default is the current time.
+        /// </param>
+        /// <param name="additionalBehaviors">A collection of <see cref="BatchClientBehavior"/> instances that are applied to the Batch service request after the <see cref="CustomBehaviors"/>.</param>
+        /// <remarks>
+        /// This is for gathering Azure Batch service log files in an automated fashion from nodes if you are experiencing an error and wish to escalate to Azure support.
+        /// The Azure Batch service log files should be shared with Azure support to aid in debugging issues with the Batch service.
+        /// </remarks>
+        /// <returns>The result of uploading the batch service logs.</returns>
+        public UploadBatchServiceLogsResult UploadComputeNodeBatchServiceLogs(
+            string containerUrl,
+            DateTime startTime,
+            DateTime? endTime = null,
+            IEnumerable<BatchClientBehavior> additionalBehaviors = null)
+        {
+            var asyncTask = this.UploadComputeNodeBatchServiceLogsAsync(
+                containerUrl,
+                startTime,
+                endTime,
+                additionalBehaviors);
+            return asyncTask.WaitAndUnaggregateException(this.CustomBehaviors, additionalBehaviors);
+        }
+
+        #endregion ComputeNode
 
-#region IRefreshable
+        #region IRefreshable
 
         /// <summary>
         /// Refreshes the current <see cref="ComputeNode"/>.