[Python] Support for paging with fragment URLs

src/client/Python/msrest/msrest/paging.py

@@ -25,10 +25,6 @@
 # --------------------------------------------------------------------------
 
 import collections
-try:
- from urlparse import urlparse
-except ImportError:
- from urllib.parse import urlparse
 
 from .serialization import Deserializer
 from .pipeline import ClientRawResponse
@@ -76,13 +72,6 @@ def raw(self):
  raw.add_headers(self._raw_headers)
  return raw
 
- def _validate_url(self):
- """Validate next page URL."""
- if self.next_link:
- parsed = urlparse(self.next_link)
- if not parsed.scheme or not parsed.netloc:
- raise ValueError("Invalid URL: " + self.next_link)
-
  def get(self, url):
  """Get arbitrary page.
 
@@ -100,7 +89,6 @@ def next(self):
  """Get next page."""
  if self.next_link is None:
  raise GeneratorExit("End of paging")
- self._validate_url()
  self._response = self._get_next(self.next_link)
  self._derserializer(self, self._response)
  return self.current_page

src/generator/AutoRest.Python.Azure.Tests/AcceptanceTests/paging_tests.py

@@ -145,7 +145,7 @@ def test_paging_sad_path(self):
 
  pages = self.client.paging.get_multiple_pages_failure_uri()
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(CloudError):
  items = [i for i in pages]
 
  pages = self.client.paging.get_single_pages_failure(raw=True)
@@ -160,9 +160,19 @@ def test_paging_sad_path(self):
 
  pages = self.client.paging.get_multiple_pages_failure_uri(raw=True)
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(CloudError):
  items = [i for i in pages]
 
+ def test_paging_fragment_path(self):
+
+ pages = self.client.paging.get_multiple_pages_fragment_next_link("1.6", "test_user")
+ items = [i for i in pages]
+ self.assertIsNone(pages.next_link)
+ self.assertEqual(len(items), 10)
+
+ with self.assertRaises(AttributeError):
+ self.client.paging.get_multiple_pages_fragment_next_link_next()
+
 
 if __name__ == '__main__':
  unittest.main()

src/generator/AutoRest.Python.Azure.Tests/Expected/AcceptanceTests/Paging/autorestpagingtestservice/operations/paging_operations.py

@@ -649,78 +649,15 @@ def internal_paging(next_link=None, raw=False):
  query_parameters['api_version'] = self._serialize.query("api_version", api_version, 'str')
 
  else:
- url = next_link
- query_parameters = {}
-
- # Construct headers
- header_parameters = {}
- header_parameters['Content-Type'] = 'application/json; charset=utf-8'
- if self.config.generate_client_request_id:
- header_parameters['x-ms-client-request-id'] = str(uuid.uuid1())
- if custom_headers:
- header_parameters.update(custom_headers)
- if self.config.accept_language is not None:
- header_parameters['accept-language'] = self._serialize.header("self.config.accept_language", self.config.accept_language, 'str')
-
- # Construct and send request
- request = self._client.get(url, query_parameters)
- response = self._client.send(
- request, header_parameters, **operation_config)
-
- if response.status_code not in [200]:
- exp = CloudError(response)
- exp.request_id = response.headers.get('x-ms-request-id')
- raise exp
-
- return response
-
- # Deserialize response
- deserialized = models.ProductPaged1(internal_paging, self._deserialize.dependencies)
-
- if raw:
- header_dict = {}
- client_raw_response = models.ProductPaged1(internal_paging, self._deserialize.dependencies, header_dict)
- return client_raw_response
-
- return deserialized
-
- def next_fragment(
- self, api_version, tenant, next_link, custom_headers=None, raw=False, **operation_config):
- """A paging operation that doesn't return a full URL, just a fragment.
-
- :param api_version: Sets the api version to use.
- :type api_version: str
- :param tenant: Sets the tenant to use.
- :type tenant: str
- :param next_link: Next link for list operation.
- :type next_link: str
- :param dict custom_headers: headers that will be added to the request
- :param bool raw: returns the direct response alongside the
- deserialized response
- :param operation_config: :ref:`Operation configuration
- overrides<msrest:optionsforoperations>`.
- :rtype: :class:`ProductPaged1
- <Fixtures.AcceptanceTestsPaging.models.ProductPaged1>`
- """
- def internal_paging(next_link=None, raw=False):
-
- if not next_link:
- # Construct URL
  url = '/paging/multiple/fragment/{tenant}/{nextLink}'
  path_format_arguments = {
  'tenant': self._serialize.url("tenant", tenant, 'str'),
  'nextLink': self._serialize.url("next_link", next_link, 'str', skip_quote=True)
  }
  url = self._client.format_url(url, **path_format_arguments)
-
- # Construct parameters
  query_parameters = {}
  query_parameters['api_version'] = self._serialize.query("api_version", api_version, 'str')
 
- else:
- url = next_link
- query_parameters = {}
-
  # Construct headers
  header_parameters = {}
  header_parameters['Content-Type'] = 'application/json; charset=utf-8'

src/generator/AutoRest.Python.Azure/TemplateModels/AzureMethodGroupTemplateModel.cs

@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System;
 using System.Linq;
+using System.Collections.Generic;
 using AutoRest.Core.ClientModel;
 using AutoRest.Core.Utilities;
 using AutoRest.Extensions.Azure;
@@ -17,6 +19,26 @@ public AzureMethodGroupTemplateModel(ServiceClient serviceClient, string methodG
  // Clear base initialized MethodTemplateModels and re-populate with
  // AzureMethodTemplateModel
  MethodTemplateModels.Clear();
+
+ var currentMethods = Methods.Where(m => m.Group == MethodGroupName && m.Extensions.ContainsKey(AzureExtensions.PageableExtension));
+ var nextListMethods = new List<Method>();
+ foreach (var method in currentMethods)
+ {
+ var pageableExtension = method.Extensions[AzureExtensions.PageableExtension] as Newtonsoft.Json.Linq.JContainer;
+ var operationName = (string)pageableExtension["operationName"];
+ if (operationName != null)
+ {
+ var nextLinkMethod = Methods.FirstOrDefault(m =>
+ operationName.Equals(m.SerializedName, StringComparison.OrdinalIgnoreCase));
+ if (nextLinkMethod != null)
+ {
+ method.Extensions["nextLinkURL"] = nextLinkMethod.Url;
+ method.Extensions["nextLinkParameters"] = nextLinkMethod.LogicalParameters;
+ nextListMethods.Add(nextLinkMethod);
+ }
+ }
+ }
+ Methods.RemoveAll(m => nextListMethods.Contains(m));
  Methods.Where(m => m.Group == methodGroupName)
  .ForEach(m => MethodTemplateModels.Add(new AzureMethodTemplateModel(m, serviceClient)));
  }

src/generator/AutoRest.Python.Azure/TemplateModels/AzureMethodTemplateModel.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Linq;
 using System.Net;
+using System.Collections.Generic;
 using AutoRest.Core.ClientModel;
 using AutoRest.Core.Utilities;
 using AutoRest.Extensions.Azure;
@@ -33,6 +34,30 @@ public bool IsPagingMethod
  }
  }
 
+ public string PagingURL
+ {
+ get
+ {
+ if (this.Extensions.ContainsKey("nextLinkURL"))
+ {
+ return this.Extensions["nextLinkURL"] as string;
+ }
+ return null;
+ }
+ }
+
+ public IEnumerable<Parameter> PagingParameters
+ {
+ get
+ {
+ if (this.Extensions.ContainsKey("nextLinkParameters"))
+ {
+ return this.Extensions["nextLinkParameters"] as IEnumerable<Parameter>;
+ }
+ return null;
+ }
+ }
+
  public string PagedResponseClassName
  {
  get

src/generator/AutoRest.Python.Azure/Templates/AzureLongRunningMethodTemplate.cshtml

@@ -41,11 +41,11 @@
 }
  # Construct URL
  url = '@(Model.Url)'
- @(Model.BuildUrlPath("url"))
+ @(Model.BuildUrlPath("url", Model.LogicalParameters))
 @EmptyLine
  # Construct parameters
  query_parameters = {}
- @(Model.BuildUrlQuery("query_parameters"))
+ @(Model.BuildUrlQuery("query_parameters", Model.LogicalParameters))
 @EmptyLine
  # Construct headers
  header_parameters = {}

src/generator/AutoRest.Python.Azure/Templates/AzurePagingMethodTemplate.cshtml

@@ -37,15 +37,24 @@
  if not next_link:
  # Construct URL
  url = '@(Model.Url)'
- @(Model.BuildUrlPath("url"))
+ @(Model.BuildUrlPath("url", Model.LogicalParameters))
 @EmptyLine
  # Construct parameters
  query_parameters = {}
- @(Model.BuildUrlQuery("query_parameters"))
+ @(Model.BuildUrlQuery("query_parameters", Model.LogicalParameters))
 @EmptyLine
  else:
- url = next_link
+@if (Model.PagingURL != null)
+{
+ @:url = '@(Model.PagingURL)'
+}
+else
+{
+ @:url = next_link
+}
+ @(Model.BuildUrlPath("url", Model.PagingParameters))
  query_parameters = {}
+ @(Model.BuildUrlQuery("query_parameters", Model.PagingParameters))
 @EmptyLine
  # Construct headers
  header_parameters = {}

src/generator/AutoRest.Python/TemplateModels/MethodTemplateModel.cs

@@ -273,25 +273,28 @@ private static string BuildValidationParameters(Dictionary<Constraint, string> c
  /// Generate code to build the URL from a url expression and method parameters
  /// </summary>
  /// <param name="variableName">The variable to store the url in.</param>
+ /// <param name="pathParameters">The list of parameters for url construction.</param>
  /// <returns></returns>
- [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Globalization", "CA1303:Do not pass literals as localized parameters", MessageId = "AutoRest.Core.Utilities.IndentedStringBuilder.AppendLine(System.String)"), System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "pathformatarguments"), System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1055:UriReturnValuesShouldNotBeStrings")]
- public virtual string BuildUrlPath(string variableName)
+ [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Globalization", "CA1303:Do not pass literals as localized parameters", MessageId = "Microsoft.Rest.Generator.Utilities.IndentedStringBuilder.AppendLine(System.String)"), System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA2204:Literals should be spelled correctly", MessageId = "pathformatarguments"), System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1055:UriReturnValuesShouldNotBeStrings")]
+ public virtual string BuildUrlPath(string variableName, IEnumerable<Parameter> pathParameters)
  {
  var builder = new IndentedStringBuilder(" ");
+ if (pathParameters == null)
+ return builder.ToString();
 
- var pathParameterList = this.LogicalParameters.Where(p => p.Location == ParameterLocation.Path).ToList();
+ var pathParameterList = pathParameters.Where(p => p.Location == ParameterLocation.Path).ToList();
  if (pathParameterList.Any())
  {
  builder.AppendLine("path_format_arguments = {").Indent();
 
- for (int i = 0; i < pathParameterList.Count; i ++)
+ for (int i = 0; i < pathParameterList.Count; i++)
  {
  builder.AppendLine("'{0}': {1}{2}{3}",
  pathParameterList[i].SerializedName,
  BuildSerializeDataCall(pathParameterList[i], "url"),
  pathParameterList[i].IsRequired ? string.Empty :
  string.Format(CultureInfo.InvariantCulture, "if {0} else ''", pathParameterList[i].Name),
- i == pathParameterList.Count-1 ? "" : ",");
+ i == pathParameterList.Count - 1 ? "" : ",");
  }
 
  builder.Outdent().AppendLine("}");
@@ -305,13 +308,16 @@ public virtual string BuildUrlPath(string variableName)
  /// Generate code to build the query of URL from method parameters
  /// </summary>
  /// <param name="variableName">The variable to store the query in.</param>
+ /// <param name="queryParameters">The list of parameters for url construction.</param>
  /// <returns></returns>
  [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1055:UriReturnValuesShouldNotBeStrings")]
- public virtual string BuildUrlQuery(string variableName)
+ public virtual string BuildUrlQuery(string variableName, IEnumerable<Parameter> queryParameters)
  {
  var builder = new IndentedStringBuilder(" ");
+ if (queryParameters == null)
+ return builder.ToString();
 
- foreach (var queryParameter in this.LogicalParameters.Where(p => p.Location == ParameterLocation.Query))
+ foreach (var queryParameter in queryParameters.Where(p => p.Location == ParameterLocation.Query))
  {
  if (queryParameter.IsRequired)
  {

src/generator/AutoRest.Python/Templates/MethodTemplate.cshtml

@@ -41,11 +41,11 @@
 }
  # Construct URL
  url = '@(Model.Url)'
- @(Model.BuildUrlPath("url"))
+ @(Model.BuildUrlPath("url", Model.LogicalParameters))
 @EmptyLine
  # Construct parameters
  query_parameters = {}
- @(Model.BuildUrlQuery("query_parameters"))
+ @(Model.BuildUrlQuery("query_parameters", Model.LogicalParameters))
 @EmptyLine
  # Construct headers
  header_parameters = {}