Document streaming blob type

Not having streaming blobs documented is causing
users to pass in the wrong input for blob arguments.
This commit resolves the issue by explicitly marking
streaming blob types and auto-generating a usage note for
streaming blobs.

.changes/next-release/enhancement-docs-84777.json

@@ -0,0 +1,5 @@
+{
+ "type": "enhancement",
+ "category": "docs",
+ "description": "Differentiate between regular and streaming blobs and generate a usage note when a parameter is of streaming blob type."
+}

awscli/clidocs.py

@@ -22,7 +22,7 @@
 from awscli.topictags import TopicTagDB
 from awscli.utils import (
  find_service_and_method_in_event_name, is_document_type,
- operation_uses_document_types
+ operation_uses_document_types, is_streaming_blob_type
 )
 
 LOG = logging.getLogger(__name__)
@@ -48,6 +48,8 @@ def _get_argument_type_name(self, shape, default):
  return 'JSON'
  if is_document_type(shape):
  return 'document'
+ if is_streaming_blob_type(shape):
+ return 'streaming blob'
  return default
 
  def _map_handlers(self, session, event_class, mapfn):
@@ -173,6 +175,8 @@ def doc_option(self, arg_name, help_command, **kwargs):
  argument.argument_model, argument.cli_type_name)))
  doc.style.indent()
  doc.include_doc_string(argument.documentation)
+ if is_streaming_blob_type(argument.argument_model):
+ self._add_streaming_blob_note(doc)
  if hasattr(argument, 'argument_model'):
  self._document_enums(argument.argument_model, doc)
  self._document_nested_structure(argument.argument_model, doc)
@@ -264,6 +268,15 @@ def _do_doc_member(self, doc, member_name, member_shape, stack):
  doc.style.dedent()
  doc.style.new_paragraph()
 
+ def _add_streaming_blob_note(self, doc):
+ doc.style.start_note()
+ msg = ("This argument is of type: streaming blob. "
+ "Its value must be the path to a file "
+ "(e.g. ``path/to/file``) and must **not** "
+ "be prefixed with ``file://`` or ``fileb://``")
+ doc.writeln(msg)
+ doc.style.end_note()
+
 
 class ProviderDocumentEventHandler(CLIDocumentEventHandler):
 

awscli/utils.py

@@ -153,6 +153,12 @@ def is_document_type_container(shape):
  return True
 
 
+def is_streaming_blob_type(shape):
+ """Check if the shape is a streaming blob type."""
+ return (shape and shape.type_name == 'blob' and
+ shape.serialization.get('streaming', False))
+
+
 def operation_uses_document_types(operation_model):
  """Check if document types are ever used in the operation"""
  recording_visitor = ShapeRecordingVisitor()

tests/unit/test_clidocs.py

@@ -13,7 +13,7 @@
 import json
 
 from botocore.model import ShapeResolver, StructureShape, StringShape, \
- ListShape, MapShape
+ ListShape, MapShape, Shape
 
 from awscli.testutils import mock, unittest, FileCreator
 from awscli.clidocs import OperationDocumentEventHandler, \
@@ -22,6 +22,7 @@
 from awscli.bcdoc.restdoc import ReSTDocument
 from awscli.help import ServiceHelpCommand, TopicListerCommand, \
  TopicHelpCommand
+from awscli.arguments import CustomArgument
 
 
 class TestRecursiveShapes(unittest.TestCase):
@@ -406,6 +407,32 @@ def test_includes_global_args_ref_in_html_options(self):
  "global parameters", rendered
  )
 
+ def test_includes_streaming_blob_options(self):
+ help_command = self.create_help_command()
+ blob_shape = Shape('blob_shape', {'type': 'blob'})
+ blob_shape.serialization = {'streaming': True}
+ blob_arg = CustomArgument('blob_arg', argument_model=blob_shape)
+ help_command.arg_table = {'blob_arg': blob_arg}
+ operation_handler = OperationDocumentEventHandler(help_command)
+ operation_handler.doc_option(arg_name='blob_arg',
+ help_command=help_command)
+ rendered = help_command.doc.getvalue().decode('utf-8')
+ self.assertIn('streaming blob', rendered)
+
+ def test_streaming_blob_comes_after_docstring(self):
+ help_command = self.create_help_command()
+ blob_shape = Shape('blob_shape', {'type': 'blob'})
+ blob_shape.serialization = {'streaming': True}
+ blob_arg = CustomArgument(name='blob_arg',
+ argument_model=blob_shape,
+ help_text='FooBar')
+ help_command.arg_table = {'blob_arg': blob_arg}
+ operation_handler = OperationDocumentEventHandler(help_command)
+ operation_handler.doc_option(arg_name='blob_arg',
+ help_command=help_command)
+ rendered = help_command.doc.getvalue().decode('utf-8')
+ self.assertRegex(rendered, r'FooBar[\s\S]*streaming blob')
+
 
 class TestTopicDocumentEventHandlerBase(unittest.TestCase):
  def setUp(self):

tests/unit/test_utils.py

@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 import signal
 import platform
+import pytest
 import subprocess
 import os
 
@@ -20,12 +21,17 @@
 from awscli.testutils import unittest, skip_if_windows, mock
 from awscli.utils import (
  split_on_commas, ignore_ctrl_c, find_service_and_method_in_event_name,
- is_document_type, is_document_type_container,
+ is_document_type, is_document_type_container, is_streaming_blob_type,
  operation_uses_document_types, ShapeWalker, ShapeRecordingVisitor,
  OutputStreamFactory
 )
 
 
+@pytest.fixture()
+def argument_model():
+ return botocore.model.Shape('argument', {'type': 'string'})
+
+
 class TestCSVSplit(unittest.TestCase):
 
  def test_normal_csv_split(self):
@@ -409,3 +415,21 @@ def test_can_escape_recursive_shapes(self):
  }
  self.walker.walk(self.get_shape_model('Recursive'), self.visitor)
  self.assert_visited_shapes(['Recursive'])
+
+
+@pytest.mark.usefixtures('argument_model')
+class TestStreamingBlob:
+ def test_blob_is_streaming(self, argument_model):
+ argument_model.type_name = 'blob'
+ argument_model.serialization = {'streaming': True}
+ assert is_streaming_blob_type(argument_model)
+
+ def test_blob_is_not_streaming(self, argument_model):
+ argument_model.type_name = 'blob'
+ argument_model.serialization = {}
+ assert not is_streaming_blob_type(argument_model)
+
+ def test_non_blob_is_not_streaming(self, argument_model):
+ argument_model.type_name = 'string'
+ argument_model.serialization = {}
+ assert not is_streaming_blob_type(argument_model)