From 16037475c9593eeb23186642fd0e95057ebcd99e Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Thu, 12 Jan 2023 15:58:42 +0100 Subject: [PATCH 01/11] Fix pytest-asyncio dependency version --- .../amazon/smithy/python/codegen/SmithyPythonDependency.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 2e17da1dd..84a2aeab3 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -52,7 +52,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency PYTEST_ASYNCIO = new PythonDependency( "pytest-asyncio", - ">=0.2.3,<0.3.0", + ">=0.20.3,<0.30.0", Type.TEST_DEPENDENCY, false ); From ce20d33fcf35ed4699bb097552db10329d8b8ede Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Wed, 4 Jan 2023 16:28:28 +0100 Subject: [PATCH 02/11] Merge urls rather than overwriting --- .../amazon/smithy/python/codegen/ClientGenerator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 935d8b590..45f390998 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -313,6 +313,14 @@ async def _handle_attempt( endpoint = await config.endpoint_resolver.resolve_endpoint( StaticEndpointParams(url=config.endpoint_url) ) + if not endpoint.url.path: + endpoint.url.path = "" + elif endpoint.url.path.endswith("/"): + endpoint.url.path = endpoint.url.path.rstrip("/") + if context.transport_request.url.path: + endpoint.url.path += context.transport_request.url.path + endpoint.url.query = context.transport_request.url.query + endpoint.url.host = context.transport_request.url.host + endpoint.url.host context._transport_request.url = endpoint.url context._transport_request.headers.extend(endpoint.headers) From 63bd9345676451ef33f85c755bfb47ddcb292f8c Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Thu, 12 Jan 2023 17:11:47 +0100 Subject: [PATCH 03/11] Use ShapeVisistor for http member serialization --- .../HttpBindingProtocolGenerator.java | 263 ++++++++++-------- 1 file changed, 149 insertions(+), 114 deletions(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java index 6fe8c17ea..e8adfb881 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java @@ -46,7 +46,6 @@ import software.amazon.smithy.model.shapes.ListShape; import software.amazon.smithy.model.shapes.LongShape; import software.amazon.smithy.model.shapes.MemberShape; -import software.amazon.smithy.model.shapes.NumberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.ShapeVisitor; @@ -269,9 +268,10 @@ private void serializePath( var dataSource = "input." + memberName; var target = context.model().expectShape(httpBinding.getMember().getTarget()); - writer.write("$1L=urlquote($3L$2L),", memberName, urlSafe, getInputValue( - context, writer, httpBinding.getLocation(), dataSource, httpBinding.getMember(), target - )); + var inputValue = target.accept(new HttpMemberSerVisitor( + context, writer, httpBinding.getLocation(), dataSource, httpBinding.getMember(), + getDocumentTimestampFormat())); + writer.write("$1L=urlquote($3L$2L),", memberName, urlSafe, inputValue); } }); @@ -422,116 +422,6 @@ protected void serializePayloadBody( // TODO: implement this } - /** - * Given context and a source of data, generate an input value provider for the - * shape. This may use native types or invoke complex type serializers to - * manipulate the dataSource into the proper input content. - * - * @param context The generation context. - * @param writer The writer this value will be written to. Used only to add - * imports, if necessary. - * @param bindingType How this value is bound to the operation input. - * @param dataSource The in-code location of the data to provide an input of - * ({@code input.foo}, {@code entry}, etc.) - * @param member The member that points to the value being provided. - * @param target The shape of the value being provided. - * @return Returns a value or expression of the input value. - */ - protected String getInputValue( - GenerationContext context, - PythonWriter writer, - HttpBinding.Location bindingType, - String dataSource, - MemberShape member, - Shape target - ) { - if (target.isStringShape()) { - return getStringInputParam(context, writer, bindingType, dataSource, member, target); - } else if (target.isFloatShape() || target.isDoubleShape()) { - // Using float ensures we get a decimal even if there is no fraction given - // e.g. 1 => 1.0 - return String.format("str(float(%s))", dataSource); - } else if (target instanceof NumberShape) { - return String.format("str(%s)", dataSource); - } else if (target.isBooleanShape()) { - return String.format("('true' if %s else 'false')", dataSource); - } else if (target.isTimestampShape()) { - return getTimestampInputParam(context, writer, bindingType, dataSource, member, target); - } else { - // TODO: add support here for other shape types - return dataSource; - } -// throw new CodegenException(String.format( -// "Unsupported %s binding of %s to %s in %s using the %s protocol", -// bindingType, member.getMemberName(), target.getType(), member.getContainer(), getName())); - } - - /** - * Given context and a source of data, generate an input value provider for a - * string. By default, this base64 encodes content in headers if there is a - * mediaType applied to the string, and passes through for all other cases. - * - * @param context The generation context. - * @param writer The writer this value will be written to. Used only to add - * imports, if necessary. - * @param bindingType How this value is bound to the operation input. - * @param dataSource The in-code location of the data to provide an input of - * ({@code input.foo}, {@code entry}, etc.) - * @param target The shape of the value being provided. - * @return Returns a value or expression of the input string. - */ - protected String getStringInputParam( - GenerationContext context, - PythonWriter writer, - HttpBinding.Location bindingType, - String dataSource, - MemberShape member, - Shape target - ) { - if (bindingType == Location.HEADER) { - if (target.hasTrait(MediaTypeTrait.class)) { - writer.addStdlibImport("base64", "b64encode"); - return "b64encode(" + dataSource + "))"; - } - } - return dataSource; - } - - /** - * Given context and a source of data, generate an input value provider for the - * shape. This uses the format specified, converting to strings when in a header, - * label, or query string. - * - * @param context The generation context. - * @param writer The writer this value will be written to. Used only to add - * imports, if necessary. - * @param bindingType How this value is bound to the operation input. - * @param dataSource The in-code location of the data to provide an input of - * ({@code input.foo}, {@code entry}, etc.) - * @param member The member that points to the value being provided. - * @return Returns a value or expression of the input shape. - */ - protected String getTimestampInputParam( - GenerationContext context, - PythonWriter writer, - HttpBinding.Location bindingType, - String dataSource, - MemberShape member, - Shape target - ) { - var httpIndex = HttpBindingIndex.of(context.model()); - var format = switch (bindingType) { - case HEADER -> httpIndex.determineTimestampFormat(member, bindingType, Format.HTTP_DATE); - case LABEL -> httpIndex.determineTimestampFormat(member, bindingType, getDocumentTimestampFormat()); - case QUERY -> httpIndex.determineTimestampFormat(member, bindingType, Format.DATE_TIME); - default -> - throw new CodegenException("Unexpected named member shape binding location `" + bindingType + "`"); - }; - - return HttpProtocolGeneratorUtils.getTimestampInputParam( - context, writer, dataSource, member, format); - } - @Override public void generateResponseDeserializers(GenerationContext context) { // TODO: Generate deserializers for http bindings, e.g. non-body parts of the http response @@ -816,6 +706,151 @@ protected void deserializePayloadBody( // This will have a default implementation since it'll mostly be standard } + /** + * Given context and a source of data, generate an input value provider for the + * shape. This may use native types or invoke complex type serializers to + * manipulate the dataSource into the proper input content. + */ + private static class HttpMemberSerVisitor extends ShapeVisitor.Default { + private final GenerationContext context; + private final PythonWriter writer; + private final String dataSource; + private final Location bindingType; + private final MemberShape member; + private final Format defaultTimestampFormat; + + /** + * @param context The generation context. + * @param writer The writer to add dependencies to. + * @param bindingType How this value is bound to the operation input. + * @param dataSource The in-code location of the data to provide an output of + * ({@code input.foo}, {@code entry}, etc.) + * @param member The member that points to the value being provided. + * @param defaultTimestampFormat The default timestamp format to use. + */ + HttpMemberSerVisitor( + GenerationContext context, + PythonWriter writer, + Location bindingType, + String dataSource, + MemberShape member, + Format defaultTimestampFormat + ) { + this.context = context; + this.writer = writer; + this.dataSource = dataSource; + this.bindingType = bindingType; + this.member = member; + this.defaultTimestampFormat = defaultTimestampFormat; + } + + @Override + protected String getDefault(Shape shape) { + var protocolName = context.protocolGenerator().getName(); + throw new CodegenException(String.format( + "Unsupported %s binding of %s to %s in %s using the %s protocol", + bindingType, member.getMemberName(), shape.getType(), member.getContainer(), protocolName)); + } + + @Override + public String blobShape(BlobShape shape) { + // TODO: implement this + return dataSource; + } + + @Override + public String booleanShape(BooleanShape shape) { + return String.format("('true' if %s else 'false')", dataSource); + } + + @Override + public String stringShape(StringShape shape) { + if (bindingType == Location.HEADER) { + if (shape.hasTrait(MediaTypeTrait.class)) { + writer.addStdlibImport("base64", "b64encode"); + return "b64encode(" + dataSource + "))"; + } + } + return dataSource; + } + + @Override + public String byteShape(ByteShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String shortShape(ShortShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String integerShape(IntegerShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String longShape(LongShape shape) { + // TODO: perform bounds checks + return integerShape(); + } + + @Override + public String bigIntegerShape(BigIntegerShape shape) { + return integerShape(); + } + + private String integerShape() { + return String.format("str(%s)", dataSource); + } + + @Override + public String floatShape(FloatShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + @Override + public String doubleShape(DoubleShape shape) { + // TODO: use strict parsing + return floatShapes(); + } + + private String floatShapes() { + return String.format("str(float(%s))", dataSource); + } + + + @Override + public String bigDecimalShape(BigDecimalShape shape) { + return String.format("str(%s)", dataSource); + } + + @Override + public String timestampShape(TimestampShape shape) { + var httpIndex = HttpBindingIndex.of(context.model()); + var format = switch (bindingType) { + case HEADER -> httpIndex.determineTimestampFormat(member, bindingType, Format.HTTP_DATE); + case LABEL -> httpIndex.determineTimestampFormat(member, bindingType, defaultTimestampFormat); + case QUERY -> httpIndex.determineTimestampFormat(member, bindingType, Format.DATE_TIME); + default -> + throw new CodegenException("Unexpected named member shape binding location `" + bindingType + "`"); + }; + + return HttpProtocolGeneratorUtils.getTimestampInputParam( + context, writer, dataSource, member, format); + } + + @Override + public String listShape(ListShape shape) { + // TODO: implement this + return dataSource; + } + } + /** * Given context and a source of data, generate an output value provider for the * shape. This may use native types (like generating a datetime for timestamps) From d24c2c954142341fcc5bb4761166d622a53a93e4 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Fri, 13 Jan 2023 12:55:51 +0100 Subject: [PATCH 04/11] Fix non-numeric float serialization --- .../HttpBindingProtocolGenerator.java | 13 ++++++----- .../smithy-python/smithy_python/utils.py | 22 ++++++++++++++++++ .../smithy-python/tests/unit/test_utils.py | 23 +++++++++++++++++++ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java index e8adfb881..8777f7e36 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpBindingProtocolGenerator.java @@ -819,14 +819,15 @@ public String doubleShape(DoubleShape shape) { return floatShapes(); } - private String floatShapes() { - return String.format("str(float(%s))", dataSource); - } - - @Override public String bigDecimalShape(BigDecimalShape shape) { - return String.format("str(%s)", dataSource); + return floatShapes(); + } + + private String floatShapes() { + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_float"); + return String.format("serialize_float(%s)", dataSource); } @Override diff --git a/python-packages/smithy-python/smithy_python/utils.py b/python-packages/smithy-python/smithy_python/utils.py index 186bf92c5..270dceb24 100644 --- a/python-packages/smithy-python/smithy_python/utils.py +++ b/python-packages/smithy-python/smithy_python/utils.py @@ -1,5 +1,7 @@ import re from datetime import datetime, timezone +from decimal import Decimal +from math import isinf, isnan from types import UnionType from typing import Any, TypeVar, overload @@ -144,3 +146,23 @@ def strict_parse_float(given: str) -> float: if _FLOAT_REGEX.fullmatch(given): return float(given) raise ExpectationNotMetException(f"Expected float, found: {given}") + + +def serialize_float(given: float | Decimal) -> str: + """Serializes a float to a string. + + This ensures non-numeric floats are serialized correctly, and ensures that there is + a fractional part. + + :param given: A float or Decimal to be serialized. + :returns: The string representation of the given float. + """ + if isnan(given): + return "NaN" + if isinf(given): + return "-Infinity" if given < 0 else "Infinity" + + result = str(given) + if "." not in result: + result += ".0" + return result diff --git a/python-packages/smithy-python/tests/unit/test_utils.py b/python-packages/smithy-python/tests/unit/test_utils.py index b2c8cb06e..d2cc07a19 100644 --- a/python-packages/smithy-python/tests/unit/test_utils.py +++ b/python-packages/smithy-python/tests/unit/test_utils.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone +from decimal import Decimal from math import isnan from typing import Any @@ -9,6 +10,7 @@ ensure_utc, expect_type, limited_parse_float, + serialize_float, strict_parse_bool, strict_parse_float, ) @@ -139,3 +141,24 @@ def test_strict_parse_float_nan() -> None: def test_strict_parse_float_raises(given: str) -> None: with pytest.raises(ExpectationNotMetException): strict_parse_float(given) + + +@pytest.mark.parametrize( + "given, expected", + [ + (1, "1.0"), + (1.0, "1.0"), + (1.1, "1.1"), + (float("NaN"), "NaN"), + (float("Infinity"), "Infinity"), + (float("-Infinity"), "-Infinity"), + (Decimal("1"), "1.0"), + (Decimal("1.0"), "1.0"), + (Decimal("1.1"), "1.1"), + (Decimal("NaN"), "NaN"), + (Decimal("Infinity"), "Infinity"), + (Decimal("-Infinity"), "-Infinity"), + ], +) +def test_serialize_float(given: float | Decimal, expected: str) -> None: + assert serialize_float(given) == expected From 9f265ffbc1400ed995a13cffb03fc6e3995b9551 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Fri, 13 Jan 2023 13:19:18 +0100 Subject: [PATCH 05/11] Fix RFC3339 serialization --- .../integration/HttpProtocolGeneratorUtils.java | 5 ++++- .../smithy-python/smithy_python/utils.py | 16 ++++++++++++++++ .../smithy-python/tests/unit/test_utils.py | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java index c1760c833..7925c9fb8 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java @@ -20,6 +20,7 @@ import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonWriter; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; import software.amazon.smithy.utils.SmithyUnstableApi; /** @@ -55,7 +56,9 @@ public static String getTimestampInputParam( var result = "ensure_utc(" + dataSource + ")"; switch (format) { case DATE_TIME: - return result + ".isoformat()"; + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_rfc3339"); + return String.format("serialize_rfc3339(%s)", result); case EPOCH_SECONDS: return "str(" + result + ".timestamp())"; case HTTP_DATE: diff --git a/python-packages/smithy-python/smithy_python/utils.py b/python-packages/smithy-python/smithy_python/utils.py index 270dceb24..fb1fc10f4 100644 --- a/python-packages/smithy-python/smithy_python/utils.py +++ b/python-packages/smithy-python/smithy_python/utils.py @@ -7,6 +7,10 @@ from .exceptions import ExpectationNotMetException +RFC3339 = "%Y-%m-%dT%H:%M:%SZ" +# Same as RFC3339, but with microsecond precision. +RFC3339_MICRO = "%Y-%m-%dT%H:%M:%S.%fZ" + def ensure_utc(value: datetime) -> datetime: """Ensures that the given datetime is a UTC timezone-aware datetime. @@ -166,3 +170,15 @@ def serialize_float(given: float | Decimal) -> str: if "." not in result: result += ".0" return result + + +def serialize_rfc3339(given: datetime) -> str: + """Serializes a datetime into an RFC3339 string respresentation. + + :param given: The datetime to serialize. + :returns: An RFC3339 formatted timestamp. + """ + if given.microsecond != 0: + return given.strftime(RFC3339_MICRO) + else: + return given.strftime(RFC3339) diff --git a/python-packages/smithy-python/tests/unit/test_utils.py b/python-packages/smithy-python/tests/unit/test_utils.py index d2cc07a19..5c294405f 100644 --- a/python-packages/smithy-python/tests/unit/test_utils.py +++ b/python-packages/smithy-python/tests/unit/test_utils.py @@ -11,6 +11,7 @@ expect_type, limited_parse_float, serialize_float, + serialize_rfc3339, strict_parse_bool, strict_parse_float, ) @@ -162,3 +163,17 @@ def test_strict_parse_float_raises(given: str) -> None: ) def test_serialize_float(given: float | Decimal, expected: str) -> None: assert serialize_float(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [ + (datetime(2017, 1, 1, tzinfo=timezone.utc), "2017-01-01T00:00:00Z"), + ( + datetime(2017, 1, 1, microsecond=1, tzinfo=timezone.utc), + "2017-01-01T00:00:00.000001Z", + ), + ], +) +def test_serialize_rfc3339(given: datetime, expected: str) -> None: + assert serialize_rfc3339(given) == expected From 8e95d14edf07f9d136c8c5d57985bce71942231b Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Fri, 13 Jan 2023 14:11:04 +0100 Subject: [PATCH 06/11] Parse protocol test timestamps in the generator This updates the protocol tests to parse any timestamps inside the generator and output a constructor rather than delegating the parsing to python. This was already being done in the structure generator for default values, so this shares that effort. --- .../smithy/python/codegen/CodegenUtils.java | 79 +++++++++++++++++++ .../codegen/HttpProtocolTestGenerator.java | 4 +- .../python/codegen/StructureGenerator.java | 57 +------------ 3 files changed, 83 insertions(+), 57 deletions(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index f074ec55f..3d1931959 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -22,17 +22,28 @@ import java.io.InputStreamReader; import java.nio.charset.Charset; import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Logger; import software.amazon.smithy.codegen.core.CodegenException; import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.traits.ErrorTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait; +import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; import software.amazon.smithy.utils.SetUtils; import software.amazon.smithy.utils.StringUtils; @@ -209,4 +220,72 @@ public static List toTuples(Map map) { return map.entrySet().stream().map((entry) -> List.of(entry.getKey(), entry.getValue()).toArray()).toList(); } + + /** + * Generates a Python datetime constructor for the given ZonedDateTime. + * + * @param writer A writer to add dependencies to. + * @param value The ZonedDateTime to convert. + * @return A string containing a Python datetime constructor representing the given ZonedDateTime. + */ + public static String getDatetimeConstructor(PythonWriter writer, ZonedDateTime value) { + writer.addStdlibImport("datetime", "datetime"); + writer.addStdlibImport("datetime", "timezone"); + var timezone = "timezone.utc"; + if (value.getOffset() != ZoneOffset.UTC) { + writer.addStdlibImport("datetime", "timedelta"); + timezone = String.format("timezone(timedelta(seconds=%d))", value.getOffset().getTotalSeconds()); + } + return String.format("datetime(%d, %d, %d, %d, %d, %d, %d, %s)", value.get(ChronoField.YEAR), + value.get(ChronoField.MONTH_OF_YEAR), value.get(ChronoField.DAY_OF_MONTH), + value.get(ChronoField.HOUR_OF_DAY), value.get(ChronoField.MINUTE_OF_HOUR), + value.get(ChronoField.SECOND_OF_MINUTE), value.get(ChronoField.MICRO_OF_SECOND), timezone); + } + + /** + * Parses a timestamp Node. + * + *

This is used to offload modeled timestamp parsing from Python runtime to Java build time. + * + * @param model The model being generated. + * @param shape The shape of the node. + * @param value The node to parse. + * @return A parsed ZonedDateTime representation of the given node. + */ + public static ZonedDateTime parseTimestampNode(Model model, Shape shape, Node value) { + if (value.isNumberNode()) { + return parseEpochTime(value); + } + + Optional trait = shape.getTrait(TimestampFormatTrait.class); + if (shape.isMemberShape()) { + trait = shape.asMemberShape().get().getMemberTrait(model, TimestampFormatTrait.class); + } + var format = Format.DATE_TIME; + if (trait.isPresent()) { + format = trait.get().getFormat(); + } + + return switch (format) { + case DATE_TIME -> parseDateTime(value); + case HTTP_DATE -> parseHttpDate(value); + default -> throw new CodegenException("Unexpected timestamp format: " + format); + }; + } + + private static ZonedDateTime parseEpochTime(Node value) { + Number number = value.expectNumberNode().getValue(); + Instant instant = Instant.ofEpochMilli(Double.valueOf(number.doubleValue() * 1000).longValue()); + return instant.atZone(ZoneId.of("UTC")); + } + + private static ZonedDateTime parseDateTime(Node value) { + Instant instant = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(value.expectStringNode().getValue())); + return instant.atZone(ZoneId.of("UTC")); + } + + private static ZonedDateTime parseHttpDate(Node value) { + Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(value.expectStringNode().getValue())); + return instant.atZone(ZoneId.of("UTC")); + } } diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java index 6a0438e29..e1fe0819f 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -479,8 +479,8 @@ public Void nullNode(NullNode node) { public Void numberNode(NumberNode node) { // TODO: Add support for timestamp, int-enum, and others if (inputShape.isTimestampShape()) { - writer.addStdlibImport("datetime", "datetime"); - writer.writeInline("datetime.fromtimestamp($L)", node.getValue()); + var parsed = CodegenUtils.parseTimestampNode(model, inputShape, node); + writer.writeInline(CodegenUtils.getDatetimeConstructor(writer, parsed)); } else if (inputShape.isFloatShape() || inputShape.isDoubleShape()) { writer.writeInline("float($L)", node.getValue()); } else { diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java index 283a1da82..1ce0047e3 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/StructureGenerator.java @@ -18,14 +18,9 @@ import static java.lang.String.format; import static software.amazon.smithy.python.codegen.CodegenUtils.isErrorMessage; -import java.time.Instant; -import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -43,7 +38,6 @@ import software.amazon.smithy.model.traits.ErrorTrait; import software.amazon.smithy.model.traits.RequiredTrait; import software.amazon.smithy.model.traits.SensitiveTrait; -import software.amazon.smithy.model.traits.TimestampFormatTrait; /** @@ -299,15 +293,8 @@ private String getDefaultValue(PythonWriter writer, MemberShape member) { var defaultNode = member.expectTrait(DefaultTrait.class).toNode(); var target = model.expectShape(member.getTarget()); if (target.isTimestampShape()) { - writer.addStdlibImport("datetime", "datetime"); - writer.addStdlibImport("datetime", "timezone"); - // We *could* let python do this parsing, but then that work has to be done every time a customer - // runs their code. - ZonedDateTime value = parseDefaultTimestamp(member, defaultNode); - return String.format("datetime(%d, %d, %d, %d, %d, %d, %d, timezone.utc)", value.get(ChronoField.YEAR), - value.get(ChronoField.MONTH_OF_YEAR), value.get(ChronoField.DAY_OF_MONTH), - value.get(ChronoField.HOUR_OF_DAY), value.get(ChronoField.MINUTE_OF_HOUR), - value.get(ChronoField.SECOND_OF_MINUTE), value.get(ChronoField.MICRO_OF_SECOND)); + ZonedDateTime value = CodegenUtils.parseTimestampNode(model, member, defaultNode); + return CodegenUtils.getDatetimeConstructor(writer, value); } else if (target.isBlobShape()) { return String.format("b'%s'", defaultNode.expectStringNode().getValue()); } @@ -318,46 +305,6 @@ private String getDefaultValue(PythonWriter writer, MemberShape member) { }; } - private ZonedDateTime parseDefaultTimestamp(MemberShape member, Node value) { - Optional trait = member.getMemberTrait(model, TimestampFormatTrait.class); - if (trait.isPresent()) { - switch (trait.get().getFormat()) { - case EPOCH_SECONDS: - return parseEpochTime(value); - case DATE_TIME: - return parseDateTime(value); - case HTTP_DATE: - return parseHttpDate(value); - default: - break; - } - } - - if (value.isNumberNode()) { - return parseEpochTime(value); - } else { - // Smithy's node validator asserts that string nodes are in the http date format if there - // is no format explicitly given. - return parseDateTime(value); - } - } - - private ZonedDateTime parseEpochTime(Node value) { - Number number = value.expectNumberNode().getValue(); - Instant instant = Instant.ofEpochMilli(Double.valueOf(number.doubleValue() * 1000).longValue()); - return instant.atZone(ZoneId.of("UTC")); - } - - private ZonedDateTime parseDateTime(Node value) { - Instant instant = Instant.from(DateTimeFormatter.ISO_INSTANT.parse(value.expectStringNode().getValue())); - return instant.atZone(ZoneId.of("UTC")); - } - - private ZonedDateTime parseHttpDate(Node value) { - Instant instant = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(value.expectStringNode().getValue())); - return instant.atZone(ZoneId.of("UTC")); - } - private boolean hasDocs() { if (shape.hasTrait(DocumentationTrait.class)) { return true; From 66a4759fac8f8528e57b78c8bb4e7c16957941d0 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Fri, 13 Jan 2023 14:44:14 +0100 Subject: [PATCH 07/11] Omit 0 as fractional epoch seconds This updates unix timestamp serialization to omit fractional seconds when that fraction is 0. --- .../integration/HttpProtocolGeneratorUtils.java | 4 +++- .../smithy-python/smithy_python/utils.py | 14 ++++++++++++++ .../smithy-python/tests/unit/test_utils.py | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java index 7925c9fb8..0070a69b0 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/integration/HttpProtocolGeneratorUtils.java @@ -60,7 +60,9 @@ public static String getTimestampInputParam( writer.addImport("smithy_python.utils", "serialize_rfc3339"); return String.format("serialize_rfc3339(%s)", result); case EPOCH_SECONDS: - return "str(" + result + ".timestamp())"; + writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON); + writer.addImport("smithy_python.utils", "serialize_epoch_seconds"); + return String.format("serialize_epoch_seconds(%s)", result); case HTTP_DATE: writer.addStdlibImport("email.utils", "format_datetime"); return "format_datetime(" + result + ", usegmt=True)"; diff --git a/python-packages/smithy-python/smithy_python/utils.py b/python-packages/smithy-python/smithy_python/utils.py index fb1fc10f4..3c3720933 100644 --- a/python-packages/smithy-python/smithy_python/utils.py +++ b/python-packages/smithy-python/smithy_python/utils.py @@ -182,3 +182,17 @@ def serialize_rfc3339(given: datetime) -> str: return given.strftime(RFC3339_MICRO) else: return given.strftime(RFC3339) + + +def serialize_epoch_seconds(given: datetime) -> str: + """Serializes a datetime into a string containing the epoch seconds. + + If ``microseconds`` is 0, no fractional part is serialized. + + :param given: The datetime to serialize. + :retursn: A string containing the seconds since the UNIX epoch. + """ + result = given.timestamp() + if given.microsecond == 0: + result = int(result) + return str(result) diff --git a/python-packages/smithy-python/tests/unit/test_utils.py b/python-packages/smithy-python/tests/unit/test_utils.py index 5c294405f..cf50f2806 100644 --- a/python-packages/smithy-python/tests/unit/test_utils.py +++ b/python-packages/smithy-python/tests/unit/test_utils.py @@ -10,6 +10,7 @@ ensure_utc, expect_type, limited_parse_float, + serialize_epoch_seconds, serialize_float, serialize_rfc3339, strict_parse_bool, @@ -177,3 +178,17 @@ def test_serialize_float(given: float | Decimal, expected: str) -> None: ) def test_serialize_rfc3339(given: datetime, expected: str) -> None: assert serialize_rfc3339(given) == expected + + +@pytest.mark.parametrize( + "given, expected", + [ + (datetime(2017, 1, 1, tzinfo=timezone.utc), "1483228800"), + ( + datetime(2017, 1, 1, microsecond=1, tzinfo=timezone.utc), + "1483228800.000001", + ), + ], +) +def test_serialize_epoch_seconds(given: datetime, expected: str) -> None: + assert serialize_epoch_seconds(given) == expected From f5f9d2da799fa53958101c54da687c8922746d93 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Thu, 19 Jan 2023 14:59:40 +0100 Subject: [PATCH 08/11] Add tests to cover serializing scientific notation --- .../smithy-python/smithy_python/utils.py | 5 ++++- .../smithy-python/tests/unit/test_utils.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/python-packages/smithy-python/smithy_python/utils.py b/python-packages/smithy-python/smithy_python/utils.py index 3c3720933..d47b25bac 100644 --- a/python-packages/smithy-python/smithy_python/utils.py +++ b/python-packages/smithy-python/smithy_python/utils.py @@ -166,8 +166,11 @@ def serialize_float(given: float | Decimal) -> str: if isinf(given): return "-Infinity" if given < 0 else "Infinity" + if isinstance(given, Decimal): + given = given.normalize() + result = str(given) - if "." not in result: + if result.isnumeric(): result += ".0" return result diff --git a/python-packages/smithy-python/tests/unit/test_utils.py b/python-packages/smithy-python/tests/unit/test_utils.py index cf50f2806..770fd65be 100644 --- a/python-packages/smithy-python/tests/unit/test_utils.py +++ b/python-packages/smithy-python/tests/unit/test_utils.py @@ -151,12 +151,27 @@ def test_strict_parse_float_raises(given: str) -> None: (1, "1.0"), (1.0, "1.0"), (1.1, "1.1"), + # It's not particularly important whether the result of this is "1.1e3" or + # "1100.0" since both are valid representations. This is how float behaves + # by default in python though, and there's no reason to do extra work to + # change it. + (1.1e3, "1100.0"), + (1e1, "10.0"), + (32.100, "32.1"), + (0.321000e2, "32.1"), + # It's at about this point that floats start using scientific notation. + (1e16, "1e+16"), (float("NaN"), "NaN"), (float("Infinity"), "Infinity"), (float("-Infinity"), "-Infinity"), (Decimal("1"), "1.0"), (Decimal("1.0"), "1.0"), (Decimal("1.1"), "1.1"), + (Decimal("1.1e3"), "1.1E+3"), + (Decimal("1e1"), "1E+1"), + (Decimal("32.100"), "32.1"), + (Decimal("0.321000e+2"), "32.1"), + (Decimal("1e16"), "1E+16"), (Decimal("NaN"), "NaN"), (Decimal("Infinity"), "Infinity"), (Decimal("-Infinity"), "-Infinity"), From b62a878d2f4cfe767b59a9dce5a0b219ac7d3b19 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Tue, 24 Jan 2023 15:20:24 +0100 Subject: [PATCH 09/11] Update serialize_rfc3339 docs --- .../amazon/smithy/python/codegen/SmithyPythonDependency.java | 3 ++- python-packages/smithy-python/smithy_python/utils.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index 84a2aeab3..c33410a8f 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -52,7 +52,8 @@ public final class SmithyPythonDependency { */ public static final PythonDependency PYTEST_ASYNCIO = new PythonDependency( "pytest-asyncio", - ">=0.20.3,<0.30.0", + ">=0.20.3,<0.30" + + ".0", Type.TEST_DEPENDENCY, false ); diff --git a/python-packages/smithy-python/smithy_python/utils.py b/python-packages/smithy-python/smithy_python/utils.py index d47b25bac..9faa7e988 100644 --- a/python-packages/smithy-python/smithy_python/utils.py +++ b/python-packages/smithy-python/smithy_python/utils.py @@ -178,6 +178,8 @@ def serialize_float(given: float | Decimal) -> str: def serialize_rfc3339(given: datetime) -> str: """Serializes a datetime into an RFC3339 string respresentation. + If ``microseconds`` is 0, no fractional part is serialized. + :param given: The datetime to serialize. :returns: An RFC3339 formatted timestamp. """ From 6d7635b983febc7f11c514d9cb40ca48e8a4d572 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Tue, 24 Jan 2023 15:21:19 +0100 Subject: [PATCH 10/11] Fix pytest-asyncio version range --- .../amazon/smithy/python/codegen/SmithyPythonDependency.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index c33410a8f..f026c9a85 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -52,8 +52,7 @@ public final class SmithyPythonDependency { */ public static final PythonDependency PYTEST_ASYNCIO = new PythonDependency( "pytest-asyncio", - ">=0.20.3,<0.30" + - ".0", + ">=0.20.3,<0.21.0", Type.TEST_DEPENDENCY, false ); From 377d259c01cb3c6b9e4256ba5439671964be8671 Mon Sep 17 00:00:00 2001 From: Jordon Phillips Date: Tue, 24 Jan 2023 15:22:59 +0100 Subject: [PATCH 11/11] Use a slice to strip trailing endpoint slash --- .../software/amazon/smithy/python/codegen/ClientGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java index 45f390998..ed66ed84d 100644 --- a/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java +++ b/codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java @@ -316,7 +316,7 @@ async def _handle_attempt( if not endpoint.url.path: endpoint.url.path = "" elif endpoint.url.path.endswith("/"): - endpoint.url.path = endpoint.url.path.rstrip("/") + endpoint.url.path = endpoint.url.path[:-1] if context.transport_request.url.path: endpoint.url.path += context.transport_request.url.path endpoint.url.query = context.transport_request.url.query