feat: Support multiple models/dependencies in Rust, within a single c…

…rate (#583)

Modifies the polymorph code generation process for Rust to apply to all services in scope at once, to get around the fact that Dafny's Rust code generation (and hence smithy-dafny's as well) only supports a single crate at a time. The polymorph CLI now supports multiple `--namespace` options, and the makefile now requires a `MAIN_SERVICE_FOR_RUST` service name in order to still select a single primary service to generate at the top level of the crate. All other services are qualified under `deps::<service_name>`. This layout may be changed before GA but I assert it is better to merge this first in order to lock down progress in the CI.

To implement this change, introduced a `MergedServicesGenerator` which instantiates a separate generator for each service. The individual generators were changed throughout to use the root path/module name for the given service.

Also made several other improvements to support building the complete DB ESDK dependency graph:

* Fixed several bugs in the handling of `@positional` (testing followup tracked in #599)
* Handled the Smithy `Unit` type properly (testing followup tracked in #602)
* Properly handled introducing a single top-level error type for SDK wrappers, and wrapping errors from dependent services.
* Miscellaneous bug fixes exposed by the DB ESDK source.
* Deleted generated source for several test models to avoid review noise (and CI sensitivity to non-deterministic code generation)

.github/actions/build_dafny_from_source/action.yml

@@ -56,8 +56,12 @@ runs:
 
  # update the copy of the Rust runtime we have inside TestModels
  # (since it's not published yet)
- - name: Update dafny_runtime_rust
- shell: bash
- run: |
- rm -rf TestModels/dafny-dependencies/dafny_runtime_rust
- cp -r dafny/Source/DafnyRuntime/DafnyRuntimeRust TestModels/dafny-dependencies/dafny_runtime_rust
+ # TODO: Temporarily not doing this because we need a targeted fix to the runtime,
+ # but can't pick it up by moving to a newer Dafny commit because that would pull in
+ # other breaking changes.
+
+# - name: Update dafny_runtime_rust
+# shell: bash
+# run: |
+# rm -rf TestModels/dafny-dependencies/dafny_runtime_rust
+# cp -r dafny/Source/DafnyRuntime/DafnyRuntimeRust TestModels/dafny-dependencies/dafny_runtime_rust

SmithyDafnyMakefile.mk

@@ -288,7 +288,7 @@ _polymorph:
  --dependent-model $(PROJECT_ROOT)/$(SMITHY_DEPS) \
  $(patsubst %, --dependent-model $(PROJECT_ROOT)/%/Model, $($(service_deps_var))) \
  $(DEPENDENCY_MODULE_NAMES) \
- --namespace $($(namespace_var)) \
+ $(patsubst %, --namespace %, $($(namespace_var))) \
  $(OUTPUT_LOCAL_SERVICE_$(SERVICE)) \
  $(AWS_SDK_CMD) \
  $(POLYMORPH_OPTIONS) \
@@ -428,20 +428,15 @@ setup_prettier:
  npm i --no-save prettier@3 prettier-plugin-java@2.5
 
 # Generates rust code for all namespaces in this project
-# Note that we rely on the patching feature of polymorph
-# to also patch the results of transpile_rust,
-# so we assume that is run first!
 .PHONY: polymorph_rust
-polymorph_rust: POLYMORPH_LANGUAGE_TARGET=rust
-polymorph_rust: _polymorph_dependencies
+
 polymorph_rust:
- set -e; for service in $(PROJECT_SERVICES) ; do \
- export service_deps_var=SERVICE_DEPS_$${service} ; \
- export namespace_var=SERVICE_NAMESPACE_$${service} ; \
- export SERVICE=$${service} ; \
- $(MAKE) _polymorph_rust ; \
- done
+ $(MAKE) _polymorph_rust
 
+_polymorph_rust: POLYMORPH_LANGUAGE_TARGET=rust
+_polymorph_rust: service_deps_var=SERVICE_DEPS_$(MAIN_SERVICE_FOR_RUST)
+_polymorph_rust: namespace_var=SERVICE_NAMESPACE_$(MAIN_SERVICE_FOR_RUST)
+_polymorph_rust: SERVICE=$(MAIN_SERVICE_FOR_RUST)
 _polymorph_rust: OUTPUT_RUST=--output-rust $(LIBRARY_ROOT)/runtimes/rust
 # For several TestModels we've just manually written the code generation target,
 # So we just want to ensure we can transpile and pass the tests in CI.
@@ -562,7 +557,7 @@ rust: polymorph_dafny transpile_rust polymorph_rust test_rust
 
 # The Dafny Rust code generator only supports a single crate for everything,
 # so (among other consequences) we compile src and test code together.
-transpile_rust: | transpile_implementation_rust transpile_dependencies_rust
+transpile_rust: | transpile_implementation_rust
 
 transpile_implementation_rust: TARGET=rs
 transpile_implementation_rust: OUT=implementation_from_dafny
@@ -589,17 +584,15 @@ _mv_implementation_rust:
 # rustfmt has a recurring bug where it leaves behind trailing spaces and then complains about it.
 # Pre-process the Dafny-generated Rust code to remove them.
  sed -i -e 's/[[:space:]]*$$//' runtimes/rust/src/implementation_from_dafny.rs 
-
+ rm -f runtimes/rust/src/implementation_from_dafny.rs-e
  rustfmt --edition 2021 runtimes/rust/src/implementation_from_dafny.rs
  rm -rf implementation_from_dafny-rust
 
 patch_after_transpile_rust:
- set -e; for service in $(PROJECT_SERVICES) ; do \
- export service_deps_var=SERVICE_DEPS_$${service} ; \
- export namespace_var=SERVICE_NAMESPACE_$${service} ; \
- export SERVICE=$${service} ; \
- $(MAKE) _patch_after_transpile_rust ; \
- done
+ export service_deps_var=SERVICE_DEPS_$(MAIN_SERVICE_FOR_RUST) ; \
+ export namespace_var=SERVICE_NAMESPACE_$(MAIN_SERVICE_FOR_RUST) ; \
+ export SERVICE=$(MAIN_SERVICE_FOR_RUST) ; \
+ $(MAKE) _patch_after_transpile_rust ; \
 
 _patch_after_transpile_rust: OUTPUT_RUST=--output-rust $(LIBRARY_ROOT)/runtimes/rust
 _patch_after_transpile_rust:
@@ -615,6 +608,7 @@ _patch_after_transpile_rust:
  --namespace $($(namespace_var)) \
  $(AWS_SDK_CMD) \
  $(POLYMORPH_OPTIONS) \
+ $(if $(TRANSPILE_TESTS_IN_RUST), --local-service-test, ) \
  ";
 
 build_rust:
@@ -726,12 +720,28 @@ local_transpile_impl_net_single: TARGET=cs
 local_transpile_impl_net_single: OUT=runtimes/net/ImplementationFromDafny
 local_transpile_impl_net_single: local_transpile_impl_single
 
+local_transpile_impl_rust_single: TARGET=rs
+local_transpile_impl_rust_single: OUT=implementation_from_dafny
+local_transpile_impl_rust_single: SRC_INDEX=$(RUST_SRC_INDEX)
+local_transpile_impl_rust_single: TEST_INDEX=$(RUST_TEST_INDEX)
+local_transpile_impl_rust_single: DAFNY_OPTIONS=--emit-uncompilable-code --allow-warnings --compile-suffix
+local_transpile_impl_rust_single: TRANSPILE_DEPENDENCIES=
+local_transpile_impl_rust_single: STD_LIBRARY=
+local_transpile_impl_rust_single: SRC_INDEX_TRANSPILE=$(if $(SRC_INDEX),$(SRC_INDEX),src)
+local_transpile_impl_rust_single: TEST_INDEX_TRANSPILE=$(if $(TEST_INDEX),$(TEST_INDEX),test)
+local_transpile_impl_rust_single: DAFNY_OTHER_FILES=$(RUST_OTHER_FILES)
+local_transpile_impl_rust_single: deps_var=SERVICE_DEPS_$(SERVICE)
+local_transpile_impl_rust_single: service_deps_var=SERVICE_DEPS_$(SERVICE)
+local_transpile_impl_rust_single: namespace_var=SERVICE_NAMESPACE_$(SERVICE)
+local_transpile_impl_rust_single: $(if $(TRANSPILE_TESTS_IN_RUST), transpile_test, transpile_implementation) _mv_implementation_rust _patch_after_transpile_rust
+
+
 local_transpile_impl_single: deps_var=SERVICE_DEPS_$(SERVICE)
 local_transpile_impl_single: TRANSPILE_TARGETS=./dafny/$(SERVICE)/src/$(FILE)
 local_transpile_impl_single: TRANSPILE_DEPENDENCIES= \
- $(patsubst %, -library:$(PROJECT_ROOT)/%/src/Index.dfy, $($(deps_var))) \
- $(patsubst %, -library:$(PROJECT_ROOT)/%, $(PROJECT_INDEX)) \
- -library:$(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy
+ $(patsubst %, --library:$(PROJECT_ROOT)/%/src/Index.dfy, $($(deps_var))) \
+ $(patsubst %, --library:$(PROJECT_ROOT)/%, $(PROJECT_INDEX)) \
+ --library:$(PROJECT_ROOT)/$(STD_LIBRARY)/src/Index.dfy
 local_transpile_impl_single: transpile_implementation
 
 # Targets to transpile single local service for convenience.
@@ -766,4 +776,4 @@ local_polymorph_rust_single: local_polymorph_single
 local_polymorph_single: service_deps_var=SERVICE_DEPS_$(SERVICE)
 local_polymorph_single: namespace_var=SERVICE_NAMESPACE_$(SERVICE)
 local_polymorph_single: PROJECT_DEPENDENCIES=
-local_polymorph_single: _polymorph
+local_polymorph_single: _polymorph

TestModels/Aggregate/Makefile

@@ -12,6 +12,8 @@ include ../SharedMakefile.mk
 PROJECT_SERVICES := \
  SimpleAggregate
 
+MAIN_SERVICE_FOR_RUST := SimpleAggregate
+
 TRANSLATION_RECORD_PYTHON := \
  --translation-record ../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr
 

TestModels/Constructor/Makefile

@@ -14,6 +14,8 @@ NAMESPACE=simple.constructor
 PROJECT_SERVICES := \
  SimpleConstructor
 
+MAIN_SERVICE_FOR_RUST := SimpleConstructor
+
 SERVICE_NAMESPACE_SimpleConstructor=simple.constructor
 
 SERVICE_DEPS_SimpleConstructor :=

TestModels/Dependencies/Makefile

@@ -5,11 +5,15 @@ CORES=2
 
 ENABLE_EXTERN_PROCESSING=1
 
+TRANSPILE_TESTS_IN_RUST=1
+
 include ../SharedMakefile.mk
 
 PROJECT_SERVICES := \
  SimpleDependencies
 
+MAIN_SERVICE_FOR_RUST := SimpleDependencies
+
 # Dependencies external to this project
 # Order is important
 # In java they MUST be built

TestModels/Dependencies/runtimes/rust/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "dependencies"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+wrapped-client = []
+
+[dependencies]
+aws-smithy-runtime-api = {version = "1.7.0", features = ["client"] }
+aws-smithy-types = "1.2.0"
+dafny_runtime = { path = "../../../dafny-dependencies/dafny_runtime_rust"}
+
+[dev-dependencies]
+dependencies = { path = ".", features = ["wrapped-client"] }
+
+[dependencies.tokio]
+version = "1.26.0"
+features = ["full"]
+
+[lib]
+path = "src/implementation_from_dafny.rs"

TestModels/Errors/Makefile

@@ -12,6 +12,8 @@ include ../SharedMakefile.mk
 PROJECT_SERVICES := \
  SimpleErrors
 
+MAIN_SERVICE_FOR_RUST := SimpleErrors
+
 SERVICE_NAMESPACE_SimpleErrors=simple.errors
 
 SERVICE_DEPS_SimpleErrors :=

TestModels/LanguageSpecificLogic/Makefile

@@ -10,6 +10,8 @@ include ../SharedMakefile.mk
 PROJECT_SERVICES := \
  LanguageSpecificLogic
 
+MAIN_SERVICE_FOR_RUST := LanguageSpecificLogic
+
 SERVICE_NAMESPACE_LanguageSpecificLogic=language.specific.logic
 
 SERVICE_DEPS_LanguageSpecificLogic :=

TestModels/aws-sdks/ddb-lite/runtimes/rust/src/conversions/client.rs → TestModels/LanguageSpecificLogic/runtimes/rust/src/deps.rs

@@ -1,3 +1,3 @@
 // Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
-// Do not modify this file. This file is machine generated, and any changes to it will be overwritten.
+// Do not modify this file. This file is machine generated, and any changes to it will be overwritten.

TestModels/LocalService/Makefile

@@ -13,6 +13,8 @@ NAMESPACE=simple.localService
 PROJECT_SERVICES := \
  SimpleLocalService
 
+MAIN_SERVICE_FOR_RUST := SimpleLocalService
+
 SERVICE_NAMESPACE_SimpleLocalService=simple.localService
 
 SERVICE_DEPS_SimpleLocalService :=

TestModels/MultipleModels/Makefile

@@ -4,6 +4,7 @@
 CORES=2
 
 ENABLE_EXTERN_PROCESSING=1
+TRANSPILE_TESTS_IN_RUST=1
 
 include ../SharedMakefile.mk
 
@@ -14,6 +15,8 @@ PROJECT_SERVICES := \
  DependencyProject \
  PrimaryProject \
 
+MAIN_SERVICE_FOR_RUST := PrimaryProject
+
 SERVICE_NAMESPACE_PrimaryProject=simple.multiplemodels.primaryproject
 SERVICE_NAMESPACE_DependencyProject=simple.multiplemodels.dependencyproject
 

TestModels/MultipleModels/dafny/DependencyProject/Model/dependency-project.smithy

@@ -5,7 +5,8 @@ namespace simple.multiplemodels.dependencyproject
 @aws.polymorph#localService(
  sdkId: "DependencyProject",
  config: DependencyProjectConfig,
-) service DependencyProject {
+)
+service DependencyProject {
  version: "2021-11-01",
  resources: [],
  operations: [ SomeDependencyOperation ],

TestModels/MultipleModels/dafny/DependencyProject/src/Index.dfy

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 include "SimpleMultiplemodelsDependencyprojectImpl.dfy"
 
-module DependencyProject refines AbstractSimpleMultiplemodelsDependencyprojectService {
+module {:extern "simple.multiplemodels.dependencyproject.internaldafny" } DependencyProject refines AbstractSimpleMultiplemodelsDependencyprojectService {
  import Operations = SimpleMultiplemodelsDependencyprojectImpl
 
  function method DefaultDependencyProjectConfig(): DependencyProjectConfig {

TestModels/MultipleModels/dafny/DependencyProject/src/WrappedSimpleMultiplemodelsDependencyprojectImpl.dfy

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 include "../Model/SimpleMultiplemodelsDependencyprojectTypesWrapped.dfy"
 
-module WrappedSimpleMultiplemodelsDependencyprojectService refines WrappedAbstractSimpleMultiplemodelsDependencyprojectService {
+module {:extern "simple.multiplemodels.dependencyproject.internaldafny.wrapped"} WrappedSimpleMultiplemodelsDependencyprojectService refines WrappedAbstractSimpleMultiplemodelsDependencyprojectService {
  import WrappedService = DependencyProject
  function method WrappedDefaultDependencyProjectConfig(): DependencyProjectConfig {
  DependencyProjectConfig

TestModels/MultipleModels/dafny/PrimaryProject/Model/primary-project.smithy

@@ -7,6 +7,7 @@ use simple.multiplemodels.dependencyproject#SomeDependencyOperation
 @aws.polymorph#localService(
  sdkId: "PrimaryProject",
  config: PrimaryProjectConfig,
+ dependencies: [ simple.multiplemodels.dependencyproject#DependencyProject ]
 ) service PrimaryProject {
  version: "2021-11-01",
  resources: [],

TestModels/MultipleModels/dafny/PrimaryProject/src/Index.dfy

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 include "SimpleMultiplemodelsPrimaryprojectImpl.dfy"
 
-module PrimaryProject refines AbstractSimpleMultiplemodelsPrimaryprojectService {
+module {:extern "simple.multiplemodels.primaryproject.internaldafny" } PrimaryProject refines AbstractSimpleMultiplemodelsPrimaryprojectService {
  import Operations = SimpleMultiplemodelsPrimaryprojectImpl
 
  function method DefaultPrimaryProjectConfig(): PrimaryProjectConfig {

TestModels/MultipleModels/dafny/PrimaryProject/src/WrappedSimpleMultiplemodelsPrimaryprojectImpl.dfy

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 include "../Model/SimpleMultiplemodelsPrimaryprojectTypesWrapped.dfy"
 
-module WrappedSimpleMultiplemodelsPrimaryprojectService refines WrappedAbstractSimpleMultiplemodelsPrimaryprojectService {
+module {:extern "simple.multiplemodels.primaryproject.internaldafny.wrapped"} WrappedSimpleMultiplemodelsPrimaryprojectService refines WrappedAbstractSimpleMultiplemodelsPrimaryprojectService {
  import WrappedService = PrimaryProject
  function method WrappedDefaultPrimaryProjectConfig(): PrimaryProjectConfig {
  PrimaryProjectConfig

TestModels/MultipleModels/runtimes/rust/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "multiple_models"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[features]
+wrapped-client = []
+
+[dependencies]
+aws-smithy-runtime-api = {version = "1.7.0", features = ["client"] }
+aws-smithy-types = "1.2.0"
+dafny_runtime = { path = "../../../dafny-dependencies/dafny_runtime_rust"}
+
+[dev-dependencies]
+multiple_models = { path = ".", features = ["wrapped-client"] }
+
+[dependencies.tokio]
+version = "1.26.0"
+features = ["full"]
+
+[lib]
+path = "src/implementation_from_dafny.rs"

TestModels/Positional/Makefile

@@ -13,6 +13,8 @@ NAMESPACE=simple.positional
 PROJECT_SERVICES := \
  SimplePositional
 
+MAIN_SERVICE_FOR_RUST := SimplePositional
+
 SERVICE_NAMESPACE_SimplePositional=simple.positional
 
 SERVICE_DEPS_SimplePositional :=