kn: checksum bindings

.github/actions/setup-build/action.yml

@@ -5,11 +5,20 @@ description: >
 runs:
   using: composite
   steps:
-    - name: Checkout tools
+    - name: Extract aws-kotlin-repo-tools version
+      working-directory: ./smithy-kotlin
+      shell: bash
+      run: |
+        export AWS_KOTLIN_REPO_TOOLS_VERSION=$(grep '^aws-kotlin-repo-tools-version' ./gradle/libs.versions.toml | sed -E 's/.*= "(.*)"/\1/')
+        echo "Using aws-kotlin-repo-tools version $AWS_KOTLIN_REPO_TOOLS_VERSION"
+        echo "aws_kotlin_repo_tools_version=$AWS_KOTLIN_REPO_TOOLS_VERSION" >> $GITHUB_ENV
+
+    - name: Checkout aws-kotlin-repo-tools
       uses: actions/checkout@v4
       with:
         path: 'aws-kotlin-repo-tools'
         repository: 'awslabs/aws-kotlin-repo-tools'
+        ref: ${{ env.aws_kotlin_repo_tools_version }}
         sparse-checkout: |
           .github
 
@@ -19,6 +28,7 @@ runs:
         # checkout aws-crt-kotlin as a sibling which will automatically make it an included build
         path: 'aws-crt-kotlin'
         repository: 'awslabs/aws-crt-kotlin'
+        submodules: 'true'
 
     # Cache the Kotlin/Native toolchain based on the input Kotlin version from version catalog
     # see https://kotlinlang.org/docs/native-improving-compilation-time.html

.github/scripts/run-container-test.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Run precompiled Kotlin/Native test binaries in a Docker container for a specific Linux distribution and architecture.
+
+This requires Docker multiarch support, see https://docs.docker.com/build/building/multi-platform/ and https://github.com/multiarch/qemu-user-static
+In GitHub we use a provided action for this: https://github.com/docker/setup-qemu-action
+
+Locally you would need to run one of:
+
+`docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes`
+
+OR
+
+`docker run --privileged --rm tonistiigi/binfmt --install all`
+"""
+
+import argparse
+import os
+import subprocess
+import shlex
+import shutil
+
+VERBOSE = False
+
+DISTRO_TO_IMAGE_NAME = {
+    "ubuntu-22.04": "public.ecr.aws/lts/ubuntu:22.04_stable",
+    "al2023": "public.ecr.aws/amazonlinux/amazonlinux:2023",
+    "al2": "public.ecr.aws/amazonlinux/amazonlinux:2"
+}
+
+DOCKER_PLATFORM_BY_ARCH = {
+    "x64": "linux/amd64",
+    "arm64": "linux/arm64"
+}
+
+
+def vprint(message):
+    global VERBOSE
+    if VERBOSE:
+        print(message)
+
+
+def running_in_github_action():
+    """
+    Test if currently running in a GitHub action or running locally
+    :return: True if running in GH, False otherwise
+    """
+    return "GITHUB_WORKFLOW" in os.environ
+
+
+def shell(command, cwd=None, check=True, capture_output=False):
+    """
+    Run a command
+    :param command: command to run
+    :param cwd: the current working directory to change to before executing the command
+    :param check: flag indicating if the status code should be checked. When true an exception will be
+    thrown if the command exits with a non-zero exit status.
+    :returns: the subprocess CompletedProcess output
+    """
+    vprint(f"running `{command}`")
+    return subprocess.run(command, shell=True, check=check, cwd=cwd, capture_output=capture_output)
+
+
+def oci_executable():
+    """
+    Attempt to find the OCI container executor used to build and run docker containers
+    """
+    oci_exe = os.environ.get('OCI_EXE')
+    if oci_exe is not None:
+        return oci_exe
+
+    executors = ['finch', 'podman', 'docker']
+
+    for exe in executors:
+        if shutil.which(exe) is not None:
+            return exe
+
+    print("cannot find container executor")
+    exit(1)
+
+
+def run_docker_test(opts): 
+    """
+    Run a docker test for a precompiled Kotlin/Native binary
+
+    :param opts: the parsed command line options
+    """
+    platform = DOCKER_PLATFORM_BY_ARCH[opts.arch]
+    oci_exe = oci_executable()
+
+    test_bin_dir = os.path.abspath(opts.test_bin_dir)
+    image_name = DISTRO_TO_IMAGE_NAME[opts.distro]
+    path_to_exe = f'./linux{opts.arch.capitalize()}/debugTest/test.kexe'
+
+    cmd = [
+        oci_exe,
+        'run',
+        '--rm',
+        f'-v{test_bin_dir}:/test',
+    ]
+    if not opts.no_system_certs:
+        cmd.append(f'-v/etc/ssl:/etc/ssl')
+
+    cmd.extend(
+        [
+            '-w/test',
+            '-e DEBIAN_FRONTEND=noninteractive',
+            '--platform',
+            platform,
+            image_name,
+            path_to_exe,
+        ]
+    )
+
+    cmd = shlex.join(cmd)
+    print(cmd)
+    shell(cmd)
+
+
+def create_cli():
+    parser = argparse.ArgumentParser(
+        prog="run-container-test",
+        description="Run cross platform test binaries in a container",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true")
+
+    parser.add_argument("--distro", required=True, choices=DISTRO_TO_IMAGE_NAME.keys(), help="the distribution name to run the task on")
+    parser.add_argument("--arch", required=True, choices=DOCKER_PLATFORM_BY_ARCH.keys(), help="the architecture to use")
+    parser.add_argument("--test-bin-dir", required=True, help="the path to the test binary directory root")
+    parser.add_argument("--no-system-certs", action='store_true', help="disable mounting system certificates into the container")
+
+    return parser
+
+
+def main():
+    cli = create_cli()
+    opts = cli.parse_args()
+    if opts.verbose:
+        global VERBOSE
+        VERBOSE = True
+
+    run_docker_test(opts)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/ci.yml

@@ -67,7 +67,7 @@ jobs:
           # FIXME K2. Re-enable warnings as errors after this warning is removed: https://youtrack.jetbrains.com/issue/KT-68532
           # echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties              
           ./gradlew apiCheck
-          ./gradlew -Paws.sdk.kotlin.crt.disableCrossCompile=true build
+          ./gradlew -Paws.sdk.kotlin.crt.disableCrossCompile=true -Paws.kotlin.native.disableCrossCompile=true build
 
       - name: Save Test Reports
         if: failure()
@@ -126,7 +126,7 @@ jobs:
         working-directory: ./smithy-kotlin
         run: |
           ./gradlew apiCheck
-          ./gradlew -P"aws.sdk.kotlin.crt.disableCrossCompile"=true build
+          ./gradlew -P"aws.kotlin.native=false" build
 
       - name: Save Test Reports
         if: failure()

CONTRIBUTING.md

@@ -223,3 +223,25 @@ If you discover a potential security issue in this project we ask that you notif
 See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
 
 We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.
+
+## Kotlin/Native
+Support for Kotlin/Native is in active development. The following sections cover how to effectively test your work.
+
+### Linux/Unix
+
+#### Testing Different Linux Distros and Architectures
+
+1. Build the test executable(s) for the architecture(s) you want to test.
+
+```sh
+# build specific arch
+./gradlew linuxX64TestBinaries
+```
+
+2. Use the `run-container-test.py` helper script to execute tests locally. Change the directory as needed.
+
+```sh
+OCI_EXE=docker python3 .github/scripts/run-container-test.py --distro al2 --arch x64 --test-bin-dir ./runtime/runtime-core/build/bin
+```
+
+See the usage/help for different distributions provided: `python3 .github/scripts/run-container.py -h`

bom/build.gradle.kts

@@ -7,8 +7,6 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
 import org.jetbrains.kotlin.gradle.plugin.KotlinMultiplatformPluginWrapper
 import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMetadataTarget
-import org.jetbrains.kotlin.gradle.targets.js.KotlinJsTarget
-import java.util.*
 
 plugins {
     `maven-publish`
@@ -52,7 +50,6 @@ fun createBomConstraintsAndVersionCatalog() {
 
 fun Project.artifactId(target: KotlinTarget): String = when (target) {
     is KotlinMetadataTarget -> name
-    is KotlinJsTarget -> "$name-js"
     else -> "$name-${target.targetName.lowercase()}"
 }
 

gradle/libs.versions.toml

@@ -2,7 +2,7 @@
 kotlin-version = "2.0.21"
 dokka-version = "1.9.10"
 
-aws-kotlin-repo-tools-version = "0.4.15-kn"
+aws-kotlin-repo-tools-version = "0.4.20-kn"
 
 # libs
 coroutines-version = "1.9.0"
@@ -96,6 +96,7 @@ ktor-server-netty = { module = "io.ktor:ktor-server-netty", version.ref = "ktor-
 ktor-server-jetty-jakarta = { module = "io.ktor:ktor-server-jetty-jakarta", version.ref = "ktor-version" }
 ktor-server-cio = { module = "io.ktor:ktor-server-cio", version.ref = "ktor-version" }
 ktor-network-tls-certificates = { module = "io.ktor:ktor-network-tls-certificates", version.ref = "ktor-version" }
+ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor-version" }
 
 kaml = { module = "com.charleskorn.kaml:kaml", version.ref = "kaml-version" }
 jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup-version" }

runtime/auth/aws-signing-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/auth/awssigning/crt/CrtSigningSuiteTest.kt

@@ -9,10 +9,4 @@ import aws.smithy.kotlin.runtime.auth.awssigning.tests.SigningSuiteTestBase
 
 class CrtSigningSuiteTest : SigningSuiteTestBase() {
     override val signer: AwsSigner = CrtAwsSigner
-
-    override val disabledTests = super.disabledTests + setOf(
-        // FIXME - Signature mismatch possibly related to https://github.com/awslabs/aws-crt-java/pull/419. Needs
-        // investigation.
-        "get-utf8",
-    )
 }

runtime/build.gradle.kts

@@ -4,6 +4,7 @@
  */
 import aws.sdk.kotlin.gradle.dsl.configurePublishing
 import aws.sdk.kotlin.gradle.kmp.*
+import aws.sdk.kotlin.gradle.util.typedProp
 import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 
 plugins {
@@ -17,6 +18,15 @@ val sdkVersion: String by project
 // Apply KMP configuration from build plugin
 configureKmpTargets()
 
+// Disable cross compilation if flag is set
+val disableCrossCompile = typedProp<Boolean>("aws.kotlin.native.disableCrossCompile") == true
+if (disableCrossCompile) {
+    logger.warn("aws.kotlin.native.disableCrossCompile=true: Cross compilation is disabled.")
+    allprojects {
+        disableCrossCompileTargets()
+    }
+}
+
 // capture locally - scope issue with custom KMP plugin
 val libraries = libs
 

runtime/protocol/http-client-engines/http-client-engine-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/http/engine/crt/AsyncStressTest.kt

@@ -13,7 +13,6 @@ import aws.smithy.kotlin.runtime.http.request.url
 import aws.smithy.kotlin.runtime.httptest.TestWithLocalServer
 import aws.smithy.kotlin.runtime.net.Host
 import aws.smithy.kotlin.runtime.net.Scheme
-import aws.smithy.kotlin.runtime.testing.IgnoreWindows
 import io.ktor.server.cio.*
 import io.ktor.server.engine.*
 import io.ktor.server.response.*
@@ -25,9 +24,7 @@ import kotlinx.coroutines.yield
 import kotlin.test.Test
 import kotlin.time.Duration.Companion.seconds
 
-// FIXME This test implements [TestWithLocalServer] which is JVM-only.
 class AsyncStressTest : TestWithLocalServer() {
-
     override val server = embeddedServer(CIO, serverPort) {
         routing {
             get("/largeResponse") {
@@ -37,9 +34,8 @@ class AsyncStressTest : TestWithLocalServer() {
                 call.respondText(text.repeat(respSize / text.length))
             }
         }
-    }
+    }.start()
 
-    @IgnoreWindows("FIXME - times out after upgrade to kotlinx.coroutines 1.6.0")
     @Test
     fun testStreamNotConsumed() = runBlocking {
         // test that filling the stream window and not consuming the body stream still cleans up resources

runtime/protocol/http-client-engines/http-client-engine-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/http/engine/crt/SendChunkedBodyTest.kt

@@ -33,7 +33,7 @@ class SendChunkedBodyTest {
 
         val chunkedBytes = """
            100;chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(256)}\r\n\r\n 
-        """.trimIndent().toByteArray()
+        """.trimIndent().encodeToByteArray()
 
         val source = chunkedBytes.source()
 
@@ -52,7 +52,7 @@ class SendChunkedBodyTest {
 
         val chunkedBytes = """
            ${chunkSize.toString(16)};chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(chunkSize)}\r\n\r\n 
-        """.trimIndent().toByteArray()
+        """.trimIndent().encodeToByteArray()
 
         val source = chunkedBytes.source()
 
@@ -71,7 +71,7 @@ class SendChunkedBodyTest {
 
         val chunkedBytes = """
            100;chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(256)}\r\n\r\n 
-        """.trimIndent().toByteArray()
+        """.trimIndent().encodeToByteArray()
 
         val channel = SdkByteReadChannel(chunkedBytes)
 
@@ -91,7 +91,7 @@ class SendChunkedBodyTest {
 
         val chunkedBytes = """
            ${chunkSize.toString(16)};chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(chunkSize)}\r\n\r\n 
-        """.trimIndent().toByteArray()
+        """.trimIndent().encodeToByteArray()
 
         val channel = SdkByteReadChannel(chunkedBytes)
 

runtime/protocol/http-test/build.gradle.kts

@@ -28,6 +28,14 @@ kotlin {
             }
         }
 
+        jvmAndNativeMain {
+            dependencies {
+                api(libs.ktor.server.cio)
+                implementation(libs.ktor.client.core)
+                implementation(libs.ktor.server.core)
+            }
+        }
+
         all {
             languageSettings.optIn("aws.smithy.kotlin.runtime.InternalApi")
         }