Skip to content

Commit bbc7cf1

Browse files
aajtoddaajtodd
authored
chore: bootstrap CRT build for native platforms (#91)
1 parent 1974bbe commit bbc7cf1

31 files changed

+1322
-47
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 63 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -18,48 +18,73 @@ jobs:
1818
runs-on: ubuntu-latest
1919
steps:
2020
- name: Checkout sources
21-
uses: actions/checkout@v2
22-
- uses: actions/cache@v2
21+
uses: actions/checkout@v4
2322
with:
24-
path: |
25-
~/.gradle/caches
26-
~/.gradle/wrapper
27-
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
28-
restore-keys: |
29-
${{ runner.os }}-gradle-
30-
- name: Build and Test ${{ env.PACKAGE_NAME }}
31-
run: |
32-
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
33-
chmod a+x builder.pyz
34-
echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties
35-
./builder.pyz build -p ${{ env.PACKAGE_NAME }}
36-
37-
macos-compat:
38-
runs-on: macos-latest
39-
steps:
40-
- name: Checkout sources
41-
uses: actions/checkout@v2
42-
- uses: actions/cache@v2
23+
submodules: true
24+
- name: Configure JDK
25+
uses: actions/setup-java@v3
26+
with:
27+
distribution: 'corretto'
28+
java-version: 17
29+
cache: 'gradle'
30+
# Cache the Kotlin/Native toolchain based on the input Kotlin version from version catalog
31+
# see https://kotlinlang.org/docs/native-improving-compilation-time.html
32+
- uses: actions/cache@v4
4333
with:
4434
path: |
45-
~/.gradle/caches
46-
~/.gradle/wrapper
47-
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
35+
~/.konan
36+
key: ${{ runner.os }}-konan-${{ hashFiles('gradle/libs.versions.toml') }}
4837
restore-keys: |
49-
${{ runner.os }}-gradle-
50-
- name: Build and Test ${{ env.PACKAGE_NAME }}
38+
${{ runner.os }}-konan-
39+
# FIXME - we will need to migrate this to ECR to avoid throttle limits from dockerhub, for now rebuild the images during CI
40+
- name: Configure Docker Images
5141
run: |
52-
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
53-
chmod a+x builder.pyz
54-
echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties
55-
./builder.pyz build -p ${{ env.PACKAGE_NAME }}
56-
57-
windows-compat:
58-
runs-on: windows-latest
59-
steps:
60-
- name: Checkout sources
61-
uses: actions/checkout@v2
42+
./docker-images/build-all.sh
43+
docker images
6244
- name: Build and Test ${{ env.PACKAGE_NAME }}
6345
run: |
64-
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
65-
python3 builder.pyz build -p ${{ env.PACKAGE_NAME }}
46+
docker images
47+
./gradlew apiCheck
48+
./gradlew allTests
49+
50+
# macos-compat:
51+
# runs-on: macos-latest
52+
# steps:
53+
# - name: Checkout sources
54+
# uses: actions/checkout@v4
55+
# with:
56+
# submodules: true
57+
# - name: Configure JDK
58+
# uses: actions/setup-java@v3
59+
# with:
60+
# distribution: 'corretto'
61+
# java-version: 17
62+
# cache: 'gradle'
63+
# - name: Configure Docker Images
64+
# run: |
65+
# ./docker-images/build-all.sh
66+
# - name: Build and Test ${{ env.PACKAGE_NAME }}
67+
# run: |
68+
# ./gradlew apiCheck
69+
# ./gradlew allTests
70+
#
71+
# windows-compat:
72+
# runs-on: windows-latest
73+
# steps:
74+
# - name: Checkout sources
75+
# uses: actions/checkout@v4
76+
# with:
77+
# submodules: true
78+
# - name: Configure JDK
79+
# uses: actions/setup-java@v3
80+
# with:
81+
# distribution: 'corretto'
82+
# java-version: 17
83+
# cache: 'gradle'
84+
# - name: Configure Docker Images
85+
# run: |
86+
# ./docker-images/build-all.sh
87+
# - name: Build and Test ${{ env.PACKAGE_NAME }}
88+
# run: |
89+
# ./gradlew apiCheck
90+
# ./gradlew allTests

‎.gitmodules‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
[submodule "crt/aws-c-common"]
2+
path = crt/aws-c-common
3+
url = https://github.com/awslabs/aws-c-common
4+
[submodule "crt/aws-c-auth"]
5+
path = crt/aws-c-auth
6+
url = https://github.com/awslabs/aws-c-auth
7+
[submodule "crt/aws-c-cal"]
8+
path = crt/aws-c-cal
9+
url = https://github.com/awslabs/aws-c-cal
10+
[submodule "crt/aws-c-compression"]
11+
path = crt/aws-c-compression
12+
url = https://github.com/awslabs/aws-c-compression
13+
[submodule "crt/aws-c-http"]
14+
path = crt/aws-c-http
15+
url = https://github.com/awslabs/aws-c-http
16+
[submodule "crt/aws-c-io"]
17+
path = crt/aws-c-io
18+
url = https://github.com/awslabs/aws-c-io
19+
[submodule "crt/aws-checksums"]
20+
path = crt/aws-checksums
21+
url = https://github.com/awslabs/aws-checksums
22+
[submodule "crt/aws-c-mqtt"]
23+
path = crt/aws-c-mqtt
24+
url = https://github.com/awslabs/aws-c-mqtt
25+
[submodule "crt/s2n"]
26+
path = crt/s2n
27+
url = https://github.com/aws/s2n-tls
28+
[submodule "crt/aws-lc"]
29+
path = crt/aws-lc
30+
url = https://github.com/aws/aws-lc
31+
[submodule "crt/aws-c-sdkutils"]
32+
path = crt/aws-c-sdkutils
33+
url = https://github.com/awslabs/aws-c-sdkutils

‎CMakeLists.txt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
cmake_minimum_required(VERSION 3.1)
2+
project(aws-crt-kotlin C)
3+
message(STATUS "CMake ${CMAKE_VERSION}")
4+
5+
option(BUILD_DEPS "Builds aws common runtime dependencies as part of build. Turn off if you want to control your dependency chain." ON)
6+
option(BUILD_SHARED_LIBS "Build shared library for FFI: default: ON" ON)
7+
8+
if (POLICY CMP0069)
9+
cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler
10+
endif()
11+
12+
if (DEFINED CMAKE_PREFIX_PATH)
13+
file(TO_CMAKE_PATH "${CMAKE_PREFIX_PATH}" CMAKE_PREFIX_PATH)
14+
endif()
15+
16+
if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
17+
set(CMAKE_INSTALL_LIBDIR "lib")
18+
endif()
19+
20+
if (UNIX AND NOT APPLE)
21+
set(FIND_LIBRARY_USE_LIB64_PATHS true)
22+
endif()
23+
24+
# This is required in order to append /lib/cmake to each element in CMAKE_PREFIX_PATH
25+
set(AWS_MODULE_DIR "/${CMAKE_INSTALL_LIBDIR}/cmake")
26+
string(REPLACE ";" "${AWS_MODULE_DIR};" AWS_MODULE_PATH "${CMAKE_PREFIX_PATH}${AWS_MODULE_DIR}")
27+
# Append that generated list to the module search path
28+
list(APPEND CMAKE_MODULE_PATH ${AWS_MODULE_PATH})
29+
30+
31+
if (BUILD_DEPS)
32+
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/crt/aws-c-common/cmake")
33+
34+
include(AwsFindPackage)
35+
36+
set(IN_SOURCE_BUILD ON)
37+
set(SEARCH_LIBCRYPTO OFF CACHE BOOL "Let S2N use libcrypto from AWS-LC.")
38+
39+
# Don't compile tests in subdirectories.
40+
# Turn off using `option` instead of `set`, or CTest will declare
41+
# it as an option later and override the existing variable.
42+
set(BUILD_TESTING OFF)
43+
44+
# Disable BUILD_SHARED_LIBS for all CRT libs
45+
set(SHARED_FFI_LIB ${BUILD_SHARED_LIBS})
46+
set(BUILD_SHARED_LIBS OFF)
47+
48+
# CRT Libraries
49+
add_subdirectory(crt/aws-c-common)
50+
if (UNIX AND NOT APPLE)
51+
if (NOT USE_OPENSSL)
52+
if (NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) AND
53+
NOT (CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l))
54+
set(DISABLE_PERL ON)
55+
endif()
56+
if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
57+
set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512")
58+
set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
59+
endif()
60+
set(DISABLE_GO ON)
61+
set(SEARCH_LIBCRYPTO OFF)
62+
set(BUILD_LIBSSL OFF)
63+
add_subdirectory(crt/aws-lc)
64+
else()
65+
set(SEARCH_LIBCRYPTO ON)
66+
# Find the system libcrypto and propagate its location to s2n's find script
67+
find_package(OpenSSL REQUIRED)
68+
set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
69+
set(LibCrypto_LIBRARY ${OPENSSL_CRYPTO_LIBRARY})
70+
if (LibCrypto_LIBRARY MATCHES ".so$")
71+
set(LibCrypto_SHARED_LIBRARY ${LibCrypto_LIBRARY})
72+
else()
73+
set(LibCrypto_STATIC_LIBRARY ${LibCrypto_LIBRARY})
74+
endif()
75+
endif()
76+
add_subdirectory(crt/s2n)
77+
endif()
78+
add_subdirectory(crt/aws-c-sdkutils)
79+
add_subdirectory(crt/aws-c-io)
80+
add_subdirectory(crt/aws-c-cal)
81+
add_subdirectory(crt/aws-c-compression)
82+
add_subdirectory(crt/aws-c-http)
83+
add_subdirectory(crt/aws-c-auth)
84+
add_subdirectory(crt/aws-checksums)
85+
else()
86+
include(AwsFindPackage)
87+
set(IN_SOURCE_BUILD OFF)
88+
endif()
89+
90+
# Restore BUILD_SHARED_LIBS for this project
91+
set(BUILD_SHARED_LIBS ${SHARED_FFI_LIB})
92+
93+
include(AwsCFlags)
94+
include(AwsSharedLibSetup)
95+
include(AwsSanitizers)
96+
97+
aws_use_package(aws-c-common)
98+
aws_use_package(aws-c-sdkutils)
99+
aws_use_package(aws-c-io)
100+
aws_use_package(aws-c-cal)
101+
aws_use_package(aws-c-compression)
102+
aws_use_package(aws-c-http)
103+
aws_use_package(aws-c-auth)
104+
aws_use_package(aws-checksums)
105+

‎README.md‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ This project is licensed under the Apache-2.0 License.
1313

1414
## Building
1515

16-
Kotlin Multiplatform projects are in [Alpha](https://kotlinlang.org/docs/reference/evolution/components-stability.html). The CRT interfaces are subject to change.
16+
CRT interfaces are subject to change.
1717

1818
### Linux/Unix
1919
Install some version of libcrypto on which s2n depends. See the [s2n](https://github.com/awslabs/s2n) documentation.
@@ -23,6 +23,7 @@ apt-get install libssl-dev
2323
```
2424

2525
OR
26+
2627
```sh
2728
yum install openssl-devel
2829
```
@@ -32,6 +33,36 @@ Set the path to `libcrypto.a` either as a command line argument to gradle `-Plib
3233

3334
### OSX
3435

36+
#### Debugging simulator test issues
37+
38+
**Xcode does not support simulator tests for \<native target>**
39+
40+
```
41+
* What went wrong:
42+
Execution failed for task ':aws-crt-kotlin:iosX64Test'.
43+
> Error while evaluating property 'device' of task ':aws-crt-kotlin:iosX64Test'.
44+
> Failed to calculate the value of task ':aws-crt-kotlin:iosX64Test' property 'device'.
45+
> Xcode does not support simulator tests for ios_x64. Check that requested SDK is installed.
46+
```
47+
48+
Ensure that you have an appropriate simulator runtime installed.
49+
50+
e.g. to install `iOS` platform support including simulator runtimes:
51+
```sh
52+
xcodebuild -downloadPlatform iOS
53+
```
54+
55+
List simulator runtimes with:
56+
57+
```sh
58+
xcrun simctl list devices available
59+
```
60+
61+
62+
See also:
63+
64+
* https://developer.apple.com/documentation/xcode/installing-additional-simulator-runtimes
65+
* https://www.iosdev.recipes/simctl/
3566

3667
### Windows
3768

@@ -69,3 +100,4 @@ CRTDEBUG=trace=2 ./elasticurl/bin/macosX64/elasticurl.kexe -v trace https://aws.
69100
Run the simple elasticurl integration test script
70101

71102
`./scripts/elasticurl-test.sh`
103+

‎aws-crt-kotlin/build.gradle.kts‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,17 @@
22
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
33
* SPDX-License-Identifier: Apache-2.0
44
*/
5+
import aws.sdk.kotlin.gradle.crt.cmakeInstallDir
6+
import aws.sdk.kotlin.gradle.crt.configureCrtCMakeBuild
57
import aws.sdk.kotlin.gradle.dsl.configurePublishing
68
import aws.sdk.kotlin.gradle.kmp.IDEA_ACTIVE
79
import aws.sdk.kotlin.gradle.kmp.configureKmpTargets
10+
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
811

912
plugins {
1013
alias(libs.plugins.kotlin.multiplatform)
14+
alias(libs.plugins.aws.kotlin.repo.tools.kmp)
15+
id("crt-build-support")
1116
}
1217

1318
val sdkVersion: String by project
@@ -24,6 +29,10 @@ configureKmpTargets()
2429
kotlin {
2530
explicitApi()
2631

32+
iosArm64()
33+
iosSimulatorArm64()
34+
iosX64()
35+
2736
jvm {
2837
attributes {
2938
attribute<org.gradle.api.attributes.java.TargetJvmEnvironment>(
@@ -60,10 +69,6 @@ kotlin {
6069
}
6170
}
6271

63-
val kotlinVersion: String by project
64-
val coroutinesVersion: String by project
65-
val mockServerVersion: String by project
66-
6772
sourceSets {
6873
val commonMain by getting {
6974
dependencies {
@@ -116,6 +121,37 @@ kotlin {
116121
sourceSets.all {
117122
optinAnnotations.forEach { languageSettings.optIn(it) }
118123
}
124+
125+
// create a single "umbrella" cinterop will all the aws-c-* API's we want to consume
126+
// see: https://github.com/JetBrains/kotlin-native/issues/2423#issuecomment-466300153
127+
targets.withType<KotlinNativeTarget> {
128+
val knTarget = this
129+
logger.info("configuring $knTarget: ${knTarget.name}")
130+
val cmakeInstallTask = configureCrtCMakeBuild(knTarget)
131+
val targetInstallDir = project.cmakeInstallDir(knTarget)
132+
val headerDir = targetInstallDir.resolve("include")
133+
val libDir = targetInstallDir.resolve("lib")
134+
135+
compilations["main"].cinterops {
136+
val interopDir = "$projectDir/native/interop"
137+
println("configuring crt cinterop for: ${knTarget.name}")
138+
val interopSettings = create("aws-crt") {
139+
defFile("$interopDir/crt.def")
140+
includeDirs(headerDir)
141+
compilerOpts("-L${libDir.absolutePath}")
142+
}
143+
144+
// cinterop tasks processes header files which requires the corresponding CMake build/install to run
145+
val cinteropTask = tasks.named(interopSettings.interopProcessingTaskName)
146+
cinteropTask.configure {
147+
dependsOn(cmakeInstallTask)
148+
}
149+
}
150+
151+
compilations["test"].compilerOptions.configure {
152+
freeCompilerArgs.addAll(listOf("-linker-options", "-L${libDir.absolutePath}"))
153+
}
154+
}
119155
}
120156

121157
// Publishing

0 commit comments

Comments
 (0)