Skip to content

Commit

Permalink
FIPS with classifier (#772)
Browse files Browse the repository at this point in the history
Co-authored-by: Michael Graeb <graebm@amazon.com>
  • Loading branch information
TingDaoK and graebm authored Mar 15, 2024
1 parent 834f542 commit 42b09c3
Show file tree
Hide file tree
Showing 25 changed files with 305 additions and 103 deletions.
22 changes: 22 additions & 0 deletions .builder/actions/aws_crt_java_build.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

import Builder
import os
import argparse


class AWSCrtJavaBuild(Builder.Action):

def run(self, env):
if os.getenv("CRT_FIPS") is not None:
env.shell.exec("mvn", "-P", "continuous-integration", "-B", "compile",
"-Dcmake.crt_fips=ON", check=True)
else:
env.shell.exec("mvn", "-P", "continuous-integration",
"-B", "compile", check=True)

parser = argparse.ArgumentParser()
parser.add_argument('--classifier')
args = parser.parse_known_args(env.args.args)[0]
if args.classifier:
env.shell.exec("mvn", "-B", "install", "-DskipTests", "-Dshared-lib.skip=true",
f"-Dcrt.classifier={args.classifier}", check=True)
15 changes: 0 additions & 15 deletions .builder/actions/build-classifier.py

This file was deleted.

41 changes: 38 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
- 'docs'

env:
BUILDER_VERSION: v0.9.55
BUILDER_VERSION: v0.9.58
BUILDER_SOURCE: releases
BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
PACKAGE_NAME: aws-crt-java
Expand Down Expand Up @@ -101,6 +101,42 @@ jobs:
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
linux-fips-x64:
runs-on: ubuntu-22.04 # latest
steps:
- name: Checkout Sources
uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Build ${{ env.PACKAGE_NAME }}
run: |
bash codebuild/cd/test-fips-branch.sh
export CRT_FIPS=ON
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
chmod a+x builder
./builder build -p ${{ env.PACKAGE_NAME }}
linux-fips-armv8:
runs-on: ubuntu-22.04 # latest
steps:
- name: Checkout Sources
uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Build ${{ env.PACKAGE_NAME }}
run: |
bash codebuild/cd/test-fips-branch.sh
export CRT_FIPS=ON
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
chmod a+x builder
./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-arm64 --spec=downstream --cmake-extra=-DCRT_FIPS=ON
# armv7 needs its own action due to inability to differentiate at runtime armv6 vs armv7 in the jvm:
#
# At build time we can properly figure out that we're targeting armv7.
Expand All @@ -117,9 +153,8 @@ jobs:
submodules: true
- name: Build ${{ env.PACKAGE_NAME }}
run: |
export AWS_CRT_ARCH=armv7
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }}
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }} --classifier "armv7"
raspberry:
runs-on: ubuntu-22.04 # latest
Expand Down
25 changes: 23 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.1)

project(aws-crt-jni C)
option(BUILD_DEPS "Builds aws common runtime dependencies as part of build" ON)
option(CRT_FIPS "Whether to build aws-lc with FIPS compliance" OFF)

if (POLICY CMP0069)
cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
Expand Down Expand Up @@ -60,8 +61,24 @@ if (BUILD_DEPS)
add_subdirectory(crt/aws-c-common)
if (UNIX AND NOT APPLE)
set(BUILD_LIBSSL OFF CACHE BOOL "Don't need libssl, only need libcrypto")
set(DISABLE_PERL ON CACHE BOOL "Disable codegen")
set(DISABLE_GO ON CACHE BOOL "Disable codegen")
message("Build with FIPS: " ${CRT_FIPS})
if (CRT_FIPS)
set(FIPS ON CACHE BOOL "FIPS compliance")
set(PERL_EXECUTABLE "perl")
if (DEFINED ENV{GO_PATH})
set(GO_EXECUTABLE $ENV{GO_PATH}/go)
message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}")
endif()
else()
set(DISABLE_PERL ON CACHE BOOL "Disable codegen")
set(DISABLE_GO ON CACHE BOOL "Disable codegen")
endif()

if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0")
set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512")
set(PERL_EXECUTABLE "perl")
set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it")
endif()

# temporarily disable certain warnings as errors for the aws-lc build
set(OLD_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
Expand Down Expand Up @@ -107,6 +124,10 @@ include(AwsPlatformDetect)
include(AwsSharedLibSetup)
include(AwsCRuntime)

if (CRT_FIPS AND NOT FIPS)
message(FATAL_ERROR "CRT_FIPS can only be set when build with aws-lc.")
endif()

aws_determine_local_c_runtime(AWS_C_RUNTIME)

file(GLOB AWS_CRT_JAVA_HEADERS
Expand Down
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ From maven: (https://search.maven.org/artifact/software.amazon.awssdk.crt/aws-cr

The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled C libraries for many different platforms (Windows, Linux, etc). If size is an issue, you can pick a smaller platform-specific JAR by setting the `<classifier>`.

The classifier `fips-compat` provides an "uber" jar with FIPS compliance on *some platforms*. **WARNING:** Platforms without FIPS compliance are also included in this jar, for compatibility's sake. Check `CRT.isFIPS()` at runtime to ensure you are on a FIPS compliant platform. The current breakdown is:
* **FIPS compliant**: linux-aarch_64, linux-x86_64
* **NOT compliant**: linux-armv6, linux-armv7, linux-armv7-musl, linux-aarch_64-musl, linux-x86_32, linux-x86_64-musl, osx-aarch_64, osx-x86_64, windows-x86_32, windows-x86_64

``` xml
<!-- Platform-specific Linux x86_64 JAR -->
<dependency>
Expand Down Expand Up @@ -115,6 +119,7 @@ The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled
- osx-x86_64
- windows-x86_32
- windows-x86_64
- fips-compat (no auto-detect)

### Auto-detect

Expand Down
25 changes: 7 additions & 18 deletions builder.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@
"maven"
],
"build_steps": [
"mvn -P continuous-integration -B compile"
"aws-crt-java-build"
],
"test_steps": [
"aws-crt-java-test"
],
"+imports": [
"JDK8"
"JDK8",
"golang"
],
"build_dir": "target/cmake-build",
"build_env": {
"JAVA_HOME": "{java_home}",
"OVERRIDE_JAVA_HOME": "{java_home}"
"OVERRIDE_JAVA_HOME": "{java_home}",
"GOPROXY": "https://goproxy.io,direct"
},
"test_env": {
"JAVA_HOME": "{java_home}",
Expand All @@ -35,12 +37,6 @@
},
"targets": {
"linux": {
"!build_steps": [
"mvn -P continuous-integration -B compile"
],
"!test_steps": [
"aws-crt-java-test"
],
"architectures": {
"armv6": {
"!packages": [],
Expand Down Expand Up @@ -87,18 +83,14 @@
"openjdk8"
],
"!build_steps": [
"mvn -P continuous-integration -B compile",
"build-classifier"
"aws-crt-java-build"
],
"!test_steps": [
"aws-crt-java-test"
]
},
"openwrt": {
"!packages": [],
"!build_steps": [
"mvn -P continuous-integration -B compile"
]
"!packages": []
}
}
},
Expand Down Expand Up @@ -146,9 +138,6 @@
"!build_env": {
"JAVA_HOME": "/usr/local/openjdk8"
},
"!build_steps": [
"mvn -P continuous-integration -B compile"
],
"!test_steps": [
"mvn -B test -DrerunFailingTestsCount=5"
]
Expand Down
6 changes: 3 additions & 3 deletions codebuild/cd/deploy-platform-specific-jars.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ if [[ $DEPLOY_VERSION = *-SNAPSHOT ]]; then
# snapshot doesn't need to gpg sign the file to deploy
DEPLOY_FILE_GOAL=deploy:deploy-file
DEPLOY_REPOSITORY_URL=https://aws.oss.sonatype.org/content/repositories/snapshots

else
# Need to sign the file to deploy to staging repo
DEPLOY_FILE_GOAL=gpg:sign-and-deploy-file
# pin to 3.1.0, somehow the latest 3.2.0 breaks the file deploy with 401.
DEPLOY_FILE_GOAL=org.apache.maven.plugins:maven-gpg-plugin:3.1.0:sign-and-deploy-file
DEPLOY_REPOSITORY_URL=https://aws.oss.sonatype.org:443/service/local/staging/deployByRepositoryId/${STAGING_REPO_ID}
fi

CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl")
CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl" "fips-compat")

for str in ${CLASSIFIERS_ARRAY[@]}; do
FILES="${FILES}target/aws-crt-1.0.0-SNAPSHOT-$str.jar,"
Expand Down
7 changes: 2 additions & 5 deletions codebuild/cd/deploy-snapshot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,9 @@ phases:
- mkdir -p target/cmake-build/lib
# mv all the platform-specific jars to target/
- aws s3 cp --recursive s3://aws-crt-java-pipeline/v${PKG_VERSION}/jar $CODEBUILD_SRC_DIR/aws-crt-java/target/
- mv $CODEBUILD_SRC_DIR_linux_x64/dist/aws-crt-1.0.0-SNAPSHOT-linux-x86_64.jar $CODEBUILD_SRC_DIR/aws-crt-java/target/
- mv $CODEBUILD_SRC_DIR_linux_x86/dist/aws-crt-1.0.0-SNAPSHOT-linux-x86_32.jar $CODEBUILD_SRC_DIR/aws-crt-java/target/
# cp all the shared libs to cmake-build
# cp all the shared libs to cmake-build/lib/
- aws s3 cp --recursive s3://aws-crt-java-pipeline/v${PKG_VERSION}/lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib
- cp -rv $CODEBUILD_SRC_DIR_linux_x64/dist/* $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/
- cp -rv $CODEBUILD_SRC_DIR_linux_x86/dist/* $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/
# log the downloaded files
- ls -alR $CODEBUILD_SRC_DIR/aws-crt-java/target
# install settings.xml to ~/.m2/settings.xml
- mkdir -p $HOME/.m2
Expand Down
33 changes: 33 additions & 0 deletions codebuild/cd/fips-compat-jar-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
version: 0.2
#this buildspec assumes the aws-common-runtime/ubuntu-16.04 image
# This job is responsible for artifacting the JAR which will have all of the other shared libs stuffed
# into it once all platforms are built and artifacted
phases:
install:
commands:
- sudo add-apt-repository ppa:openjdk-r/ppa
- sudo apt-get update -y
- sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f
build:
commands:
- cd $CODEBUILD_SRC_DIR/aws-crt-java
# upload artifacts to S3
- export GIT_TAG=$(git describe --tags)
- mkdir -p target/cmake-build/lib
# prepare fips-compat uber jar, download the regular libs
- aws s3 cp --recursive s3://aws-crt-java-pipeline/${GIT_TAG}/lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib
# Override with the FIPS libs
- aws s3 cp --recursive s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib
# Run a test to make sure we get the FIPS libs
- CRT_FIPS=ON mvn test -Dtest=software.amazon.awssdk.crt.test.SystemInfoTest#testIsFIPS -Dshared-lib.skip=true
- mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=fips-compat

post_build:
commands:
# upload artifacts to S3
- export GIT_TAG=$(git describe --tags)
- aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar

cache:
paths:
- "/root/.m2/**/*"
11 changes: 2 additions & 9 deletions codebuild/cd/generic-unix-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,16 @@

set -ex

cd `dirname $0`/../..
cd $(dirname $0)/../..

git submodule update --init

AWS_CRT_HOST=`uname | tr '[:upper:]' '[:lower:]'`-`uname -m`
AWS_CRT_HOST=$(uname | tr '[:upper:]' '[:lower:]')-$(uname -m)

if [ -z "$AWS_CRT_TARGET" ]; then
AWS_CRT_TARGET=$AWS_CRT_HOST
fi

SKIP_INSTALL=

if [[ "$AWS_CRT_TARGET" != "$AWS_CRT_HOST" ]]; then
SKIP_INSTALL=--skip-install
fi


if [[ $AWS_CRT_TARGET == linux-armv8 ]]; then
CLASSIFIER=linux-aarch_64
else
Expand Down
25 changes: 25 additions & 0 deletions codebuild/cd/linux-aarch64-fips-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env bash

set -ex

cd $(dirname $0)/../..

git submodule update --init
# double check aws-lc is the FIPS approved branch.
bash ./codebuild/cd/test-fips-branch.sh

# Pry the builder version this CRT is using out of ci.yml
BUILDER_VERSION=$(cat .github/workflows/ci.yml | grep 'BUILDER_VERSION:' | sed 's/\s*BUILDER_VERSION:\s*\(.*\)/\1/')
echo "Using builder version ${BUILDER_VERSION}"

aws s3 cp s3://aws-crt-builder/releases/${BUILDER_VERSION}/builder.pyz ./builder
chmod a+x builder

GIT_TAG=$(git describe --tags)

./builder build -p aws-crt-java run_tests=false --target=linux-arm64 --cmake-extra=-DCRT_FIPS=ON
mv target/cmake-build/aws-crt-java/* target/cmake-build/

JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=linux-aarch_64-fips

aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib
19 changes: 8 additions & 11 deletions codebuild/cd/manylinux-x64-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,21 @@ phases:
pre_build:
commands:
- export CC=gcc
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
- unzip awscliv2.zip
- ./aws/install
build:
commands:
- cd aws-crt-java
- git submodule update --init
- JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64 -Dcmake.disable_perl=OFF -Dcmake.disable_aws_lc_512avx=ON
- JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64

post_build:
commands:
# get the shared libs from the native build
- mkdir -p ../dist
- cp -rv target/cmake-build/lib ../dist/
# get the platform specific jar with classifier
- cp target/*.jar ../dist/

artifacts:
files:
- 'dist/**/*'

# upload artifacts to S3
- export GIT_TAG=$(git describe --tags)
- aws s3 cp --recursive --exclude "*" --include "*.so" ./target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib
- aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar
cache:
paths:
- '/root/.m2/**/*'
Loading

0 comments on commit 42b09c3

Please sign in to comment.