From 42b09c35099699ebea3a57e994a748c9cf57a3ce Mon Sep 17 00:00:00 2001 From: Dengke Tang Date: Fri, 15 Mar 2024 08:50:16 -0700 Subject: [PATCH] FIPS with classifier (#772) Co-authored-by: Michael Graeb --- .builder/actions/aws_crt_java_build.py | 22 ++++++++++ .builder/actions/build-classifier.py | 15 ------- .github/workflows/ci.yml | 41 +++++++++++++++++-- CMakeLists.txt | 25 ++++++++++- README.md | 5 +++ builder.json | 25 ++++------- codebuild/cd/deploy-platform-specific-jars.sh | 6 +-- codebuild/cd/deploy-snapshot.yml | 7 +--- codebuild/cd/fips-compat-jar-build.yml | 33 +++++++++++++++ codebuild/cd/generic-unix-build.sh | 11 +---- codebuild/cd/linux-aarch64-fips-build.sh | 25 +++++++++++ codebuild/cd/manylinux-x64-build.yml | 19 ++++----- codebuild/cd/manylinux-x64-fips-build.yml | 32 +++++++++++++++ codebuild/cd/manylinux-x86-build.yml | 16 ++++---- codebuild/cd/musl-linux-build.sh | 4 +- codebuild/cd/osx-arm64-build.sh | 3 +- codebuild/cd/osx-x64-build.sh | 4 +- codebuild/cd/test-fips-branch.sh | 24 +++++++++++ .../cd/test-platform-specific-jar-snapshot.sh | 2 +- crt/aws-lc | 2 +- pom.xml | 6 +-- .../java/software/amazon/awssdk/crt/CRT.java | 3 ++ src/native/crt.c | 30 +++++++++++++- .../amazon/awssdk/crt/test/S3ClientTest.java | 39 +++++++++++------- .../awssdk/crt/test/SystemInfoTest.java | 9 ++++ 25 files changed, 305 insertions(+), 103 deletions(-) create mode 100644 .builder/actions/aws_crt_java_build.py delete mode 100644 .builder/actions/build-classifier.py create mode 100644 codebuild/cd/fips-compat-jar-build.yml create mode 100644 codebuild/cd/linux-aarch64-fips-build.sh create mode 100644 codebuild/cd/manylinux-x64-fips-build.yml create mode 100755 codebuild/cd/test-fips-branch.sh diff --git a/.builder/actions/aws_crt_java_build.py b/.builder/actions/aws_crt_java_build.py new file mode 100644 index 000000000..dc220d0bf --- /dev/null +++ b/.builder/actions/aws_crt_java_build.py @@ -0,0 +1,22 @@ + +import Builder +import os +import argparse + + +class AWSCrtJavaBuild(Builder.Action): + + def run(self, env): + if os.getenv("CRT_FIPS") is not None: + env.shell.exec("mvn", "-P", "continuous-integration", "-B", "compile", + "-Dcmake.crt_fips=ON", check=True) + else: + env.shell.exec("mvn", "-P", "continuous-integration", + "-B", "compile", check=True) + + parser = argparse.ArgumentParser() + parser.add_argument('--classifier') + args = parser.parse_known_args(env.args.args)[0] + if args.classifier: + env.shell.exec("mvn", "-B", "install", "-DskipTests", "-Dshared-lib.skip=true", + f"-Dcrt.classifier={args.classifier}", check=True) diff --git a/.builder/actions/build-classifier.py b/.builder/actions/build-classifier.py deleted file mode 100644 index 55836d0be..000000000 --- a/.builder/actions/build-classifier.py +++ /dev/null @@ -1,15 +0,0 @@ - -import Builder -import sys -import os -import tempfile -from builder.core.host import current_host - -# This action is used by the musl-linux-build.sh script to build the binaries for release. -class BuildClassifier(Builder.Action): - - - def run(self, env): - crt_classifier = os.getenv("CRT_CLASSIFIER") - if crt_classifier is not None: - env.shell.exec("mvn", "-B", "install", "-DskipTests", "-Dshared-lib.skip=true", f"-Dcrt.classifier={crt_classifier}", check=True) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8bc609d1..8f445bdfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.55 + BUILDER_VERSION: v0.9.58 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-java @@ -101,6 +101,42 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + + linux-fips-x64: + runs-on: ubuntu-22.04 # latest + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Build ${{ env.PACKAGE_NAME }} + run: | + bash codebuild/cd/test-fips-branch.sh + export CRT_FIPS=ON + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + + + linux-fips-armv8: + runs-on: ubuntu-22.04 # latest + steps: + - name: Checkout Sources + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Build ${{ env.PACKAGE_NAME }} + run: | + bash codebuild/cd/test-fips-branch.sh + export CRT_FIPS=ON + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} --target=linux-arm64 --spec=downstream --cmake-extra=-DCRT_FIPS=ON + + # armv7 needs its own action due to inability to differentiate at runtime armv6 vs armv7 in the jvm: # # At build time we can properly figure out that we're targeting armv7. @@ -117,9 +153,8 @@ jobs: submodules: true - name: Build ${{ env.PACKAGE_NAME }} run: | - export AWS_CRT_ARCH=armv7 aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }} + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-alpine-3.16-armv7 build -p ${{ env.PACKAGE_NAME }} --classifier "armv7" raspberry: runs-on: ubuntu-22.04 # latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 082668493..5c9062578 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.1) project(aws-crt-jni C) option(BUILD_DEPS "Builds aws common runtime dependencies as part of build" ON) +option(CRT_FIPS "Whether to build aws-lc with FIPS compliance" OFF) if (POLICY CMP0069) cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags @@ -60,8 +61,24 @@ if (BUILD_DEPS) add_subdirectory(crt/aws-c-common) if (UNIX AND NOT APPLE) set(BUILD_LIBSSL OFF CACHE BOOL "Don't need libssl, only need libcrypto") - set(DISABLE_PERL ON CACHE BOOL "Disable codegen") - set(DISABLE_GO ON CACHE BOOL "Disable codegen") + message("Build with FIPS: " ${CRT_FIPS}) + if (CRT_FIPS) + set(FIPS ON CACHE BOOL "FIPS compliance") + set(PERL_EXECUTABLE "perl") + if (DEFINED ENV{GO_PATH}) + set(GO_EXECUTABLE $ENV{GO_PATH}/go) + message(STATUS "Overriding GO_EXECUTABLE to ${GO_EXECUTABLE}") + endif() + else() + set(DISABLE_PERL ON CACHE BOOL "Disable codegen") + set(DISABLE_GO ON CACHE BOOL "Disable codegen") + endif() + + if(CMAKE_C_COMPILER_ID MATCHES "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS "5.0") + set(DISABLE_PERL OFF CACHE BOOL "Build with Perl to avoid using pre-compiled binary with AVX512") + set(PERL_EXECUTABLE "perl") + set(MY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX ON CACHE BOOL "Disable AVX512 on old GCC that not supports it") + endif() # temporarily disable certain warnings as errors for the aws-lc build set(OLD_CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") @@ -107,6 +124,10 @@ include(AwsPlatformDetect) include(AwsSharedLibSetup) include(AwsCRuntime) +if (CRT_FIPS AND NOT FIPS) + message(FATAL_ERROR "CRT_FIPS can only be set when build with aws-lc.") +endif() + aws_determine_local_c_runtime(AWS_C_RUNTIME) file(GLOB AWS_CRT_JAVA_HEADERS diff --git a/README.md b/README.md index 604449de5..7ecb3ae79 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,10 @@ From maven: (https://search.maven.org/artifact/software.amazon.awssdk.crt/aws-cr The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled C libraries for many different platforms (Windows, Linux, etc). If size is an issue, you can pick a smaller platform-specific JAR by setting the ``. +The classifier `fips-compat` provides an "uber" jar with FIPS compliance on *some platforms*. **WARNING:** Platforms without FIPS compliance are also included in this jar, for compatibility's sake. Check `CRT.isFIPS()` at runtime to ensure you are on a FIPS compliant platform. The current breakdown is: +* **FIPS compliant**: linux-aarch_64, linux-x86_64 +* **NOT compliant**: linux-armv6, linux-armv7, linux-armv7-musl, linux-aarch_64-musl, linux-x86_32, linux-x86_64-musl, osx-aarch_64, osx-x86_64, windows-x86_32, windows-x86_64 + ``` xml @@ -115,6 +119,7 @@ The `aws-crt` JAR in Maven Central is a large "uber" jar that contains compiled - osx-x86_64 - windows-x86_32 - windows-x86_64 +- fips-compat (no auto-detect) ### Auto-detect diff --git a/builder.json b/builder.json index 94586bcfc..92b5e126b 100644 --- a/builder.json +++ b/builder.json @@ -4,18 +4,20 @@ "maven" ], "build_steps": [ - "mvn -P continuous-integration -B compile" + "aws-crt-java-build" ], "test_steps": [ "aws-crt-java-test" ], "+imports": [ - "JDK8" + "JDK8", + "golang" ], "build_dir": "target/cmake-build", "build_env": { "JAVA_HOME": "{java_home}", - "OVERRIDE_JAVA_HOME": "{java_home}" + "OVERRIDE_JAVA_HOME": "{java_home}", + "GOPROXY": "https://goproxy.io,direct" }, "test_env": { "JAVA_HOME": "{java_home}", @@ -35,12 +37,6 @@ }, "targets": { "linux": { - "!build_steps": [ - "mvn -P continuous-integration -B compile" - ], - "!test_steps": [ - "aws-crt-java-test" - ], "architectures": { "armv6": { "!packages": [], @@ -87,18 +83,14 @@ "openjdk8" ], "!build_steps": [ - "mvn -P continuous-integration -B compile", - "build-classifier" + "aws-crt-java-build" ], "!test_steps": [ "aws-crt-java-test" ] }, "openwrt": { - "!packages": [], - "!build_steps": [ - "mvn -P continuous-integration -B compile" - ] + "!packages": [] } } }, @@ -146,9 +138,6 @@ "!build_env": { "JAVA_HOME": "/usr/local/openjdk8" }, - "!build_steps": [ - "mvn -P continuous-integration -B compile" - ], "!test_steps": [ "mvn -B test -DrerunFailingTestsCount=5" ] diff --git a/codebuild/cd/deploy-platform-specific-jars.sh b/codebuild/cd/deploy-platform-specific-jars.sh index d8fefba88..a8f79b2e8 100755 --- a/codebuild/cd/deploy-platform-specific-jars.sh +++ b/codebuild/cd/deploy-platform-specific-jars.sh @@ -6,14 +6,14 @@ if [[ $DEPLOY_VERSION = *-SNAPSHOT ]]; then # snapshot doesn't need to gpg sign the file to deploy DEPLOY_FILE_GOAL=deploy:deploy-file DEPLOY_REPOSITORY_URL=https://aws.oss.sonatype.org/content/repositories/snapshots - else # Need to sign the file to deploy to staging repo - DEPLOY_FILE_GOAL=gpg:sign-and-deploy-file + # pin to 3.1.0, somehow the latest 3.2.0 breaks the file deploy with 401. + DEPLOY_FILE_GOAL=org.apache.maven.plugins:maven-gpg-plugin:3.1.0:sign-and-deploy-file DEPLOY_REPOSITORY_URL=https://aws.oss.sonatype.org:443/service/local/staging/deployByRepositoryId/${STAGING_REPO_ID} fi -CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl") +CLASSIFIERS_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl" "fips-compat") for str in ${CLASSIFIERS_ARRAY[@]}; do FILES="${FILES}target/aws-crt-1.0.0-SNAPSHOT-$str.jar," diff --git a/codebuild/cd/deploy-snapshot.yml b/codebuild/cd/deploy-snapshot.yml index 5082941ca..1b93f34e1 100644 --- a/codebuild/cd/deploy-snapshot.yml +++ b/codebuild/cd/deploy-snapshot.yml @@ -26,12 +26,9 @@ phases: - mkdir -p target/cmake-build/lib # mv all the platform-specific jars to target/ - aws s3 cp --recursive s3://aws-crt-java-pipeline/v${PKG_VERSION}/jar $CODEBUILD_SRC_DIR/aws-crt-java/target/ - - mv $CODEBUILD_SRC_DIR_linux_x64/dist/aws-crt-1.0.0-SNAPSHOT-linux-x86_64.jar $CODEBUILD_SRC_DIR/aws-crt-java/target/ - - mv $CODEBUILD_SRC_DIR_linux_x86/dist/aws-crt-1.0.0-SNAPSHOT-linux-x86_32.jar $CODEBUILD_SRC_DIR/aws-crt-java/target/ - # cp all the shared libs to cmake-build + # cp all the shared libs to cmake-build/lib/ - aws s3 cp --recursive s3://aws-crt-java-pipeline/v${PKG_VERSION}/lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib - - cp -rv $CODEBUILD_SRC_DIR_linux_x64/dist/* $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/ - - cp -rv $CODEBUILD_SRC_DIR_linux_x86/dist/* $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/ + # log the downloaded files - ls -alR $CODEBUILD_SRC_DIR/aws-crt-java/target # install settings.xml to ~/.m2/settings.xml - mkdir -p $HOME/.m2 diff --git a/codebuild/cd/fips-compat-jar-build.yml b/codebuild/cd/fips-compat-jar-build.yml new file mode 100644 index 000000000..eb2b67d4e --- /dev/null +++ b/codebuild/cd/fips-compat-jar-build.yml @@ -0,0 +1,33 @@ +version: 0.2 +#this buildspec assumes the aws-common-runtime/ubuntu-16.04 image +# This job is responsible for artifacting the JAR which will have all of the other shared libs stuffed +# into it once all platforms are built and artifacted +phases: + install: + commands: + - sudo add-apt-repository ppa:openjdk-r/ppa + - sudo apt-get update -y + - sudo apt-get install openjdk-11-jdk-headless maven wget unzip -y -f + build: + commands: + - cd $CODEBUILD_SRC_DIR/aws-crt-java + # upload artifacts to S3 + - export GIT_TAG=$(git describe --tags) + - mkdir -p target/cmake-build/lib + # prepare fips-compat uber jar, download the regular libs + - aws s3 cp --recursive s3://aws-crt-java-pipeline/${GIT_TAG}/lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib + # Override with the FIPS libs + - aws s3 cp --recursive s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib $CODEBUILD_SRC_DIR/aws-crt-java/target/cmake-build/lib + # Run a test to make sure we get the FIPS libs + - CRT_FIPS=ON mvn test -Dtest=software.amazon.awssdk.crt.test.SystemInfoTest#testIsFIPS -Dshared-lib.skip=true + - mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=fips-compat + + post_build: + commands: + # upload artifacts to S3 + - export GIT_TAG=$(git describe --tags) + - aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar + +cache: + paths: + - "/root/.m2/**/*" diff --git a/codebuild/cd/generic-unix-build.sh b/codebuild/cd/generic-unix-build.sh index 5b7d458f5..2dd4174d4 100755 --- a/codebuild/cd/generic-unix-build.sh +++ b/codebuild/cd/generic-unix-build.sh @@ -2,23 +2,16 @@ set -ex -cd `dirname $0`/../.. +cd $(dirname $0)/../.. git submodule update --init -AWS_CRT_HOST=`uname | tr '[:upper:]' '[:lower:]'`-`uname -m` +AWS_CRT_HOST=$(uname | tr '[:upper:]' '[:lower:]')-$(uname -m) if [ -z "$AWS_CRT_TARGET" ]; then AWS_CRT_TARGET=$AWS_CRT_HOST fi -SKIP_INSTALL= - -if [[ "$AWS_CRT_TARGET" != "$AWS_CRT_HOST" ]]; then - SKIP_INSTALL=--skip-install -fi - - if [[ $AWS_CRT_TARGET == linux-armv8 ]]; then CLASSIFIER=linux-aarch_64 else diff --git a/codebuild/cd/linux-aarch64-fips-build.sh b/codebuild/cd/linux-aarch64-fips-build.sh new file mode 100644 index 000000000..edece0f05 --- /dev/null +++ b/codebuild/cd/linux-aarch64-fips-build.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -ex + +cd $(dirname $0)/../.. + +git submodule update --init +# double check aws-lc is the FIPS approved branch. +bash ./codebuild/cd/test-fips-branch.sh + +# Pry the builder version this CRT is using out of ci.yml +BUILDER_VERSION=$(cat .github/workflows/ci.yml | grep 'BUILDER_VERSION:' | sed 's/\s*BUILDER_VERSION:\s*\(.*\)/\1/') +echo "Using builder version ${BUILDER_VERSION}" + +aws s3 cp s3://aws-crt-builder/releases/${BUILDER_VERSION}/builder.pyz ./builder +chmod a+x builder + +GIT_TAG=$(git describe --tags) + +./builder build -p aws-crt-java run_tests=false --target=linux-arm64 --cmake-extra=-DCRT_FIPS=ON +mv target/cmake-build/aws-crt-java/* target/cmake-build/ + +JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 mvn -B package -DskipTests -Dshared-lib.skip=true -Dcrt.classifier=linux-aarch_64-fips + +aws s3 cp --recursive --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib diff --git a/codebuild/cd/manylinux-x64-build.yml b/codebuild/cd/manylinux-x64-build.yml index 9c7548c20..998dda2d3 100644 --- a/codebuild/cd/manylinux-x64-build.yml +++ b/codebuild/cd/manylinux-x64-build.yml @@ -7,24 +7,21 @@ phases: pre_build: commands: - export CC=gcc + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install build: commands: - cd aws-crt-java - git submodule update --init - - JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64 -Dcmake.disable_perl=OFF -Dcmake.disable_aws_lc_512avx=ON + - JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64 post_build: commands: - # get the shared libs from the native build - - mkdir -p ../dist - - cp -rv target/cmake-build/lib ../dist/ - # get the platform specific jar with classifier - - cp target/*.jar ../dist/ - -artifacts: - files: - - 'dist/**/*' - + # upload artifacts to S3 + - export GIT_TAG=$(git describe --tags) + - aws s3 cp --recursive --exclude "*" --include "*.so" ./target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib + - aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar cache: paths: - '/root/.m2/**/*' diff --git a/codebuild/cd/manylinux-x64-fips-build.yml b/codebuild/cd/manylinux-x64-fips-build.yml new file mode 100644 index 000000000..90b9416d7 --- /dev/null +++ b/codebuild/cd/manylinux-x64-fips-build.yml @@ -0,0 +1,32 @@ +version: 0.2 +#this build spec assumes the manylinux CentOS5 custom image +#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4, openjdk8, maven 3.6.0, gnupg 2.0.10 +phases: + install: + commands: + pre_build: + commands: + - export CC=gcc + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + build: + commands: + - cd aws-crt-java + - git submodule update --init + # double check aws-lc is the FIPS approved branch. + - bash ./codebuild/cd/test-fips-branch.sh + - curl -OL https://go.dev/dl/go1.21.6.linux-amd64.tar.gz && mkdir ./go + - tar -C ./go -xvf go1.21.6.linux-amd64.tar.gz + - export PATH=$PATH:./go/go/bin + - JAVA_HOME=/opt/java-se-8u40-ri/ mvn -B package -DskipTests -Dcrt.classifier=linux-x86_64-fips -Dcmake.crt_fips=ON + + post_build: + commands: + # upload artifacts to S3 + - export GIT_TAG=$(git describe --tags) + - aws s3 cp --recursive --exclude "*" --include "*.so" target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/fips_lib + +cache: + paths: + - '/root/.m2/**/*' diff --git a/codebuild/cd/manylinux-x86-build.yml b/codebuild/cd/manylinux-x86-build.yml index 09148ac33..c82f62652 100644 --- a/codebuild/cd/manylinux-x86-build.yml +++ b/codebuild/cd/manylinux-x86-build.yml @@ -7,6 +7,9 @@ phases: pre_build: commands: - export CC=gcc + # AWSCLI v2 not available on x86, so installing v1 + - /opt/python/cp37-cp37m/bin/python -m pip install --upgrade awscli + - ln -s `find /opt -name aws` /usr/local/bin/aws build: commands: - cd aws-crt-java @@ -14,15 +17,10 @@ phases: - JAVA_HOME=/opt/jdk1.8.0_201/ setarch i386 mvn -B package -DskipTests -Punix-x86 -Dcrt.classifier=linux-x86_32 post_build: commands: - # get the shared libs from the native build - - mkdir -p ../dist - - cp -rv target/cmake-build/lib ../dist/ - # get the platform specific jar with classifier - - cp target/*.jar ../dist/ - -artifacts: - files: - - 'dist/**/*' + # upload artifacts to S3 + - export GIT_TAG=$(git describe --tags) + - aws s3 cp --recursive --exclude "*" --include "*.so" ./target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib + - aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar cache: paths: diff --git a/codebuild/cd/musl-linux-build.sh b/codebuild/cd/musl-linux-build.sh index 1117bb124..4613c18ce 100755 --- a/codebuild/cd/musl-linux-build.sh +++ b/codebuild/cd/musl-linux-build.sh @@ -16,10 +16,8 @@ export DOCKER_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/${IMAGE_NAME}:$ export QEMU_IMAGE=123124136734.dkr.ecr.us-east-1.amazonaws.com/multiarch-qemu-user-static:latest docker run --rm --privileged ${QEMU_IMAGE} --reset -p yes - -export CRT_CLASSIFIER=${CLASSIFIER} export BRANCH_TAG=$(git describe --tags) -docker run --mount type=bind,src=$(pwd),dst=/root/aws-crt-java --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION --env CXXFLAGS --env AWS_CRT_ARCH --env CRT_CLASSIFIER $DOCKER_IMAGE --version=${BUILDER_VERSION} build -p aws-crt-java --branch ${BRANCH_TAG} run_tests=false +docker run --mount type=bind,src=$(pwd),dst=/root/aws-crt-java --env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY --env AWS_DEFAULT_REGION --env CXXFLAGS --env AWS_CRT_ARCH $DOCKER_IMAGE --version=${BUILDER_VERSION} build -p aws-crt-java --classifier ${CLASSIFIER} --branch ${BRANCH_TAG} run_tests=false docker container prune -f # Upload the artifacts to S3 diff --git a/codebuild/cd/osx-arm64-build.sh b/codebuild/cd/osx-arm64-build.sh index feefc1956..36d864eb2 100755 --- a/codebuild/cd/osx-arm64-build.sh +++ b/codebuild/cd/osx-arm64-build.sh @@ -2,7 +2,7 @@ set -ex -cd `dirname $0`/../.. +cd $(dirname $0)/../.. git submodule update --init export GIT_TAG=$(git describe --tags) @@ -15,4 +15,5 @@ mkdir -p ../dist cp -rv target/cmake-build/lib ../dist/ aws s3 cp --recursive --exclude "*" --include "*.dylib" ./target/cmake-build/lib s3://aws-crt-java-pipeline/${GIT_TAG}/lib + aws s3 cp --recursive --exclude "*" --include "aws-crt*.jar" ./target s3://aws-crt-java-pipeline/${GIT_TAG}/jar diff --git a/codebuild/cd/osx-x64-build.sh b/codebuild/cd/osx-x64-build.sh index ed4e6aa91..5aeed52ef 100755 --- a/codebuild/cd/osx-x64-build.sh +++ b/codebuild/cd/osx-x64-build.sh @@ -2,13 +2,13 @@ set -ex -cd `dirname $0`/../.. +cd $(dirname $0)/../.. git submodule update --init export GIT_TAG=$(git describe --tags) -mvn -B package -DskipTests -P mac-x64 -Dcrt.classifier=osx-x86_64 +mvn -B package -DskipTests -P mac-x64 -Dcrt.classifier=osx-x86_64 # Copy artifacts to dist mkdir -p ../dist diff --git a/codebuild/cd/test-fips-branch.sh b/codebuild/cd/test-fips-branch.sh new file mode 100755 index 000000000..903d620fd --- /dev/null +++ b/codebuild/cd/test-fips-branch.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e +set -x + +# Store the current working directory +original_dir=$(pwd) +cd ./crt/aws-lc || exit 1 + +# Get the current commit hash +current_commit=$(git rev-parse HEAD) + +# Check if the current commit is from the "fips-2022-11-02" branch +if git merge-base --is-ancestor "$current_commit" "origin/fips-2022-11-02"; then + echo "Current aws-lc commit is from the 'fips-2022-11-02' branch" + status=0 +else + echo "Error: Current aws-lc commit is not from the 'fips-2022-11-02' branch" + status=1 +fi + +# Change back to the original working directory +cd "$original_dir" || exit 1 + +exit $status diff --git a/codebuild/cd/test-platform-specific-jar-snapshot.sh b/codebuild/cd/test-platform-specific-jar-snapshot.sh index 96fdee3b9..9cd8aca19 100644 --- a/codebuild/cd/test-platform-specific-jar-snapshot.sh +++ b/codebuild/cd/test-platform-specific-jar-snapshot.sh @@ -2,7 +2,7 @@ set -ex -PLATFORM_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl") +PLATFORM_ARRAY=("linux-armv6" "linux-armv7" "linux-aarch_64" "linux-x86_32" "linux-x86_64" "osx-aarch_64" "osx-x86_64" "windows-x86_32" "windows-x86_64" "linux-x86_64-musl" "linux-armv7-musl" "linux-aarch_64-musl" "fips-compat") # test uber jar mvn -B dependency:get -DrepoUrl=https://aws.oss.sonatype.org/content/repositories/snapshots -Dartifact=software.amazon.awssdk.crt:aws-crt:${CRT_VERSION}-SNAPSHOT -Dtransitive=false diff --git a/crt/aws-lc b/crt/aws-lc index 19d9ace40..e61907c65 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 19d9ace40f6770e062b1e9ec1d46935b300b948e +Subproject commit e61907c65c2d529ec3399e719653b5d07613acb1 diff --git a/pom.xml b/pom.xml index ccad9d1ec..3178ba924 100644 --- a/pom.xml +++ b/pom.xml @@ -43,8 +43,7 @@ OFF -DOSX_ARCH_DUMMY=1 -DOSX_DEPLOYMENT_TARGET_DUMMY=1 - ON - OFF + OFF @@ -183,8 +182,7 @@ ${cmake.osx_arch} -DS2N_NO_PQ_ASM=${cmake.s2nNoPqAsm} -DBUILD_TESTING=OFF - -DDISABLE_PERL=${cmake.disable_perl} - -DMY_ASSEMBLER_IS_TOO_OLD_FOR_512AVX=${cmake.disable_aws_lc_512avx} + -DCRT_FIPS=${cmake.crt_fips} --no-warn-unused-cli ${cmake.generator} ${cmake.toolset} diff --git a/src/main/java/software/amazon/awssdk/crt/CRT.java b/src/main/java/software/amazon/awssdk/crt/CRT.java index 0a69f0dff..6805c6279 100644 --- a/src/main/java/software/amazon/awssdk/crt/CRT.java +++ b/src/main/java/software/amazon/awssdk/crt/CRT.java @@ -548,7 +548,10 @@ public static void checkJniExceptionContract(boolean clearException) { nativeCheckJniExceptionContract(clearException); } + public static native boolean isFIPS(); + private static native void nativeCheckJniExceptionContract(boolean clearException); private static native void onJvmShutdown(); + }; diff --git a/src/native/crt.c b/src/native/crt.c index e9cec38f2..880e5bd9d 100644 --- a/src/native/crt.c +++ b/src/native/crt.c @@ -24,14 +24,19 @@ #include #include -#include - #include "crt.h" #include "java_class_ids.h" #include "logging.h" +#include + +#ifdef AWS_OS_LINUX +# include +#endif /* 0 = off, 1 = bytes, 2 = stack traces, see aws_mem_trace_level */ int g_memory_tracing = 0; +int g_crt_fips_mode = 0; + static struct aws_allocator *s_init_allocator(void) { if (g_memory_tracing) { struct aws_allocator *allocator = aws_default_allocator(); @@ -585,6 +590,19 @@ void JNICALL Java_software_amazon_awssdk_crt_CRT_awsCrtInit( g_memory_tracing = 1; } +/* FIPS mode can be checked if OpenSSL was configured and built for FIPS which then defines OPENSSL_FIPS. + * + * AWS-LC always defines FIPS_mode() that you can call and check what the library was built with. It does not define + * a public OPENSSL_FIPS/AWSLC_FIPS macro that we can (or need to) check here + * + * Safeguard with macro's, for example because Libressl dosn't define + * FIPS_mode() by default. + * */ +#if defined(OPENSSL_FIPS) || defined(OPENSSL_IS_AWSLC) + /* Try to enable fips mode for libcrypto */ + g_crt_fips_mode = FIPS_mode_set(1); +#endif + /* NOT using aws_jni_get_allocator to avoid trace leak outside the test */ struct aws_allocator *allocator = aws_default_allocator(); aws_mqtt_library_init(allocator); @@ -654,6 +672,14 @@ void JNICALL Java_software_amazon_awssdk_crt_CRT_dumpNativeMemory(JNIEnv *env, j } } +JNIEXPORT +jboolean JNICALL Java_software_amazon_awssdk_crt_CRT_isFIPS(JNIEnv *env, jclass jni_crt_class) { + (void)env; + (void)jni_crt_class; + + return g_crt_fips_mode == 1; +} + jstring aws_jni_string_from_cursor(JNIEnv *env, const struct aws_byte_cursor *native_data) { struct aws_string *string = aws_string_new_from_array(aws_jni_get_allocator(), native_data->ptr, native_data->len); AWS_FATAL_ASSERT(string != NULL); diff --git a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java index 738b2d158..8609183b5 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/S3ClientTest.java @@ -762,8 +762,8 @@ private byte[] createTestPayload(int size) { } private void testS3PutHelper(boolean useFile, boolean unknownContentLength, String objectPath, boolean s3express, - int contentLength) throws IOException { - S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express); + int contentLength, boolean contentMD5) throws IOException { + S3ClientOptions clientOptions = new S3ClientOptions().withRegion(REGION).withEnableS3Express(s3express).withComputeContentMd5(contentMD5); Path uploadFilePath = Files.createTempFile("testS3PutFilePath", ".txt"); try (S3Client client = createS3Client(clientOptions)) { CompletableFuture onFinishedFuture = new CompletableFuture<>(); @@ -788,8 +788,7 @@ public void onFinished(S3FinishedResponseContext context) { }; HttpHeader[] headers = { - new HttpHeader("Host", s3express ? S3EXPRESS_ENDPOINT : ENDPOINT), - new HttpHeader("x-amz-sdk-checksum-algorithm", "SHA1") + new HttpHeader("Host", s3express ? S3EXPRESS_ENDPOINT : ENDPOINT) }; HttpRequest httpRequest; String path = objectPath == null ? "/put_object_test_10MB.txt" : objectPath; @@ -824,12 +823,15 @@ public long getLength() { new HttpHeader("Content-Length", Integer.valueOf(contentLength).toString())); } AwsSigningConfig config = AwsSigningConfig.getDefaultS3SigningConfig(REGION, null); - ChecksumConfig checksumConfig = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.SHA1) - .withChecksumLocation(ChecksumLocation.TRAILER).withValidateChecksum(true); S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions() .withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest) - .withResponseHandler(responseHandler) - .withChecksumConfig(checksumConfig); + .withResponseHandler(responseHandler); + + if(!contentMD5) { + ChecksumConfig checksumConfig = new ChecksumConfig().withChecksumAlgorithm(ChecksumAlgorithm.SHA1) + .withChecksumLocation(ChecksumLocation.TRAILER).withValidateChecksum(true); + metaRequestOptions = metaRequestOptions.withChecksumConfig(checksumConfig); + } if (s3express) { config.setAlgorithm(AwsSigningConfig.AwsSigningAlgorithm.SIGV4_S3EXPRESS); metaRequestOptions = metaRequestOptions.withSigningConfig(config); @@ -852,7 +854,16 @@ public void testS3Put() throws IOException { skipIfAndroid(); skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(false, false, null, false, 16 * 1024 * 1024); + testS3PutHelper(false, false, null, false, 16 * 1024 * 1024, false); + } + + // MD5 is not FIPS allowed. Make sure the cypto lib we used still supports MD5. + @Test + public void testS3PutWithMD5() throws IOException { + skipIfAndroid(); + skipIfNetworkUnavailable(); + Assume.assumeTrue(hasAwsCredentials()); + testS3PutHelper(false, true, null, false, 16 * 1024 * 1024, true); } // Test that we can upload by passing a filepath instead of an HTTP body stream @@ -861,7 +872,7 @@ public void testS3PutFilePath() throws IOException { skipIfAndroid(); skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(true, false, null, false, 10 * 1024 * 1024); + testS3PutHelper(true, false, null, false, 10 * 1024 * 1024, false); } // Test that we can upload without provide the content length @@ -870,7 +881,7 @@ public void testS3PutUnknownContentLength() throws IOException { skipIfAndroid(); skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(false, true, null, false, 10 * 1024 * 1024); + testS3PutHelper(false, true, null, false, 10 * 1024 * 1024, false); } // Test that we can upload to a path with special characters @@ -879,21 +890,21 @@ public void testS3PutSpecialCharPath() throws IOException { skipIfAndroid(); skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(false, true, "/put_object_test_10MB@$%.txt", false, 10 * 1024 * 1024); + testS3PutHelper(false, true, "/put_object_test_10MB@$%.txt", false, 10 * 1024 * 1024, false); } @Test public void testS3PutS3Express() throws IOException { skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(false, false, null, true, 16 * 1024 * 1024); + testS3PutHelper(false, false, null, true, 16 * 1024 * 1024, false); } @Test public void testS3PutS3ExpressSpecialCharPath() throws IOException { skipIfNetworkUnavailable(); Assume.assumeTrue(hasAwsCredentials()); - testS3PutHelper(false, true, "/put_object_test_10MB@$%.txt", true, 10 * 1024 * 1024); + testS3PutHelper(false, true, "/put_object_test_10MB@$%.txt", true, 10 * 1024 * 1024, false); } // Test that passing a nonexistent file path will cause an error diff --git a/src/test/java/software/amazon/awssdk/crt/test/SystemInfoTest.java b/src/test/java/software/amazon/awssdk/crt/test/SystemInfoTest.java index 697bca555..1560a0057 100644 --- a/src/test/java/software/amazon/awssdk/crt/test/SystemInfoTest.java +++ b/src/test/java/software/amazon/awssdk/crt/test/SystemInfoTest.java @@ -6,10 +6,13 @@ package software.amazon.awssdk.crt.test; import org.junit.Test; + import software.amazon.awssdk.crt.*; import static org.junit.Assert.*; +import org.junit.Assume; + public class SystemInfoTest extends CrtTestFixture { public SystemInfoTest() { } @@ -35,4 +38,10 @@ public void testCpuIteration() { assertEquals(processorCount, iteratedCpuCount); } + + @Test + public void testIsFIPS() { + Assume.assumeTrue(System.getenv("CRT_FIPS") != null); + assertTrue(CRT.isFIPS()); + } }