Speed up CodeBuild CI (#1426)

* Speed up CodeBuild CI:
1. Split OpenSSH into two jobs, one for main branch, and another for 8.9
2. Use RelWithDebInfo build for MySQL
3. Force MySQL to use double the number of jobs to run the tests
4. Update ARM fuzzer to run for a shorter period of time

* PR feedback: go back to using parallel=auto for mtr

Author: andrewhop

tests/ci/cdk/cdk/codebuild/github_ci_integration_omnibus.yaml

@@ -36,7 +36,7 @@ batch:
         variables:
           AWS_LC_CI_TARGET: "tests/ci/integration/run_crt_integration.sh"
 
-    - identifier: openssh_integration_x86_64
+    - identifier: openssh_integration_master_x86_64
       buildspec: tests/ci/codebuild/common/run_simple_target.yml
       env:
         type: LINUX_CONTAINER
@@ -45,8 +45,31 @@ batch:
         image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:amazonlinux-2023_clang-15x_sanitizer_latest
         variables:
           AWS_LC_CI_TARGET: "tests/ci/integration/run_openssh_integration.sh"
+          OPENSSH_BRANCH: "master"
 
-    - identifier: openssh_integration_aarch
+    - identifier: openssh_integration_8_9_x86_64
+      buildspec: tests/ci/codebuild/common/run_simple_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:amazonlinux-2023_clang-15x_sanitizer_latest
+        variables:
+          AWS_LC_CI_TARGET: "tests/ci/integration/run_openssh_integration.sh"
+          OPENSSH_BRANCH: "V_8_9"
+
+    - identifier: openssh_integration_master_aarch
+      buildspec: tests/ci/codebuild/common/run_simple_target.yml
+      env:
+        type: ARM_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_LARGE
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-aarch:amazonlinux-2023_clang-15x_sanitizer_latest
+        variables:
+          AWS_LC_CI_TARGET: "tests/ci/integration/run_openssh_integration.sh"
+          OPENSSH_BRANCH: "master"
+
+    - identifier: openssh_integration_8_9_aarch
       buildspec: tests/ci/codebuild/common/run_simple_target.yml
       env:
         type: ARM_CONTAINER
@@ -55,6 +78,7 @@ batch:
         image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-aarch:amazonlinux-2023_clang-15x_sanitizer_latest
         variables:
           AWS_LC_CI_TARGET: "tests/ci/integration/run_openssh_integration.sh"
+          OPENSSH_BRANCH: "V_8_9"
 
     - identifier: postgres_integration_x86_64
       buildspec: tests/ci/codebuild/common/run_nonroot_target.yml

tests/ci/integration/run_mysql_integration.sh

@@ -45,8 +45,8 @@ function mysql_patch_reminder() {
 }
 
 function mysql_build() {
-  cmake ${MYSQL_SRC_FOLDER} -GNinja -DWITH_BOOST=${BOOST_INSTALL_FOLDER} -DWITH_SSL=${AWS_LC_INSTALL_FOLDER} "-B${MYSQL_BUILD_FOLDER}"
-  ninja -C ${MYSQL_BUILD_FOLDER}
+  cmake ${MYSQL_SRC_FOLDER} -GNinja -DWITH_BOOST=${BOOST_INSTALL_FOLDER} -DWITH_SSL=${AWS_LC_INSTALL_FOLDER} "-B${MYSQL_BUILD_FOLDER}" -DCMAKE_BUILD_TYPE=RelWithDebInfo
+  time ninja -C ${MYSQL_BUILD_FOLDER}
   ls -R ${MYSQL_BUILD_FOLDER}
 }
 

tests/ci/integration/run_openssh_integration.sh

@@ -60,7 +60,6 @@ function openssh_build() {
 
 function checkout_openssh_branch() {
   pushd "${OPENSSH_WORKSPACE_FOLDER}"
-  make clean
   git clean -f -d
   git checkout --track origin/"$1"
   popd
@@ -80,24 +79,25 @@ function openssh_run_tests() {
 mkdir -p "${AWS_LC_BUILD_FOLDER}" "${AWS_LC_INSTALL_FOLDER}" "${OPENSSH_INSTALL_FOLDER}"
 
 # Get latest OpenSSH version.
-git clone https://github.com/openssh/openssh-portable.git
+git clone https://github.com/openssh/openssh-portable.git "${OPENSSH_WORKSPACE_FOLDER}"
 ls
 
-# Buld AWS-LC as a shared library
+# Build AWS-LC as a shared library
 aws_lc_build "$SRC_ROOT" "$AWS_LC_BUILD_FOLDER" "$AWS_LC_INSTALL_FOLDER" -DBUILD_TESTING=OFF -DBUILD_TOOL=OFF -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_SHARED_LIBS=1
 install_aws_lc
 
-CODEBUILD_SKIPPED_TESTS="agent-subprocess forwarding multiplex channel-timeout forward-control agent-restrict connection-timeout"
+if [ "$OPENSSH_BRANCH" != "master" ]; then
+  checkout_openssh_branch "$OPENSSH_BRANCH"
+fi
 
-# Using default branch. Build openssh and run tests.
 openssh_build
-openssh_run_tests "${CODEBUILD_SKIPPED_TESTS}"
 
-# Using branch V_8_9
-checkout_openssh_branch V_8_9
-openssh_build
-# In v8.9, the "percent" test requires the 'openssl' cli command
-openssh_run_tests "percent ${CODEBUILD_SKIPPED_TESTS}"
+CODEBUILD_SKIPPED_TESTS="agent-subprocess forwarding multiplex channel-timeout forward-control agent-restrict connection-timeout"
+if [ "$OPENSSH_BRANCH" == "V_8_9" ]; then
+    # In v8.9, the "percent" test requires the 'openssl' cli command
+    openssh_run_tests "percent ${CODEBUILD_SKIPPED_TESTS}"
+else
+    openssh_run_tests "${CODEBUILD_SKIPPED_TESTS}"
+fi
 
 popd
-

tests/ci/run_fuzz_tests.sh

@@ -15,8 +15,8 @@ set -u
 # 18 minutes for cleanup and merging files
 if [[ $PLATFORM == "aarch64" ]]; then
   # Arm sanitizers are very slow which causes the clean up time to take longer per
-  # fuzz test, only run for 16 minutes
-  TOTAL_FUZZ_TEST_TIME=1000
+  # fuzz test, only run for 8 minutes
+  TOTAL_FUZZ_TEST_TIME=500
 else
   TOTAL_FUZZ_TEST_TIME=1500
 fi