From 10418eeb55590d06d20f59b944c643c037a5c4a9 Mon Sep 17 00:00:00 2001 From: Benson Ma Date: Mon, 3 Apr 2023 12:19:02 -0700 Subject: [PATCH] [T149624848] Break up setup_env.bash - Break up setup_env.bash into separate files for easier maintainability --- .github/scripts/base_utilities.bash | 154 ++++++++++++ .github/scripts/fbgemm_gpu_docs.bash | 73 ++++++ .github/scripts/fbgemm_gpu_lint.bash | 137 +++++++++++ .github/scripts/setup_env.bash | 324 +------------------------- .github/workflows/fbgemm_gpu_lint.yml | 2 +- 5 files changed, 371 insertions(+), 319 deletions(-) create mode 100644 .github/scripts/base_utilities.bash create mode 100644 .github/scripts/fbgemm_gpu_docs.bash create mode 100644 .github/scripts/fbgemm_gpu_lint.bash diff --git a/.github/scripts/base_utilities.bash b/.github/scripts/base_utilities.bash new file mode 100644 index 0000000000..755a0d8aea --- /dev/null +++ b/.github/scripts/base_utilities.bash @@ -0,0 +1,154 @@ +#!/bin/bash + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +################################################################################ +# Command Execution Functions +################################################################################ + +print_exec () { + echo "+ $*" + echo "" + if "$@"; then + local retcode=0 + else + local retcode=$? + fi + echo "" + return $retcode +} + +exec_with_retries () { + local max=5 + local delay=2 + local retcode=0 + + for i in $(seq 1 ${max}); do + echo "[EXEC] [ATTEMPT ${i}/${max}] + $*" + + if "$@"; then + retcode=0 + break + else + retcode=$? + echo "[EXEC] [ATTEMPT ${i}/${max}] Command attempt failed." + echo "" + sleep $delay + fi + done + + if [ $retcode -ne 0 ]; then + echo "[EXEC] The command has failed after ${max} attempts; aborting." + fi + + return $retcode +} + + +################################################################################ +# Assert Functions +################################################################################ + +test_python_import () { + local env_name="$1" + local python_import="$2" + if [ "$python_import" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME PYTHON_IMPORT" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env numpy" + return 1 + fi + + if conda run -n "${env_name}" python -c "import ${python_import}"; then + echo "[CHECK] Python package ${python_import} found." + else + echo "[CHECK] Python package ${python_import} not found!" + return 1 + fi +} + +test_binpath () { + local env_name="$1" + local bin_name="$2" + if [ "$bin_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME BIN_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env nvcc" + return 1 + fi + + if conda run -n "${env_name}" which "${bin_name}"; then + echo "[CHECK] Binary ${bin_name} found in PATH" + else + echo "[CHECK] Binary ${bin_name} not found in PATH!" + return 1 + fi +} + +test_filepath () { + local env_name="$1" + local file_name="$2" + if [ "$file_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME FILE_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env cuda_runtime.h" + return 1 + fi + + # shellcheck disable=SC2155 + local conda_prefix=$(conda run -n "${env_name}" printenv CONDA_PREFIX) + # shellcheck disable=SC2155 + local file_path=$(find "${conda_prefix}" -type f -name "${file_name}") + # shellcheck disable=SC2155 + local link_path=$(find "${conda_prefix}" -type l -name "${file_name}") + if [ "${file_path}" != "" ]; then + echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (file): ${file_path}" + elif [ "${link_path}" != "" ]; then + echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (symbolic link): ${link_path}" + else + echo "[CHECK] ${file_name} not found in CONDA_PREFIX PATH!" + return 1 + fi +} + +test_env_var () { + local env_name="$1" + local env_key="$2" + if [ "$env_key" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME ENV_KEY" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env CUDNN_INCLUDE_DIR" + return 1 + fi + + if conda run -n "${env_name}" printenv "${env_key}"; then + echo "[CHECK] Environment variable ${env_key} is defined in the Conda environment" + else + echo "[CHECK] Environment variable ${env_key} is not defined in the Conda environment!" + return 1 + fi +} + +test_library_symbol () { + local lib_path="$1" + local lib_symbol="$2" + if [ "$lib_symbol" == "" ]; then + echo "Usage: ${FUNCNAME[0]} LIB_PATH FULL_NAMESPACE_PATH_LIB_SYMBOL" + echo "Example(s):" + echo " ${FUNCNAME[0]} fbgemm_gpu_py.so fbgemm_gpu::merge_pooled_embeddings" + return 1 + fi + + # Add space and '(' to the grep string to get the full method path + symbol_entries=$(nm -gDC "${lib_path}" | grep " ${lib_symbol}(") + if [ "${symbol_entries}" != "" ]; then + echo "[CHECK] Found symbol in ${lib_path}: ${lib_symbol}" + else + echo "[CHECK] Symbol NOT found in ${lib_path}: ${lib_symbol}" + return 1 + fi +} diff --git a/.github/scripts/fbgemm_gpu_docs.bash b/.github/scripts/fbgemm_gpu_docs.bash new file mode 100644 index 0000000000..1444319716 --- /dev/null +++ b/.github/scripts/fbgemm_gpu_docs.bash @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/base_utilities.bash" + +################################################################################ +# Docs Tools Setup Functions +################################################################################ + +install_docs_tools () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Install Documentation Tools" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + echo "[INSTALL] Installing docs tools ..." + (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \ + doxygen) || return 1 + + # Check binaries are visible in the PAATH + (test_binpath "${env_name}" doxygen) || return 1 + + echo "[INSTALL] Successfully installed all the docs tools" +} + + +################################################################################ +# FBGEMM_GPU Docs Functions +################################################################################ + +build_fbgemm_gpu_docs () { + env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env # Build the docs" + return 1 + else + echo "################################################################################" + echo "# Build FBGEMM-GPU Documentation" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + echo "[BUILD] Installing docs-build dependencies ..." + (exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1 + + echo "[BUILD] Running Doxygen build ..." + (exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1 + + echo "[BUILD] Building HTML pages ..." + (exec_with_retries conda run -n "${env_name}" make html) || return 1 + + echo "[INSTALL] FBGEMM-GPU documentation build completed" +} diff --git a/.github/scripts/fbgemm_gpu_lint.bash b/.github/scripts/fbgemm_gpu_lint.bash new file mode 100644 index 0000000000..14a69c19bf --- /dev/null +++ b/.github/scripts/fbgemm_gpu_lint.bash @@ -0,0 +1,137 @@ +#!/bin/bash + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + + +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/base_utilities.bash" + +################################################################################ +# Lint Tools Setup Functions +################################################################################ + +install_lint_tools () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Install Lint Tools" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + echo "[INSTALL] Installing lint tools ..." + (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \ + click \ + flake8 \ + ufmt) || return 1 + + # Check binaries are visible in the PAATH + (test_binpath "${env_name}" flake8) || return 1 + (test_binpath "${env_name}" ufmt) || return 1 + + # Check Python packages are importable + local import_tests=( click ) + for p in "${import_tests[@]}"; do + (test_python_import "${env_name}" "${p}") || return 1 + done + + echo "[INSTALL] Successfully installed all the lint tools" +} + +################################################################################ +# FBGEMM_GPU Lint Functions +################################################################################ + +lint_fbgemm_gpu_flake8 () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run FBGEMM_GPU Lint: flake8" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json" + + # E501 = line too long + # W503 = line break before binary operator (deprecated) + # E203 = whitespace before ":" + (print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1 + + echo "[TEST] Finished running flake8 lint checks" +} + +lint_fbgemm_gpu_ufmt () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run FBGEMM_GPU Lint: ufmt" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + local lint_paths=( + fbgemm_gpu/fbgemm_gpu + fbgemm_gpu/test + fbgemm_gpu/bench + ) + + for p in "${lint_paths[@]}"; do + (print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1 + done + + echo "[TEST] Finished running ufmt lint checks" +} + +lint_fbgemm_gpu_copyright () { + local env_name="$1" + if [ "$env_name" == "" ]; then + echo "Usage: ${FUNCNAME[0]} ENV_NAME" + echo "Example(s):" + echo " ${FUNCNAME[0]} build_env" + return 1 + else + echo "################################################################################" + echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers" + echo "#" + echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" + echo "################################################################################" + echo "" + fi + + local lint_paths=( + fbgemm_gpu/fbgemm_gpu + fbgemm_gpu/test + fbgemm_gpu/bench + ) + + for p in "${lint_paths[@]}"; do + (print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1 + done + + echo "[TEST] Finished running Meta Copyright Header checks" +} diff --git a/.github/scripts/setup_env.bash b/.github/scripts/setup_env.bash index 9ce706d9fc..4f6af37dac 100755 --- a/.github/scripts/setup_env.bash +++ b/.github/scripts/setup_env.bash @@ -6,153 +6,12 @@ # LICENSE file in the root directory of this source tree. -################################################################################ -# Command Execution Functions -################################################################################ - -print_exec () { - echo "+ $*" - echo "" - if "$@"; then - local retcode=0 - else - local retcode=$? - fi - echo "" - return $retcode -} - -exec_with_retries () { - local max=5 - local delay=2 - local retcode=0 - - for i in $(seq 1 ${max}); do - echo "[EXEC] [ATTEMPT ${i}/${max}] + $*" - - if "$@"; then - retcode=0 - break - else - retcode=$? - echo "[EXEC] [ATTEMPT ${i}/${max}] Command attempt failed." - echo "" - sleep $delay - fi - done - - if [ $retcode -ne 0 ]; then - echo "[EXEC] The command has failed after ${max} attempts; aborting." - fi - - return $retcode -} - - -################################################################################ -# Assert Functions -################################################################################ - -test_python_import () { - local env_name="$1" - local python_import="$2" - if [ "$python_import" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME PYTHON_IMPORT" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env numpy" - return 1 - fi - - if conda run -n "${env_name}" python -c "import ${python_import}"; then - echo "[CHECK] Python package ${python_import} found." - else - echo "[CHECK] Python package ${python_import} not found!" - return 1 - fi -} - -test_binpath () { - local env_name="$1" - local bin_name="$2" - if [ "$bin_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME BIN_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env nvcc" - return 1 - fi - - if conda run -n "${env_name}" which "${bin_name}"; then - echo "[CHECK] Binary ${bin_name} found in PATH" - else - echo "[CHECK] Binary ${bin_name} not found in PATH!" - return 1 - fi -} - -test_filepath () { - local env_name="$1" - local file_name="$2" - if [ "$file_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME FILE_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env cuda_runtime.h" - return 1 - fi - - # shellcheck disable=SC2155 - local conda_prefix=$(conda run -n "${env_name}" printenv CONDA_PREFIX) - # shellcheck disable=SC2155 - local file_path=$(find "${conda_prefix}" -type f -name "${file_name}") - # shellcheck disable=SC2155 - local link_path=$(find "${conda_prefix}" -type l -name "${file_name}") - if [ "${file_path}" != "" ]; then - echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (file): ${file_path}" - elif [ "${link_path}" != "" ]; then - echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (symbolic link): ${link_path}" - else - echo "[CHECK] ${file_name} not found in CONDA_PREFIX PATH!" - return 1 - fi -} - -test_env_var () { - local env_name="$1" - local env_key="$2" - if [ "$env_key" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME ENV_KEY" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env CUDNN_INCLUDE_DIR" - return 1 - fi - - if conda run -n "${env_name}" printenv "${env_key}"; then - echo "[CHECK] Environment variable ${env_key} is defined in the Conda environment" - else - echo "[CHECK] Environment variable ${env_key} is not defined in the Conda environment!" - return 1 - fi -} - -test_library_symbol () { - local lib_path="$1" - local lib_symbol="$2" - if [ "$lib_symbol" == "" ]; then - echo "Usage: ${FUNCNAME[0]} LIB_PATH FULL_NAMESPACE_PATH_LIB_SYMBOL" - echo "Example(s):" - echo " ${FUNCNAME[0]} fbgemm_gpu_py.so fbgemm_gpu::merge_pooled_embeddings" - return 1 - fi - - # Add space and '(' to the grep string to get the full method path - symbol_entries=$(nm -gDC "${lib_path}" | grep " ${lib_symbol}(") - if [ "${symbol_entries}" != "" ]; then - echo "[CHECK] Found symbol in ${lib_path}: ${lib_symbol}" - else - echo "[CHECK] Symbol NOT found in ${lib_path}: ${lib_symbol}" - return 1 - fi -} - +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/base_utilities.bash" +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/fbgemm_gpu_docs.bash" +# shellcheck disable=SC1091,SC2128 +. "$( dirname -- "$BASH_SOURCE"; )/fbgemm_gpu_lint.bash" ################################################################################ # System Functions @@ -1018,66 +877,6 @@ install_build_tools () { echo "[INSTALL] Successfully installed all the build tools" } -install_lint_tools () { - local env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env" - return 1 - else - echo "################################################################################" - echo "# Install Lint Tools" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - echo "[INSTALL] Installing lint tools ..." - (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \ - click \ - flake8 \ - ufmt) || return 1 - - # Check binaries are visible in the PAATH - (test_binpath "${env_name}" flake8) || return 1 - (test_binpath "${env_name}" ufmt) || return 1 - - # Check Python packages are importable - local import_tests=( click ) - for p in "${import_tests[@]}"; do - (test_python_import "${env_name}" "${p}") || return 1 - done - - echo "[INSTALL] Successfully installed all the lint tools" -} - -install_docs_tools () { - local env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env" - return 1 - else - echo "################################################################################" - echo "# Install Documentation Tools" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - echo "[INSTALL] Installing docs tools ..." - (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \ - doxygen) || return 1 - - # Check binaries are visible in the PAATH - (test_binpath "${env_name}" doxygen) || return 1 - - echo "[INSTALL] Successfully installed all the docs tools" -} ################################################################################ # Combination Functions @@ -1473,34 +1272,6 @@ build_fbgemm_gpu_develop () { echo "[BUILD] FBGEMM-GPU build + develop completed" } -build_fbgemm_gpu_docs () { - env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env # Build the docs" - return 1 - else - echo "################################################################################" - echo "# Build FBGEMM-GPU Documentation" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - echo "[BUILD] Installing docs-build dependencies ..." - (exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1 - - echo "[BUILD] Running Doxygen build ..." - (exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1 - - echo "[BUILD] Building HTML pages ..." - (exec_with_retries conda run -n "${env_name}" make html) || return 1 - - echo "[INSTALL] FBGEMM-GPU documentation build completed" -} - install_fbgemm_gpu_package () { local env_name="$1" local package_name="$2" @@ -1608,89 +1379,6 @@ run_fbgemm_gpu_tests () { } -################################################################################ -# FBGEMM_GPU Lint Functions -################################################################################ - -lint_fbgemm_gpu_flake8 () { - local env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env" - return 1 - else - echo "################################################################################" - echo "# Run FBGEMM_GPU Lint: flake8" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json" - - # E501 = line too long - # W503 = line break before binary operator (deprecated) - # E203 = whitespace before ":" - (print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1 -} - -lint_fbgemm_gpu_ufmt () { - local env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env" - return 1 - else - echo "################################################################################" - echo "# Run FBGEMM_GPU Lint: ufmt" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - local lint_paths=( - fbgemm_gpu/fbgemm_gpu - fbgemm_gpu/test - fbgemm_gpu/bench - ) - - for p in "${lint_paths[@]}"; do - (print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1 - done -} - -lint_fbgemm_gpu_copyright () { - local env_name="$1" - if [ "$env_name" == "" ]; then - echo "Usage: ${FUNCNAME[0]} ENV_NAME" - echo "Example(s):" - echo " ${FUNCNAME[0]} build_env" - return 1 - else - echo "################################################################################" - echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers" - echo "#" - echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)" - echo "################################################################################" - echo "" - fi - - local lint_paths=( - fbgemm_gpu/fbgemm_gpu - fbgemm_gpu/test - fbgemm_gpu/bench - ) - - for p in "${lint_paths[@]}"; do - (print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1 - done -} - - ################################################################################ # FBGEMM_GPU Publish Functions ################################################################################ diff --git a/.github/workflows/fbgemm_gpu_lint.yml b/.github/workflows/fbgemm_gpu_lint.yml index 5bcbc976b7..41c8a08967 100644 --- a/.github/workflows/fbgemm_gpu_lint.yml +++ b/.github/workflows/fbgemm_gpu_lint.yml @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.10" ] + python-version: [ "3.11" ] steps: - name: Checkout the Repository