[T149624848] Break up setup_env.bash

- Break up setup_env.bash into separate files for easier maintainability

.github/scripts/base_utilities.bash

@@ -0,0 +1,154 @@
+#!/bin/bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+################################################################################
+# Command Execution Functions
+################################################################################
+
+print_exec () {
+  echo "+ $*"
+  echo ""
+  if "$@"; then
+    local retcode=0
+  else
+    local retcode=$?
+  fi
+  echo ""
+  return $retcode
+}
+
+exec_with_retries () {
+  local max=5
+  local delay=2
+  local retcode=0
+
+  for i in $(seq 1 ${max}); do
+    echo "[EXEC] [ATTEMPT ${i}/${max}]    + $*"
+
+    if "$@"; then
+      retcode=0
+      break
+    else
+      retcode=$?
+      echo "[EXEC] [ATTEMPT ${i}/${max}] Command attempt failed."
+      echo ""
+      sleep $delay
+    fi
+  done
+
+  if [ $retcode -ne 0 ]; then
+    echo "[EXEC] The command has failed after ${max} attempts; aborting."
+  fi
+
+  return $retcode
+}
+
+
+################################################################################
+# Assert Functions
+################################################################################
+
+test_python_import () {
+  local env_name="$1"
+  local python_import="$2"
+  if [ "$python_import" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME PYTHON_IMPORT"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env numpy"
+    return 1
+  fi
+
+  if conda run -n "${env_name}" python -c "import ${python_import}"; then
+    echo "[CHECK] Python package ${python_import} found."
+  else
+    echo "[CHECK] Python package ${python_import} not found!"
+    return 1
+  fi
+}
+
+test_binpath () {
+  local env_name="$1"
+  local bin_name="$2"
+  if [ "$bin_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME BIN_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env nvcc"
+    return 1
+  fi
+
+  if conda run -n "${env_name}" which "${bin_name}"; then
+    echo "[CHECK] Binary ${bin_name} found in PATH"
+  else
+    echo "[CHECK] Binary ${bin_name} not found in PATH!"
+    return 1
+  fi
+}
+
+test_filepath () {
+  local env_name="$1"
+  local file_name="$2"
+  if [ "$file_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME FILE_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env cuda_runtime.h"
+    return 1
+  fi
+
+  # shellcheck disable=SC2155
+  local conda_prefix=$(conda run -n "${env_name}" printenv CONDA_PREFIX)
+  # shellcheck disable=SC2155
+  local file_path=$(find "${conda_prefix}" -type f -name "${file_name}")
+  # shellcheck disable=SC2155
+  local link_path=$(find "${conda_prefix}" -type l -name "${file_name}")
+  if [ "${file_path}" != "" ]; then
+    echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (file): ${file_path}"
+  elif [ "${link_path}" != "" ]; then
+    echo "[CHECK] ${file_name} found in CONDA_PREFIX PATH (symbolic link): ${link_path}"
+  else
+    echo "[CHECK] ${file_name} not found in CONDA_PREFIX PATH!"
+    return 1
+  fi
+}
+
+test_env_var () {
+  local env_name="$1"
+  local env_key="$2"
+  if [ "$env_key" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME ENV_KEY"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env CUDNN_INCLUDE_DIR"
+    return 1
+  fi
+
+  if conda run -n "${env_name}" printenv "${env_key}"; then
+    echo "[CHECK] Environment variable ${env_key} is defined in the Conda environment"
+  else
+    echo "[CHECK] Environment variable ${env_key} is not defined in the Conda environment!"
+    return 1
+  fi
+}
+
+test_library_symbol () {
+  local lib_path="$1"
+  local lib_symbol="$2"
+  if [ "$lib_symbol" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} LIB_PATH FULL_NAMESPACE_PATH_LIB_SYMBOL"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} fbgemm_gpu_py.so fbgemm_gpu::merge_pooled_embeddings"
+    return 1
+  fi
+
+  # Add space and '(' to the grep string to get the full method path
+  symbol_entries=$(nm -gDC "${lib_path}" | grep " ${lib_symbol}(")
+  if [ "${symbol_entries}" != "" ]; then
+    echo "[CHECK] Found symbol in ${lib_path}: ${lib_symbol}"
+  else
+    echo "[CHECK] Symbol NOT found in ${lib_path}: ${lib_symbol}"
+    return 1
+  fi
+}

.github/scripts/fbgemm_gpu_docs.bash

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# shellcheck disable=SC1091,SC2128
+. "$( dirname -- "$BASH_SOURCE"; )/base_utilities.bash"
+
+################################################################################
+# Docs Tools Setup Functions
+################################################################################
+
+install_docs_tools () {
+  local env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Install Documentation Tools"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  echo "[INSTALL] Installing docs tools ..."
+  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
+    doxygen) || return 1
+
+  # Check binaries are visible in the PAATH
+  (test_binpath "${env_name}" doxygen) || return 1
+
+  echo "[INSTALL] Successfully installed all the docs tools"
+}
+
+
+################################################################################
+# FBGEMM_GPU Docs Functions
+################################################################################
+
+build_fbgemm_gpu_docs () {
+  env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env      # Build the docs"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Build FBGEMM-GPU Documentation"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  echo "[BUILD] Installing docs-build dependencies ..."
+  (exec_with_retries conda run -n "${env_name}" python -m pip install -r requirements.txt) || return 1
+
+  echo "[BUILD] Running Doxygen build ..."
+  (exec_with_retries conda run -n "${env_name}" doxygen Doxyfile.in) || return 1
+
+  echo "[BUILD] Building HTML pages ..."
+  (exec_with_retries conda run -n "${env_name}" make html) || return 1
+
+  echo "[INSTALL] FBGEMM-GPU documentation build completed"
+}

.github/scripts/fbgemm_gpu_lint.bash

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+# shellcheck disable=SC1091,SC2128
+. "$( dirname -- "$BASH_SOURCE"; )/base_utilities.bash"
+
+################################################################################
+# Lint Tools Setup Functions
+################################################################################
+
+install_lint_tools () {
+  local env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Install Lint Tools"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  echo "[INSTALL] Installing lint tools ..."
+  (exec_with_retries conda install -n "${env_name}" -c conda-forge -y \
+    click \
+    flake8 \
+    ufmt) || return 1
+
+  # Check binaries are visible in the PAATH
+  (test_binpath "${env_name}" flake8) || return 1
+  (test_binpath "${env_name}" ufmt) || return 1
+
+  # Check Python packages are importable
+  local import_tests=( click )
+  for p in "${import_tests[@]}"; do
+    (test_python_import "${env_name}" "${p}") || return 1
+  done
+
+  echo "[INSTALL] Successfully installed all the lint tools"
+}
+
+################################################################################
+# FBGEMM_GPU Lint Functions
+################################################################################
+
+lint_fbgemm_gpu_flake8 () {
+  local env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Run FBGEMM_GPU Lint: flake8"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  echo "::add-matcher::fbgemm_gpu/test/lint/flake8_problem_matcher.json"
+
+  # E501 = line too long
+  # W503 = line break before binary operator (deprecated)
+  # E203 = whitespace before ":"
+  (print_exec conda run -n "${env_name}" flake8 --ignore=E501,W503,E203 .) || return 1
+
+  echo "[TEST] Finished running flake8 lint checks"
+}
+
+lint_fbgemm_gpu_ufmt () {
+  local env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Run FBGEMM_GPU Lint: ufmt"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  local lint_paths=(
+    fbgemm_gpu/fbgemm_gpu
+    fbgemm_gpu/test
+    fbgemm_gpu/bench
+  )
+
+  for p in "${lint_paths[@]}"; do
+    (print_exec conda run -n "${env_name}" ufmt diff "${p}") || return 1
+  done
+
+  echo "[TEST] Finished running ufmt lint checks"
+}
+
+lint_fbgemm_gpu_copyright () {
+  local env_name="$1"
+  if [ "$env_name" == "" ]; then
+    echo "Usage: ${FUNCNAME[0]} ENV_NAME"
+    echo "Example(s):"
+    echo "    ${FUNCNAME[0]} build_env"
+    return 1
+  else
+    echo "################################################################################"
+    echo "# Run FBGEMM_GPU Lint: Meta Copyright Headers"
+    echo "#"
+    echo "# [TIMESTAMP] $(date --utc +%FT%T.%3NZ)"
+    echo "################################################################################"
+    echo ""
+  fi
+
+  local lint_paths=(
+    fbgemm_gpu/fbgemm_gpu
+    fbgemm_gpu/test
+    fbgemm_gpu/bench
+  )
+
+  for p in "${lint_paths[@]}"; do
+    (print_exec conda run -n "${env_name}" python fbgemm_gpu/test/lint/check_meta_header.py --path="${p}" --fixit=False) || return 1
+  done
+
+  echo "[TEST] Finished running Meta Copyright Header checks"
+}