Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update cuda config for CUDA11.1 #2218

Merged
merged 2 commits into from
Nov 22, 2020
Merged

Update cuda config for CUDA11.1 #2218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9aa16fb
Update cuda config for CUDA11.1
perfinion Oct 26, 2020
ec138ca
* Fix flake8 and formatting errors
seanpmorgan Nov 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion build_deps/toolchains/gpu/cuda_configure.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,7 @@ def _find_libs(repository_ctx, cuda_config):
repository_ctx,
cpu_value,
cuda_config.config["cuda_library_dir"],
cuda_config.cuda_version,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is cuda_version removed?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The runtime version is what's needed for so naming.

cuda_config.cudart_version,
),
"cudart_static": _find_cuda_lib(
"cudart_static",
Expand Down Expand Up @@ -622,6 +622,7 @@ def _get_cuda_config(repository_ctx):
cuda_toolkit_path: The CUDA toolkit installation directory.
cudnn_install_basedir: The cuDNN installation directory.
cuda_version: The version of CUDA on the system.
cudart_version: The CUDA runtime version on the system.
cudnn_version: The version of cuDNN on the system.
compute_capabilities: A list of the system's CUDA compute capabilities.
cpu_value: The name of the host operating system.
Expand All @@ -639,6 +640,11 @@ def _get_cuda_config(repository_ctx):
cudnn_version = ("64_%s" if is_windows else "%s") % config["cudnn_version"]

if int(cuda_major) >= 11:
# The libcudart soname in CUDA 11.x is versioned as 11.0 for backward compatability.
if int(cuda_major) == 11:
cudart_version = "64_110" if is_windows else "11.0"
else:
cudart_version = ("64_%s" if is_windows else "%s") % cuda_major
cublas_version = ("64_%s" if is_windows else "%s") % config["cublas_version"].split(".")[0]
cusolver_version = ("64_%s" if is_windows else "%s") % config["cusolver_version"].split(".")[0]
curand_version = ("64_%s" if is_windows else "%s") % config["curand_version"].split(".")[0]
Expand All @@ -651,15 +657,18 @@ def _get_cuda_config(repository_ctx):
cusolver_version = cuda_lib_version
curand_version = cuda_lib_version
cufft_version = cuda_lib_version
cudart_version = cuda_version
else:
cublas_version = cuda_version
cusolver_version = cuda_version
curand_version = cuda_version
cufft_version = cuda_version
cudart_version = cuda_version

return struct(
cuda_toolkit_path = toolkit_path,
cuda_version = cuda_version,
cudart_version = cudart_version,
cublas_version = cublas_version,
cusolver_version = cusolver_version,
curand_version = curand_version,
Expand Down
116 changes: 85 additions & 31 deletions build_deps/toolchains/gpu/find_cuda_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@

The script searches for CUDA library and header files on the system, inspects
them to determine their version and prints the configuration to stdout.
The paths to inspect and the required versions are specified through
environment variables. If no valid configuration is found, the script prints
to stderr and returns an error code.
The paths to inspect and the required versions are specified through environment
variables. If no valid configuration is found, the script prints to stderr and
returns an error code.

The list of libraries to find is specified as arguments. Supported libraries
are CUDA (includes cuBLAS), cuDNN, NCCL, and TensorRT.
The list of libraries to find is specified as arguments. Supported libraries are
CUDA (includes cuBLAS), cuDNN, NCCL, and TensorRT.

The script takes a list of base directories specified by the TF_CUDA_PATHS
environment variable as comma-separated glob list. The script looks for headers
Expand Down Expand Up @@ -53,17 +53,20 @@
tf_<library>_library_dir: ...
"""

import io
import os
import glob
import platform
import re
import subprocess
import sys

# pylint: disable=g-import-not-at-top
try:
from shutil import which
except ImportError:
from distutils.spawn import find_executable as which
# pylint: enable=g-import-not-at-top


class ConfigError(Exception):
Expand Down Expand Up @@ -117,7 +120,7 @@ def _at_least_version(actual_version, required_version):

def _get_header_version(path, name):
"""Returns preprocessor defines in C header file."""
for line in open(path, "r", encoding="utf-8").readlines():
for line in io.open(path, "r", encoding="utf-8").readlines():
match = re.match(r"#define %s +(\d+)" % name, line)
if match:
return match.group(1)
Expand Down Expand Up @@ -178,6 +181,7 @@ def _header_paths():
"include/*-linux-gnu",
"extras/CUPTI/include",
"include/cuda/CUPTI",
"local/cuda/extras/CUPTI/include",
]


Expand All @@ -190,6 +194,8 @@ def _library_paths():
"lib/*-linux-gnu",
"lib/x64",
"extras/CUPTI/*",
"local/cuda/lib64",
"local/cuda/extras/CUPTI/lib64",
]


Expand All @@ -204,8 +210,8 @@ def _not_found_error(base_paths, relative_paths, filepattern):

def _find_file(base_paths, relative_paths, filepattern):
for path in _cartesian_product(base_paths, relative_paths):
for file_path in glob.glob(os.path.join(path, filepattern)):
return file_path
for file in glob.glob(os.path.join(path, filepattern)):
return file
raise _not_found_error(base_paths, relative_paths, filepattern)


Expand All @@ -228,16 +234,15 @@ def _find_library(base_paths, library_name, required_version):
def _find_versioned_file(
base_paths, relative_paths, filepatterns, required_version, get_version
):
"""Returns first valid path to a file that matches the requested
version."""
"""Returns first valid path to a file that matches the requested version."""
if type(filepatterns) not in [list, tuple]:
filepatterns = [filepatterns]
for path in _cartesian_product(base_paths, relative_paths):
for pattern in filepatterns:
for file_path in glob.glob(os.path.join(path, pattern)):
actual_version = get_version(file_path)
for filepattern in filepatterns:
for file in glob.glob(os.path.join(path, filepattern)):
actual_version = get_version(file)
if _matches_version(actual_version, required_version):
return file_path, actual_version
return file, actual_version
raise _not_found_error(
base_paths,
relative_paths,
Expand All @@ -246,8 +251,7 @@ def _find_versioned_file(


def _find_header(base_paths, header_name, required_version, get_version):
"""Returns first valid path to a header that matches the requested
version."""
"""Returns first valid path to a header that matches the requested version."""
return _find_versioned_file(
base_paths, _header_paths(), header_name, required_version, get_version
)
Expand Down Expand Up @@ -278,15 +282,24 @@ def get_nvcc_version(path):
nvcc_name = "nvcc.exe" if _is_windows() else "nvcc"
nvcc_path, nvcc_version = _find_versioned_file(
base_paths,
["", "bin"],
[
"",
"bin",
"local/cuda/bin",
],
nvcc_name,
cuda_version,
get_nvcc_version,
)

nvvm_path = _find_file(
base_paths,
["nvvm/libdevice", "share/cuda", "lib/nvidia-cuda-toolkit/libdevice"],
[
"nvvm/libdevice",
"share/cuda",
"lib/nvidia-cuda-toolkit/libdevice",
"local/cuda/nvvm/libdevice",
],
"libdevice*.10.bc",
)

Expand Down Expand Up @@ -330,23 +343,23 @@ def get_header_version(path):
)
return ".".join(version)

header_path, cublas_version = _find_header(
header_path, header_version = _find_header(
base_paths, "cublas_api.h", required_version, get_header_version
)

cublas_major_version = cublas_version.split(".")[0]
# cuBLAS uses the major version only.
cublas_version = header_version.split(".")[0]

else:
# There is no version info available before CUDA 10.1, just find the file.
cublas_version = cuda_version
header_version = cuda_version
header_path = _find_file(base_paths, _header_paths(), "cublas_api.h")
# cuBLAS version is the same as CUDA version (x.y).
cublas_major_version = required_version
cublas_version = required_version

library_path = _find_library(base_paths, "cublas", cublas_major_version)
library_path = _find_library(base_paths, "cublas", cublas_version)

return {
"cublas_version": cublas_version,
"cublas_version": header_version,
"cublas_include_dir": os.path.dirname(header_path),
"cublas_library_dir": os.path.dirname(library_path),
}
Expand Down Expand Up @@ -468,6 +481,40 @@ def get_header_version(path):
}


def _find_cusparse_config(base_paths, required_version, cuda_version):

if _at_least_version(cuda_version, "11.0"):

def get_header_version(path):
version = (
_get_header_version(path, name)
for name in (
"CUSPARSE_VER_MAJOR",
"CUSPARSE_VER_MINOR",
"CUSPARSE_VER_PATCH",
)
)
return ".".join(version)

header_path, header_version = _find_header(
base_paths, "cusparse.h", required_version, get_header_version
)
cusparse_version = header_version.split(".")[0]

else:
header_version = cuda_version
header_path = _find_file(base_paths, _header_paths(), "cusparse.h")
cusparse_version = required_version

library_path = _find_library(base_paths, "cusparse", cusparse_version)

return {
"cusparse_version": header_version,
"cusparse_include_dir": os.path.dirname(header_path),
"cusparse_library_dir": os.path.dirname(library_path),
}


def _find_nccl_config(base_paths, required_version):
def get_header_version(path):
version = (
Expand Down Expand Up @@ -534,9 +581,8 @@ def _get_legacy_path(env_name, default=[]):
"""Returns a path specified by a legacy environment variable.

CUDNN_INSTALL_PATH, NCCL_INSTALL_PATH, TENSORRT_INSTALL_PATH set to
'/usr/lib/x86_64-linux-gnu' would previously find both library and
header paths. Detect those and return '/usr', otherwise forward to
_list_from_env().
'/usr/lib/x86_64-linux-gnu' would previously find both library and header
paths. Detect those and return '/usr', otherwise forward to _list_from_env().
"""
if env_name in os.environ:
match = re.match(r"^(/[^/ ]*)+/lib/\w+-linux-gnu/?$", os.environ[env_name])
Expand All @@ -547,7 +593,7 @@ def _get_legacy_path(env_name, default=[]):

def _normalize_path(path):
"""Returns normalized path, with forward slashes on Windows."""
path = os.path.normpath(path)
path = os.path.realpath(path)
if _is_windows():
path = path.replace("\\", "/")
return path
Expand All @@ -558,8 +604,8 @@ def find_cuda_config():
libraries = [argv.lower() for argv in sys.argv[1:]]
cuda_version = os.environ.get("TF_CUDA_VERSION", "")
base_paths = _list_from_env("TF_CUDA_PATHS", _get_default_cuda_paths(cuda_version))

base_paths = [path for path in base_paths if os.path.exists(path)]

result = {}
if "cuda" in libraries:
cuda_paths = _list_from_env("CUDA_TOOLKIT_PATH", base_paths)
Expand Down Expand Up @@ -593,6 +639,14 @@ def find_cuda_config():
cufft_version = os.environ.get("TF_CUFFT_VERSION", "")
result.update(_find_cufft_config(cufft_paths, cufft_version, cuda_version))

cusparse_paths = base_paths
if tuple(int(v) for v in cuda_version.split(".")) < (11, 0):
cusparse_paths = cuda_paths
cusparse_version = os.environ.get("TF_CUSPARSE_VERSION", "")
result.update(
_find_cusparse_config(cusparse_paths, cusparse_version, cuda_version)
)

if "cudnn" in libraries:
cudnn_paths = _get_legacy_path("CUDNN_INSTALL_PATH", base_paths)
cudnn_version = os.environ.get("TF_CUDNN_VERSION", "")
Expand All @@ -618,7 +672,7 @@ def find_cuda_config():
def main():
try:
for key, value in sorted(find_cuda_config().items()):
print("{}: {}".format(key, value))
print("%s: %s" % (key, value))
except ConfigError as e:
sys.stderr.write(str(e))
sys.exit(1)
Expand Down