From dc8f9f6ac08e4adeed13a2532dc2865b75f4e131 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 10 Nov 2023 16:02:55 -0500 Subject: [PATCH 01/24] add horovod --- recipes/horovod/conda_build_config.yaml | 3 + recipes/horovod/meta.yaml | 91 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 recipes/horovod/conda_build_config.yaml create mode 100644 recipes/horovod/meta.yaml diff --git a/recipes/horovod/conda_build_config.yaml b/recipes/horovod/conda_build_config.yaml new file mode 100644 index 0000000000000..22f9faaa334bf --- /dev/null +++ b/recipes/horovod/conda_build_config.yaml @@ -0,0 +1,3 @@ +mpi: + - mpich + - openmpi \ No newline at end of file diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml new file mode 100644 index 0000000000000..23d74b91eef9a --- /dev/null +++ b/recipes/horovod/meta.yaml @@ -0,0 +1,91 @@ +{% set name = "horovod" %} +{% set version = "0.28.1" %} +{% set proc_type = "cuda" if cuda_compiler_version != "None" else "cpu" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + sha256: 92a43f5a94c43907a56805bad15f19700c62ffc83b7ca483f9e104e229f67ef0 + +build: + number: 0 + script_env: + - HOROVOD_GPU_OPERATIONS=NCCL # [cuda_compiler_version != 'None'] + - HOROVOD_NCCL_LINK=SHARED # [cuda_compiler_version != 'None'] + - HOROVOD_WITH_TENSORFLOW=1 + - HOROVOD_WITH_PYTORCH=1 + # mxnet is not available on conda-forge + # https://github.com/conda-forge/staged-recipes/issues/4447 + - HOROVOD_WITHOUT_MXNET=1 + - HOROVOD_WITH_MPI=1 + - HOROVOD_WITH_GLOO=1 + script: + - "{{ PYTHON }} -m pip install . -vv" # [unix] + skip: true # [win] + string: {{ proc_type }}_mpi_{{ mpi }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} + +requirements: + build: + - {{ compiler('cxx') }} + - {{ compiler('cuda') }} # [cuda_compiler_version != 'None'] + - cmake >=3.13 + - make # [unix] + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + + host: + - python + - pip + - wheel + - setuptools + - nccl # [cuda_compiler_version != 'None'] + - {{ mpi }} + - mpi4py + - tensorflow + - tensorflow * {{ proc_type }}* + - pytorch + - pytorch * {{ proc_type }}* + + run: + - python + - pyyaml + - cloudpickle + - pyyaml + - packaging + - psutil + - nccl # [cuda_compiler_version != 'None'] + - {{ mpi }} + - mpi4py + + run_constrained: + - {{ pin_compatible('tensorflow', max_pin='x.x')) }} {{ proc_type }}* + - {{ pin_compatible('pytorch', max_pin='x.x')) }} {{ proc_type }}* + +test: + requires: + - pip + - tensorflow + - pytorch + imports: + - horovod + - horovod.tensorflow + - horovod.torch + commands: + - pip check + - horovodrun -h + +about: + home: https://horovod.ai/ + license: Apache-2.0 + license_family: Apache + license_file: LICENSE + summary: 'Distributed training framework for TensorFlow, Keras, PyTorch, and Apache MXNet.' + doc_url: https://horovod.readthedocs.io/ + dev_url: https://github.com/horovod/horovod + +extra: + recipe-maintainers: + - njzjz From c40c9290a9a2ac25233f88f8337e088c5d5878f0 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 10 Nov 2023 16:05:46 -0500 Subject: [PATCH 02/24] fix typo --- recipes/horovod/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 23d74b91eef9a..79020c6acebd6 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -61,8 +61,8 @@ requirements: - mpi4py run_constrained: - - {{ pin_compatible('tensorflow', max_pin='x.x')) }} {{ proc_type }}* - - {{ pin_compatible('pytorch', max_pin='x.x')) }} {{ proc_type }}* + - {{ pin_compatible('tensorflow', max_pin='x.x') }} {{ proc_type }}* + - {{ pin_compatible('pytorch', max_pin='x.x') }} {{ proc_type }}* test: requires: From 076439a36473630b58b70a318b2fd3638a897271 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 10 Nov 2023 16:15:39 -0500 Subject: [PATCH 03/24] add pyyaml to host --- recipes/horovod/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 79020c6acebd6..c78d641779500 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -41,6 +41,7 @@ requirements: - pip - wheel - setuptools + - pyyaml - nccl # [cuda_compiler_version != 'None'] - {{ mpi }} - mpi4py From ecbcb1f4b7878bf27a9e56ac6e43723d00bd5f10 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 10 Nov 2023 16:27:26 -0500 Subject: [PATCH 04/24] it seems that run requires should be added to host --- recipes/horovod/meta.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index c78d641779500..e96475bc5c98f 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -42,6 +42,9 @@ requirements: - wheel - setuptools - pyyaml + - cloudpickle + - packaging + - psutil - nccl # [cuda_compiler_version != 'None'] - {{ mpi }} - mpi4py @@ -54,7 +57,6 @@ requirements: - python - pyyaml - cloudpickle - - pyyaml - packaging - psutil - nccl # [cuda_compiler_version != 'None'] From 29a9aed70a99677ad7f8cabec3d3d3f9d40bf41e Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 16:47:07 -0500 Subject: [PATCH 05/24] run flatc before pip --- recipes/horovod/meta.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index e96475bc5c98f..f9c1717f3940f 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -23,6 +23,8 @@ build: - HOROVOD_WITH_MPI=1 - HOROVOD_WITH_GLOO=1 script: + # See https://github.com/horovod/horovod/issues/3956 + - "flatc -c -o horovod/common/wire horovod/common/wire/message.fbs" - "{{ PYTHON }} -m pip install . -vv" # [unix] skip: true # [win] string: {{ proc_type }}_mpi_{{ mpi }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} @@ -52,6 +54,8 @@ requirements: - tensorflow * {{ proc_type }}* - pytorch - pytorch * {{ proc_type }}* + # for flatc + - flatbuffers run: - python From 78901c1c81c12831326a3cd86dc33637399e818d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 17:03:26 -0500 Subject: [PATCH 06/24] download source from github repository --- recipes/horovod/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index f9c1717f3940f..ec78c08c89d88 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -7,8 +7,8 @@ package: version: {{ version }} source: - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz - sha256: 92a43f5a94c43907a56805bad15f19700c62ffc83b7ca483f9e104e229f67ef0 + url: https://github.com/horovod/horovod/archive/refs/tags/{{ version }}.tar.gz + sha256: 88fbcd2815083607a5d27962f323eae5752e3eb96511aaac98258f280cac0f8e build: number: 0 From 50d6482edd7c5cb3ff1e1c1e2b867f1385d019e5 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 17:16:26 -0500 Subject: [PATCH 07/24] fix url --- recipes/horovod/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index ec78c08c89d88..63a12a483a709 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -7,7 +7,7 @@ package: version: {{ version }} source: - url: https://github.com/horovod/horovod/archive/refs/tags/{{ version }}.tar.gz + url: https://github.com/horovod/horovod/archive/refs/tags/v{{ version }}.tar.gz sha256: 88fbcd2815083607a5d27962f323eae5752e3eb96511aaac98258f280cac0f8e build: From 389f6e4ebaffcbdafc96cace45a8b8deac5b67ed Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 17:30:34 -0500 Subject: [PATCH 08/24] disable gloo --- recipes/horovod/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 63a12a483a709..ce060d507e671 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -21,7 +21,7 @@ build: # https://github.com/conda-forge/staged-recipes/issues/4447 - HOROVOD_WITHOUT_MXNET=1 - HOROVOD_WITH_MPI=1 - - HOROVOD_WITH_GLOO=1 + - HOROVOD_WITHOUT_GLOO=1 script: # See https://github.com/horovod/horovod/issues/3956 - "flatc -c -o horovod/common/wire horovod/common/wire/message.fbs" From 7795a4959d76934c9db622113018d6767c9bb88f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 17:50:12 -0500 Subject: [PATCH 09/24] add boost headers --- recipes/horovod/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index ce060d507e671..7f93c3cd9e58c 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -56,6 +56,7 @@ requirements: - pytorch * {{ proc_type }}* # for flatc - flatbuffers + - libboost-headers run: - python From 22f7e00af39fa25d029ed763a9ebebcaf74307d1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 18:06:50 -0500 Subject: [PATCH 10/24] add lbfgspp --- recipes/horovod/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 7f93c3cd9e58c..9633611147c71 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -57,6 +57,7 @@ requirements: # for flatc - flatbuffers - libboost-headers + - lbfgspp run: - python From c674926d61bcdba57e3474c19a8ddd665b116024 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 18:28:59 -0500 Subject: [PATCH 11/24] it seems no way to make tf or pt optional --- recipes/horovod/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 9633611147c71..d203a18833955 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -68,8 +68,6 @@ requirements: - nccl # [cuda_compiler_version != 'None'] - {{ mpi }} - mpi4py - - run_constrained: - {{ pin_compatible('tensorflow', max_pin='x.x') }} {{ proc_type }}* - {{ pin_compatible('pytorch', max_pin='x.x') }} {{ proc_type }}* From 7bae1dd4fdc9ad572fcc12573ab0126ef5e76521 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 18:56:09 -0500 Subject: [PATCH 12/24] revert c674926d61bcdba57e3474c19a8ddd665b116024; add tf to missing_dso_white_list; add pt to ignore_run_exports_from --- recipes/horovod/meta.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index d203a18833955..1f427a0471dc7 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -68,9 +68,20 @@ requirements: - nccl # [cuda_compiler_version != 'None'] - {{ mpi }} - mpi4py + + run_constrained: - {{ pin_compatible('tensorflow', max_pin='x.x') }} {{ proc_type }}* - {{ pin_compatible('pytorch', max_pin='x.x') }} {{ proc_type }}* + ignore_run_exports_from: + - flatbuffers + - tensorflow + - pytorch + + missing_dso_whitelist: + - "*/libtensorflow_framework.so.*" + - "*/libtensorflow_cc.so.*" + test: requires: - pip From 75712246e3d60906c8eec3d2713901e49b468e9a Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 18:57:36 -0500 Subject: [PATCH 13/24] put in the correct section --- recipes/horovod/meta.yaml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 1f427a0471dc7..c848c6183d3a8 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -28,6 +28,13 @@ build: - "{{ PYTHON }} -m pip install . -vv" # [unix] skip: true # [win] string: {{ proc_type }}_mpi_{{ mpi }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} + ignore_run_exports_from: + - flatbuffers + - tensorflow + - pytorch + missing_dso_whitelist: + - "*/libtensorflow_framework.so.*" + - "*/libtensorflow_cc.so.*" requirements: build: @@ -73,15 +80,6 @@ requirements: - {{ pin_compatible('tensorflow', max_pin='x.x') }} {{ proc_type }}* - {{ pin_compatible('pytorch', max_pin='x.x') }} {{ proc_type }}* - ignore_run_exports_from: - - flatbuffers - - tensorflow - - pytorch - - missing_dso_whitelist: - - "*/libtensorflow_framework.so.*" - - "*/libtensorflow_cc.so.*" - test: requires: - pip From 5b61c3cfac53b03a6a2d7ce794af70a0918fe1fd Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 19:15:49 -0500 Subject: [PATCH 14/24] add torch libraries to missing_dso_whitelist --- recipes/horovod/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index c848c6183d3a8..64825a1c4ccd3 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -35,6 +35,7 @@ build: missing_dso_whitelist: - "*/libtensorflow_framework.so.*" - "*/libtensorflow_cc.so.*" + - "*/torch/lib/*.so" requirements: build: From 1b537fa815fafa709fb9f69dc6df05a16b952393 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 21:22:05 -0500 Subject: [PATCH 15/24] move script to build.sh; fix build issues --- recipes/horovod/build.sh | 25 +++++++++++++++++++++++++ recipes/horovod/meta.yaml | 14 -------------- 2 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 recipes/horovod/build.sh diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh new file mode 100644 index 0000000000000..4783af90de301 --- /dev/null +++ b/recipes/horovod/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -ex + +# See https://github.com/horovod/horovod/issues/3956 +flatc -c -o horovod/common/wire horovod/common/wire/message.fbs + +if [[ ${cuda_compiler_version} != "None" ]]; then + export HOROVOD_GPU_OPERATIONS=NCCL + export HOROVOD_NCCL_LINK=SHARED + export HOROVOD_CUDA_HOME=/usr/local/cuda +fi +export HOROVOD_WITH_TENSORFLOW=1 +export HOROVOD_WITH_PYTORCH=1 +# mxnet is not available on conda-forge +# https://github.com/conda-forge/staged-recipes/issues/4447 +export HOROVOD_WITHOUT_MXNET=1 +export HOROVOD_WITH_MPI=1 +# gloo is not avaiable on conda-forge +export HOROVOD_WITHOUT_GLOO=1 +if [[ "${target_platform}" == osx-* ]]; then + # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk + export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" +fi +python -m pip install . -vv \ No newline at end of file diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index 64825a1c4ccd3..b078358c8523e 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -12,20 +12,6 @@ source: build: number: 0 - script_env: - - HOROVOD_GPU_OPERATIONS=NCCL # [cuda_compiler_version != 'None'] - - HOROVOD_NCCL_LINK=SHARED # [cuda_compiler_version != 'None'] - - HOROVOD_WITH_TENSORFLOW=1 - - HOROVOD_WITH_PYTORCH=1 - # mxnet is not available on conda-forge - # https://github.com/conda-forge/staged-recipes/issues/4447 - - HOROVOD_WITHOUT_MXNET=1 - - HOROVOD_WITH_MPI=1 - - HOROVOD_WITHOUT_GLOO=1 - script: - # See https://github.com/horovod/horovod/issues/3956 - - "flatc -c -o horovod/common/wire horovod/common/wire/message.fbs" - - "{{ PYTHON }} -m pip install . -vv" # [unix] skip: true # [win] string: {{ proc_type }}_mpi_{{ mpi }}_h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} ignore_run_exports_from: From 3fa92f141d4e9d97a3644fc7c0157a5d982cf6cb Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 21:53:45 -0500 Subject: [PATCH 16/24] fix errors --- recipes/horovod/build.sh | 1 + recipes/horovod/meta.yaml | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh index 4783af90de301..6e9e078b2cb14 100644 --- a/recipes/horovod/build.sh +++ b/recipes/horovod/build.sh @@ -4,6 +4,7 @@ set -ex # See https://github.com/horovod/horovod/issues/3956 flatc -c -o horovod/common/wire horovod/common/wire/message.fbs +flatc -c -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs if [[ ${cuda_compiler_version} != "None" ]]; then export HOROVOD_GPU_OPERATIONS=NCCL diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index b078358c8523e..d351d52577072 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -19,9 +19,12 @@ build: - tensorflow - pytorch missing_dso_whitelist: - - "*/libtensorflow_framework.so.*" - - "*/libtensorflow_cc.so.*" - - "*/torch/lib/*.so" + - "*/libtensorflow_framework.so.*" # [linux] + - "*/libtensorflow_cc.so.*" # [linux] + - "*/torch/lib/*.so" # [linux] + - "*/libtensorflow_framework.*.dylib" # [osx] + - "*/libtensorflow_cc.*.dylib" # [osx] + - "*/torch/lib/*.dylib" # [osx] requirements: build: From 64da1fe651575c592ed8d3739cd3f02f25f61dd1 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 22:07:51 -0500 Subject: [PATCH 17/24] set include directory for flatc --- recipes/horovod/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh index 6e9e078b2cb14..061c583947e4d 100644 --- a/recipes/horovod/build.sh +++ b/recipes/horovod/build.sh @@ -4,7 +4,7 @@ set -ex # See https://github.com/horovod/horovod/issues/3956 flatc -c -o horovod/common/wire horovod/common/wire/message.fbs -flatc -c -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs +flatc -c -I . -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs if [[ ${cuda_compiler_version} != "None" ]]; then export HOROVOD_GPU_OPERATIONS=NCCL From 10fad29094b23d341c5cee5f640a09b4faf707d3 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 11 Nov 2023 22:30:22 -0500 Subject: [PATCH 18/24] add _pywrap_tensorflow_internal.so --- recipes/horovod/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index d351d52577072..e7b5bcf36920a 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -24,6 +24,7 @@ build: - "*/torch/lib/*.so" # [linux] - "*/libtensorflow_framework.*.dylib" # [osx] - "*/libtensorflow_cc.*.dylib" # [osx] + - "*/_pywrap_tensorflow_internal.so" - "*/torch/lib/*.dylib" # [osx] requirements: From 78a5adc8579e3a09a695d53e9982ed0e02774f0b Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 12 Nov 2023 00:07:44 -0500 Subject: [PATCH 19/24] add `--keep-prefix` to flatc --- recipes/horovod/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh index 061c583947e4d..16bb7e06bb732 100644 --- a/recipes/horovod/build.sh +++ b/recipes/horovod/build.sh @@ -4,7 +4,7 @@ set -ex # See https://github.com/horovod/horovod/issues/3956 flatc -c -o horovod/common/wire horovod/common/wire/message.fbs -flatc -c -I . -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs +flatc -c -I . --keep-prefix -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs if [[ ${cuda_compiler_version} != "None" ]]; then export HOROVOD_GPU_OPERATIONS=NCCL From d6e87c25d3c01ea76d8590b0ac366df6a09a1d57 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 12 Nov 2023 00:10:35 -0500 Subject: [PATCH 20/24] set make flags --- recipes/horovod/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh index 16bb7e06bb732..b72c1a0ef0270 100644 --- a/recipes/horovod/build.sh +++ b/recipes/horovod/build.sh @@ -23,4 +23,7 @@ if [[ "${target_platform}" == osx-* ]]; then # https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk export CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" fi + +# default is -j8 +export MAKEFLAGS="-j${CPU_COUNT}" python -m pip install . -vv \ No newline at end of file From fded6209ad0c347516c8c07e0c6abaff65557833 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 12 Nov 2023 01:42:05 -0500 Subject: [PATCH 21/24] set `--include-prefix` --- recipes/horovod/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/horovod/build.sh b/recipes/horovod/build.sh index b72c1a0ef0270..894c33837aaed 100644 --- a/recipes/horovod/build.sh +++ b/recipes/horovod/build.sh @@ -4,7 +4,7 @@ set -ex # See https://github.com/horovod/horovod/issues/3956 flatc -c -o horovod/common/wire horovod/common/wire/message.fbs -flatc -c -I . --keep-prefix -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs +flatc -c -I . --include-prefix ../common/wire -o horovod/tensorflow horovod/tensorflow/custom_call_config.fbs if [[ ${cuda_compiler_version} != "None" ]]; then export HOROVOD_GPU_OPERATIONS=NCCL @@ -26,4 +26,4 @@ fi # default is -j8 export MAKEFLAGS="-j${CPU_COUNT}" -python -m pip install . -vv \ No newline at end of file +python -m pip install . -vv From c9548a4ea11cd2924b276711788dd9c77b8c044f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sat, 18 Nov 2023 17:50:50 -0500 Subject: [PATCH 22/24] add comments to missing_dso_whitelist --- recipes/horovod/meta.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index e7b5bcf36920a..f6120f7382dd9 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -18,6 +18,13 @@ build: - flatbuffers - tensorflow - pytorch + # Horovod supports multiple frameworks, and the package is built against TensorFlow and PyTorch. + # However, usually, the user only uses one of them simultaneously. (by importing either horovod.tensorflow or horovod.torch) + # Thus, TensorFlow and PyTorch are not listed in the `run` section, but in the `run_constrained` section. + # Users can manually install Horovod and only one of them. + # Both TensorFlow and PyTorch libraries are listed in `missing_dso_whitelist`, and a test is added to ensure Horovod libraries can be + # correctly loaded when TensorFlow and PyTorch are installed. + # (horovod.tensorflow loads Horovod TensorFlow library and horovod.torch loads Horovod PyTorch library) missing_dso_whitelist: - "*/libtensorflow_framework.so.*" # [linux] - "*/libtensorflow_cc.so.*" # [linux] From 15250551a7c0ff95cedbce220796ed0d299c1542 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 19 Nov 2023 05:43:33 -0500 Subject: [PATCH 23/24] Add flatbuffers to build requirements Co-authored-by: Uwe L. Korn --- recipes/horovod/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index f6120f7382dd9..b3a96261afe54 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -42,7 +42,7 @@ requirements: - make # [unix] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - + - flatbuffers host: - python - pip From 98c1532350db3c99ad8c3d76072ac440f648f014 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Sun, 19 Nov 2023 05:49:51 -0500 Subject: [PATCH 24/24] fix indents --- recipes/horovod/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/horovod/meta.yaml b/recipes/horovod/meta.yaml index b3a96261afe54..9f13329925f78 100644 --- a/recipes/horovod/meta.yaml +++ b/recipes/horovod/meta.yaml @@ -42,7 +42,7 @@ requirements: - make # [unix] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - flatbuffers + - flatbuffers host: - python - pip