[CONDA] Revamp conda recipe. (apache#6732)

* [CONDA] Revamp conda recipe.

- Combines two packages into a single recipe.
- Enable windows build.
- Better packaging hash tag (use git string).

* Address comment

.gitignore

@@ -24,7 +24,7 @@ var/
 *.egg-info/
 .installed.cfg
 *.egg
-
+.conda/
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

conda/Dockerfile.template

@@ -17,30 +17,16 @@
 
 FROM nvidia/cuda:{{ cuda_version }}-devel-ubuntu16.04
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-            bzip2 curl sudo binutils && \
-    rm -rf /var/lib/apt/lists/*
+RUN apt-get update --fix-missing && apt-get install -y bzip2 wget sudo binutils git
 
-RUN  curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v{{ cudnn_short_version }}/cudnn-{{ cuda_version }}-linux-x64-v{{ cudnn_version }}.tgz -O && \
+RUN  wget -q http://developer.download.nvidia.com/compute/redist/cudnn/v{{ cudnn_short_version }}/cudnn-{{ cuda_version }}-linux-x64-v{{ cudnn_version }}.tgz && \
     tar --no-same-owner -xzf cudnn-{{ cuda_version }}-linux-x64-v{{ cudnn_version }}.tgz -C /usr/local && \
     rm cudnn-{{ cuda_version }}-linux-x64-v{{ cudnn_version }}.tgz && \
     ldconfig
 
-
-RUN curl -o ~/miniconda.sh -O https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh  && \
-     chmod +x ~/miniconda.sh && \
-     ~/miniconda.sh -b -p /opt/conda && \
-     rm ~/miniconda.sh && \
-     /opt/conda/bin/conda upgrade --all && \
-     /opt/conda/bin/conda install conda-build conda-verify && \
-     /opt/conda/bin/conda clean -ya
-
-RUN /opt/conda/bin/conda install --download-only cmake make zlib
-RUN /opt/conda/bin/conda install --download-only -c numba llvmdev=8.0.0
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
 
 ENV PATH /opt/conda/bin:$PATH
 ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
 ENV CONDA_BLD_PATH /tmp
-
-WORKDIR /workspace
-RUN chmod -R a+w /workspace

conda/build_cpu.sh

@@ -26,6 +26,4 @@ mkdir -p /tmp/.conda/pkgs
 touch /tmp/.conda/pkgs/urls.txt
 touch /tmp/.conda/environments.txt
 
-
-conda build --output-folder=conda/pkg -c numba conda/tvm-libs
-conda build --output-folder=conda/pkg -m conda/conda_build_config.yaml conda/tvm
+conda build --output-folder=conda/pkg conda/recipe

conda/build_cuda.sh

@@ -26,5 +26,4 @@ mkdir -p /tmp/.conda/pkgs
 touch /tmp/.conda/pkgs/urls.txt
 touch /tmp/.conda/environments.txt
 
-
-conda build --output-folder=conda/pkg --variants "{cuda: True, cuda_version: ${CUDA_VERSION%.*}}" -c numba conda/tvm-libs
+conda build --output-folder=conda/pkg --variants "{cuda: True, cuda_version: ${CUDA_VERSION%.*}}" conda/recipe

conda/build_win.bat

@@ -0,0 +1,18 @@
+:: Licensed to the Apache Software Foundation (ASF) under one
+:: or more contributor license agreements.  See the NOTICE file
+:: distributed with this work for additional information
+:: regarding copyright ownership.  The ASF licenses this file
+:: to you under the Apache License, Version 2.0 (the
+:: "License"); you may not use this file except in compliance
+:: with the License.  You may obtain a copy of the License at
+::
+::   http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing,
+:: software distributed under the License is distributed on an
+:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+:: KIND, either express or implied.  See the License for the
+:: specific language governing permissions and limitations
+:: under the License.
+
+conda build --output-folder=conda/pkg conda/recipe

conda/recipe/bld.bat

@@ -0,0 +1,38 @@
+:: Licensed to the Apache Software Foundation (ASF) under one
+:: or more contributor license agreements.  See the NOTICE file
+:: distributed with this work for additional information
+:: regarding copyright ownership.  The ASF licenses this file
+:: to you under the Apache License, Version 2.0 (the
+:: "License"); you may not use this file except in compliance
+:: with the License.  You may obtain a copy of the License at
+::
+::   http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing,
+:: software distributed under the License is distributed on an
+:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+:: KIND, either express or implied.  See the License for the
+:: specific language governing permissions and limitations
+:: under the License.
+echo on
+
+rd /s /q build
+mkdir build
+cd build
+
+cmake ^
+      -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^
+      -DCMAKE_INSTALL_PREFIX:PATH=%LIBRARY_PREFIX% ^
+      -DUSE_LLVM=ON ^
+      -DUSE_RPC=ON ^
+      -DUSE_CPP_RPC=ON ^
+      -DUSE_SORT=ON ^
+      -DUSE_RANDOM=ON ^
+      -DUSE_GRAPH_RUNTIME_DEBUG=ON ^
+      -DINSTALL_DEV=ON ^
+      %SRC_DIR%
+
+cd ..
+:: defer build to install stage to avoid rebuild.
+:: sometimes windows msbuild is not very good at file
+:: caching and install will results in a rebuild

conda/tvm-libs/build.sh → conda/recipe/build.sh

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,35 +19,41 @@
 set -e
 set -u
 
+GPU_OPT=""
+TOOLCHAIN_OPT=""
+
 if [ "$target_platform" == "osx-64" ]; then
     # macOS 64 bits
-    METAL_OPT="-DUSE_METAL=ON"
-    TOOLCHAIN_OPT="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.11"
-else
-    METAL_OPT=""
-    if [ "$target_platform" == "linux-64" ]; then
-	# Linux 64 bits
-	TOOLCHAIN_OPT="-DCMAKE_TOOLCHAIN_FILE=${RECIPE_DIR}/../cross-linux.cmake"
-    else
-	# Windows (or 32 bits, which we don't support)
-	TOOLCHAIN_OPT=""
-    fi
+    GPU_OPT="-DUSE_METAL=ON"
+elif [ "$target_platform" == "linux-64" ]; then
+    TOOLCHAIN_OPT="-DCMAKE_TOOLCHAIN_FILE=${RECIPE_DIR}/cross-linux.cmake"
 fi
 
 # When cuda is not set, we default to False
 cuda=${cuda:-False}
 
 if [ "$cuda" == "True" ]; then
-    CUDA_OPT="-DUSE_CUDA=ON -DUSE_CUBLAS=ON -DUSE_CUDNN=ON"
+    GPU_OPT="-DUSE_CUDA=ON -DUSE_CUBLAS=ON -DUSE_CUDNN=ON"
     TOOLCHAIN_OPT=""
-else
-    CUDA_OPT=""
 fi
 
+# remove touched cmake config
+rm -f config.cmake
 rm -rf build || true
 mkdir -p build
 cd build
-cmake $METAL_OPT $CUDA_OPT -DUSE_LLVM=$PREFIX/bin/llvm-config -DINSTALL_DEV=ON -DCMAKE_INSTALL_PREFIX="$PREFIX" $TOOLCHAIN_OPT ..
-make -j${CPU_COUNT} VERBOSE=1
-make install
+
+cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DUSE_RPC=ON \
+      -DUSE_CPP_RPC=OFF \
+      -DUSE_SORT=ON \
+      -DUSE_RANDOM=ON \
+      -DUSE_GRAPH_RUNTIME_DEBUG=ON \
+      -DUSE_LLVM=ON \
+      -DINSTALL_DEV=ON \
+      ${GPU_OPT} ${TOOLCHAIN_OPT} \
+      ${SRC_DIR}
+
+make -j${CPU_COUNT}
 cd ..

conda/conda_build_config.yaml → conda/recipe/conda_build_config.yaml

@@ -16,9 +16,9 @@
 # under the License.
 
 python:
-  - 3.5
   - 3.6
   - 3.7
+  - 3.8
 
 cuda:
   - False

conda/recipe/install_libtvm.bat

@@ -0,0 +1,22 @@
+:: Licensed to the Apache Software Foundation (ASF) under one
+:: or more contributor license agreements.  See the NOTICE file
+:: distributed with this work for additional information
+:: regarding copyright ownership.  The ASF licenses this file
+:: to you under the Apache License, Version 2.0 (the
+:: "License"); you may not use this file except in compliance
+:: with the License.  You may obtain a copy of the License at
+::
+::   http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing,
+:: software distributed under the License is distributed on an
+:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+:: KIND, either express or implied.  See the License for the
+:: specific language governing permissions and limitations
+:: under the License.
+
+cmake --build build --config Release --target install
+
+:: Copy files into library bin so that they can be found
+cp %LIBRARY_LIB%\tvm.dll %LIBRARY_BIN%\tvm.dll
+cp %LIBRARY_LIB%\tvm_runtime.dll %LIBRARY_BIN%\tvm_runtime.dll

conda/tvm/build.sh → conda/recipe/install_libtvm.sh

@@ -19,6 +19,5 @@
 set -e
 set -u
 
-cd python
-$PYTHON setup.py install --single-version-externally-managed --record=/tmp/record.txt
-cd ..
+cd build
+make install

conda/recipe/install_tvm_python.bat

@@ -0,0 +1,20 @@
+:: Licensed to the Apache Software Foundation (ASF) under one
+:: or more contributor license agreements.  See the NOTICE file
+:: distributed with this work for additional information
+:: regarding copyright ownership.  The ASF licenses this file
+:: to you under the Apache License, Version 2.0 (the
+:: "License"); you may not use this file except in compliance
+:: with the License.  You may obtain a copy of the License at
+::
+::   http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing,
+:: software distributed under the License is distributed on an
+:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+:: KIND, either express or implied.  See the License for the
+:: specific language governing permissions and limitations
+:: under the License.
+echo on
+
+cd %SRC_DIR%\python
+%PYTHON% setup.py install --single-version-externally-managed --record=%SRC_DIR%\record.txt

conda/tvm/meta.yaml → conda/recipe/install_tvm_python.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -15,48 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-{% set version = "0.8.dev0" %}
+set -e
+set -u
 
-package:
-  name: tvm
-  version: {{ version }}
-
-source:
-  path: ../..
-
-build:
-  number: 0
-
-requirements:
-  build:
-    - {{ compiler('cxx') }}
-  host:
-    - python {{ python }}
-    - cython
-    - numpy
-    - setuptools
-    - decorator
-    - tvm-libs {{ version }}
-  run:
-    - python {{ python }}
-    - {{ pin_compatible('numpy') }}
-    - decorator
-    - tvm-libs {{ version }}
-    - psutil
-
-test:
-  imports:
-    - tvm
-  requires:
-    - pytest
-    - scipy
-  source_files:
-    - tests/python
-  commands:
-    - python -m pytest -v tests/python/integration
-
-about:
-  home: https://github.com/apache/incubator-tvm
-  license: Apache-2.0
-  license_family: Apache
-  summary: a low level domain specific language for compiling tensor computation pipelines
+cd ${SRC_DIR}/python
+${PYTHON} setup.py install --single-version-externally-managed --record=/tmp/record.txt

conda/recipe/meta.yaml

@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% set version = '0.8.dev0' %}
+{% set pkg_name = 'tvm' %}
+{% set cuda_tag = cuda_version | replace('.', '') %} # [cuda]
+{% set pkg_name = pkg_name + '-cu' + cuda_tag %} # [cuda]
+{% set build_tag = environ.get('GIT_BUILD_STR', 'unknown') %}
+{% set build_tag = build_tag + '_h' + PKG_HASH + '_' + PKG_BUILDNUM %}
+
+package:
+  name: {{ pkg_name }}-package
+  version: {{ version }}
+
+source:
+  path: '../..'
+
+build:
+  number: 0
+  include_recipe: False
+  missing_dso_whitelist:
+    - "*libcuda.*"  # [linux]
+
+requirements:
+  build:
+    # The anaconda compilers for OS X are old an annoying
+    # so we rely on the platform ones for now
+    - {{ compiler('cxx') }} # [not osx]
+    - cmake
+    - make # [not win]
+  host:
+    - zlib
+    - llvmdev ==10.0.0
+
+outputs:
+  - name: {{ pkg_name }}-libs
+    script: install_libtvm.bat # [win]
+    script: install_libtvm.sh  # [not win]
+    string: {{ build_tag }}
+    requirements:
+      build:
+        - {{ compiler('cxx') }}
+        - cmake
+        - git
+        - make # [not win]
+      host:
+        - zlib
+        - llvmdev ==10.0.0
+        - {{ pin_compatible('cudatoolkit', lower_bound=cuda_version, max_pin='x.x') }}  # [cuda]
+        - {{ pin_compatible('cudnn', lower_bound='7.6.0', max_pin='x') }}  # [cuda]
+      run:
+        - llvmdev ==10.0.0
+        - {{ pin_compatible('cudatoolkit', lower_bound=cuda_version, max_pin='x.x') }}  # [cuda]
+        - {{ pin_compatible('cudnn', lower_bound='7.6.0', max_pin='x') }}  # [cuda]
+
+  - name: {{ pkg_name }}
+    script: install_tvm_python.sh  # [not win]
+    script: install_tvm_python.bat # [win]
+    string: {{ build_tag }}_py{{ PY_VER | replace('.', '')}}
+    requirements:
+      host:
+        - python
+        - setuptools
+      run:
+        - python
+        - decorator
+        - psutil
+        - {{ pin_compatible('numpy') }}
+        - {{ pin_subpackage(pkg_name + '-libs', exact=True) }}
+
+about:
+  home: https://tvm.apache.org
+  license: Apache2
+  summary: An End to End Deep Learning Compiler Stack for CPUs, GPUs and accelerators.