[v1.x] Backport #20782, #20783 and #20786 from v1.9.x branch, #19945 …

…from master (#20785)

* [v1.9.x] Fix CD for pypi wheel version (#20782)

* Change env variable used to detect whether it's a release build or not. RELEASE_BUILD is already setup in the CD pipeline.

* Refine how variable is compared to trigger release builds.

* Check for 'true' exclusively, to prevent unwanted release builds.

* Make sure we pass any environment variables on the build.py command line to the docker run command.

* [v1.9.x] Fix aarch64 cd pipeline (#20783)

* Add wheel to installed pip packages on aarch64, so we can build the wheel in the CD pipeline.

* Better support for aarch64 docker container cache.

* Only import machine() function from platform module, as platform variable is already widely used.

* Only build containers with aarch64 in name on aarch64.

* Make all arch defaults to local machine type.

* Retry getting test data to prevent CI tests from failing - increase connect timeout from 10 to 60sec, increase retry delay from 0 (setting duplicated) to 30, increase retry attempts to 5.

* Update website for v1.9.x branch. (#20786)

* [website] Fix broken website for master version  (#19945)

* fix website

* fix header

* fix index

* remove locall test

* fix main.scss

Co-authored-by: Wei Chu <weichu@amazon.com>

Co-authored-by: waytrue17 <52505574+waytrue17@users.noreply.github.com>
Co-authored-by: Wei Chu <weichu@amazon.com>

ci/build.py

@@ -34,6 +34,7 @@
 import shutil
 import signal
 import subprocess
+from platform import machine
 from itertools import chain
 from subprocess import check_call, check_output
 from typing import *
@@ -46,7 +47,7 @@
 DOCKER_COMPOSE_FILES = set(['docker/build.centos7'])
 
 # keywords to identify arm-based dockerfiles
-AARCH_FILE_KEYWORDS = ['armv', 'aarch64']
+AARCH_FILE_KEYWORDS = ['aarch64']
 
 def get_dockerfiles_path():
     return "docker"
@@ -60,15 +61,15 @@ def get_docker_compose_platforms(path: str = get_dockerfiles_path()):
     return platforms
 
 
-def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]:
+def get_platforms(path: str = get_dockerfiles_path(), arch=machine()) -> List[str]:
     """Get a list of platforms given our dockerfiles"""
     dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
     dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles))
     files = set(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
     files = files - DOCKER_COMPOSE_FILES
     files.update(["build."+x for x in get_docker_compose_platforms()])
     arm_files = set(filter(lambda x: any(y in x for y in AARCH_FILE_KEYWORDS), files))
-    if arch == 'x86':
+    if arch == 'x86_64':
         files = files - arm_files
     elif arch == 'aarch64':
         files = arm_files
@@ -273,15 +274,10 @@ def container_run(platform: str,
         # mount mxnet/build for storing build
         '-v', "{}:/work/build".format(local_build_folder),
         '-v', "{}:/work/ccache".format(local_ccache_dir),
-        '-u', '{}:{}'.format(os.getuid(), os.getgid()),
-        '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),
-        # temp dir should be local and not shared
-        '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),
-        # this path is inside the container as /work/ccache is mounted
-        '-e', "CCACHE_DIR={}".format(environment['CCACHE_DIR']),
-        # a container-scoped log, useful for ccache verification.
-        '-e', "CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']),
+        '-u', '{}:{}'.format(os.getuid(), os.getgid())
     ]
+    for e in environment.keys():
+        docker_arg_list += ['-e', '{}={}'.format(e, environment[e])]
     docker_arg_list += [tag]
     docker_arg_list.extend(command)
 
@@ -305,7 +301,7 @@ def docker_run_cmd(cmd):
     return 0
 
 
-def list_platforms(arch='x86') -> str:
+def list_platforms(arch=machine()) -> str:
     return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch)))
 
 
@@ -362,8 +358,8 @@ def main() -> int:
                         type=str)
 
     parser.add_argument("-A", "--architecture",
-                        help="Architecture of images to build (x86 or aarch64). Default is x86.",
-                        default='x86',
+                        help="Architecture of images to build (x86_64 or aarch64). Default is current machine type.",
+                        default=machine(),
                         dest='architecture')
 
     parser.add_argument("-b", "--build-only",

ci/docker/install/requirements_aarch64

@@ -30,3 +30,4 @@ astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4
 setuptools
 coverage
+wheel

ci/docker_cache.py

@@ -30,6 +30,7 @@
 import subprocess
 import re
 import sys
+from platform import machine
 from typing import *
 
 import build as build_util
@@ -225,7 +226,7 @@ def script_name() -> str:
 
     args = parser.parse_args()
 
-    platforms = build_util.get_platforms()
+    platforms = build_util.get_platforms(arch=machine())
 
     if "dkr.ecr" in args.docker_registry:
         _ecr_login(args.docker_registry)

cpp-package/example/get_data.sh

@@ -33,11 +33,10 @@ download () {
     fi
 
     echo "Downloading ${URL} ..."
-    local CURL_OPTIONS="--connect-timeout 10 \
+    local CURL_OPTIONS="--connect-timeout 60 \
               --max-time 300 \
-              --retry-delay 10 \
-              --retry 3 \
-              --retry-delay 0 \
+              --retry-delay 30 \
+              --retry 5 \
               --location \
               --silent"
     curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME}

docs/python_docs/themes/mx-theme/mxtheme/header_top.html

@@ -1,6 +1,6 @@
 <header class="site-header" role="banner">
   <div class="wrapper">
-      <a class="site-title" rel="author" href="/versions/1.8.0/"><img
+      <a class="site-title" rel="author" href="/versions/1.9.0/"><img
             src="{{pathto('_static/mxnet_logo.png', 1)}}" class="site-header-logo"></a>
     <nav class="site-nav">
       <input type="checkbox" id="nav-trigger" class="nav-trigger"/>
@@ -13,19 +13,20 @@
       </label>
 
       <div class="trigger">
-        <a class="page-link" href="/versions/1.8.0/get_started">Get Started</a>
-        <a class="page-link" href="/versions/1.8.0/blog">Blog</a>
-        <a class="page-link" href="/versions/1.8.0/features">Features</a>
-        <a class="page-link" href="/versions/1.8.0/ecosystem">Ecosystem</a>
-        <a class="page-link page-current" href="/versions/1.8.0/api">Docs & Tutorials</a>
+        <a class="page-link" href="/versions/1.9.0/get_started">Get Started</a>
+        <a class="page-link" href="/versions/1.9.0/blog">Blog</a>
+        <a class="page-link" href="/versions/1.9.0/features">Features</a>
+        <a class="page-link" href="/versions/1.9.0/ecosystem">Ecosystem</a>
+        <a class="page-link page-current" href="/versions/1.9.0/api">Docs & Tutorials</a>
         <a class="page-link" href="https://github.com/apache/incubator-mxnet">GitHub</a>
         <div class="dropdown">
-          <span class="dropdown-header">1.8.0
+          <span class="dropdown-header">1.9.0
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a class="dropdown-option" href="/">master</a><br>
-            <a class="dropdown-option-active" href="/versions/1.8.0/">1.8.0</a><br>
+            <a class="dropdown-option-active" href="/versions/1.9.0/">1.9.0</a><br>
+            <a class="dropdown-option" href="/versions/1.8.0/">1.8.0</a><br>
             <a class="dropdown-option" href="/versions/1.7.0/">1.7.0</a><br>
             <a class="dropdown-option" href="/versions/1.6.0/">1.6.0</a><br>
             <a class="dropdown-option" href="/versions/1.5.0/">1.5.0</a><br>

docs/static_site/src/404.html

@@ -1,23 +1,19 @@
-<!---
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
--->
-
 ---
+  # Licensed to the Apache Software Foundation (ASF) under one
+  # or more contributor license agreements.  See the NOTICE file
+  # distributed with this work for additional information
+  # regarding copyright ownership.  The ASF licenses this file
+  # to you under the Apache License, Version 2.0 (the
+  # "License"); you may not use this file except in compliance
+  # with the License.  You may obtain a copy of the License at
+  #   http://www.apache.org/licenses/LICENSE-2.0
+  # Unless required by applicable law or agreed to in writing,
+  # software distributed under the License is distributed on an
+  # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  # KIND, either express or implied.  See the License for the
+  # specific language governing permissions and limitations
+  # under the License.
+
 layout: default
 ---
 

docs/static_site/src/_config.yml

@@ -37,9 +37,10 @@ description: >- # this means to ignore newlines until "baseurl:"
 twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_config_beta.yml

@@ -39,9 +39,10 @@ url: https://thomasdelteil.github.io
 twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_config_prod.yml

@@ -39,9 +39,10 @@ twitter_username: apachemxnet
 github_username:  apache/incubator-mxnet
 youtube_username: apachemxnet
 google_analytics: UA-96378503-1
-baseurl: /versions/1.8.0
+baseurl: /versions/1.9.0
 versions: 
   - master
+  - 1.9.0
   - 1.8.0
   - 1.7.0
   - 1.6.0

docs/static_site/src/_includes/get_started/get_started.html

@@ -1,7 +1,7 @@
 <script>
     /** Defaults **/
     /** See options.js for the full ugly script **/
-    var versionSelect = defaultVersion = 'v1.8.0';
+    var versionSelect = defaultVersion = 'v1.9.0';
     var platformSelect = 'linux';
     var languageSelect = 'python';
     var processorSelect = 'cpu';
@@ -24,13 +24,14 @@ <h2>Platform and use-case specific instructions for using MXNet</h2>
             <div class="col-9 install-right">
                 <div class="dropdown" id="version-dropdown-container">
                     <button class="current-version dropbtn btn" type="button" data-toggle="dropdown">
-                        v1.8.0
+                        v1.9.0
                         <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true">
                             <path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path>
                         </svg>
                     </button>
                     <ul class="opt-group version-dropdown">
-                        <li class="opt active versions"><a href="#">v1.8.0</a></li>
+                        <li class="opt active versions"><a href="#">v1.9.0</a></li>
+                        <li class="opt versions"><a href="#">v1.8.0</a></li>
                         <li class="opt versions"><a href="#">v1.7.0</a></li>
                         <li class="opt versions"><a href="#">v1.6.0</a></li>
                         <li class="opt versions"><a href="#">v1.5.1</a></li>

docs/static_site/src/_includes/get_started/linux/python/cpu/pip.md

@@ -9,11 +9,18 @@ page](https://mxnet.apache.org/get_started/download).
 
 Run the following command:
 
-<div class="v1-8-0">
+<div class="v1-9-0">
 {% highlight bash %}
 pip install mxnet
 {% endhighlight %}
 
+</div> <!-- End of v1-9-0 -->
+
+<div class="v1-8-0">
+{% highlight bash %}
+pip install mxnet==1.8.0.post0
+{% endhighlight %}
+
 Start from 1.7.0 release, oneDNN(previously known as: MKL-DNN/DNNL) is enabled
 in pip packages by default.
 
@@ -35,7 +42,7 @@ MXNet tuning guide</a>.
 To install native MXNet without oneDNN, run the following command:
 
 {% highlight bash %}
-pip install mxnet-native
+pip install mxnet-native==1.8.0.post0
 {% endhighlight %}
 
 </div> <!-- End of v1-8-0 -->

docs/static_site/src/_includes/get_started/linux/python/gpu/pip.md

@@ -10,11 +10,18 @@ page](https://mxnet.apache.org/get_started/download).
 
 Run the following command:
 
-<div class="v1-8-0">
+<div class="v1-9-0">
 {% highlight bash %}
 $ pip install mxnet-cu102
 {% endhighlight %}
 
+</div> <!-- End of v1-9-0 -->
+
+<div class="v1-8-0">
+{% highlight bash %}
+$ pip install mxnet-cu102==1.8.0.post0
+{% endhighlight %}
+
 </div> <!-- End of v1-8-0 -->
 
 <div class="v1-7-0">

docs/static_site/src/_includes/get_started/pip_snippet.md

@@ -1,7 +1,7 @@
 You can then <a href="/get_started/validate_mxnet.html">validate your MXNet installation</a>.
 
 <div style="text-align: center">
-    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.8.0.png"
+    <img src="https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/install/pip-packages-1.9.0.png"
     alt="pip packages"/>
 </div>
 

docs/static_site/src/_includes/header.html

@@ -87,13 +87,14 @@
         <a class="page-link" href="{{'/api' | relative_url }}">Docs & Tutorials</a>
         <a class="page-link" href="https://github.com/apache/incubator-mxnet">GitHub</a>
         <div class="dropdown">
-          <span class="dropdown-header">1.8.0
+          <span class="dropdown-header">1.9.0
             <svg class="dropdown-caret" viewBox="0 0 32 32" class="icon icon-caret-bottom" aria-hidden="true"><path class="dropdown-caret-path" d="M24 11.305l-7.997 11.39L8 11.305z"></path></svg>
           </span>
           <div class="dropdown-content">
             <a href="/">master</a>
-            <a class="dropdown-option-active" href="/versions/1.8.0/">1.8.0</a>
-            <a href="/versions/1.6.0/">1.7.0</a>
+            <a class="dropdown-option-active" href="/versions/1.9.0/">1.9.0</a>
+            <a href="/versions/1.8.0/">1.8.0</a>
+            <a href="/versions/1.7.0/">1.7.0</a>
             <a href="/versions/1.6.0/">1.6.0</a>
             <a href="/versions/1.5.0/">1.5.0</a>
             <a href="/versions/1.4.1/">1.4.1</a>

docs/static_site/src/_layouts/default.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+
 <!---
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
@@ -7,16 +8,15 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   KIND, either express or implied.  See the License for the
   specific language governing permissions and limitations
   under the License.
 -->
+
 <html lang="{{ page.lang | default: site.lang | default: " en" }}">
 
 {%- include head.html -%}

docs/static_site/src/_layouts/home.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+
 <!---
   Licensed to the Apache Software Foundation (ASF) under one
   or more contributor license agreements.  See the NOTICE file
@@ -7,16 +8,15 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   KIND, either express or implied.  See the License for the
   specific language governing permissions and limitations
   under the License.
 -->
+
 <html lang="{{ page.lang | default: site.lang | default: " en" }}">
 
 {%- include head.html -%}