From d040efa757e973e4e7e8efd9004918407ad11fde Mon Sep 17 00:00:00 2001
From: vfdev-5 <vfdev.5@gmail.com>
Date: Mon, 21 Sep 2020 10:03:31 +0000
Subject: [PATCH 1/2] Updated hvd images - added scripts to auto build and push
 images

---
 docker/hvd/Dockerfile.hvd-apex |  3 +-
 docker/hvd/Dockerfile.hvd-base |  3 +-
 docker/hvd/build_all.sh        | 31 +++++++++++++++++++
 docker/main/build_all.sh       | 31 +++++++++++++++++++
 docker/msdp/build_all.sh       | 31 +++++++++++++++++++
 docker/push_all.sh             | 55 ++++++++++++++++++++++++++++++++++
 6 files changed, 150 insertions(+), 4 deletions(-)
 create mode 100644 docker/hvd/build_all.sh
 create mode 100644 docker/main/build_all.sh
 create mode 100644 docker/msdp/build_all.sh
 create mode 100644 docker/push_all.sh

diff --git a/docker/hvd/Dockerfile.hvd-apex b/docker/hvd/Dockerfile.hvd-apex
index 03cc70878e7d..2e615c8bd16c 100644
--- a/docker/hvd/Dockerfile.hvd-apex
+++ b/docker/hvd/Dockerfile.hvd-apex
@@ -42,8 +42,7 @@ RUN apt-get update && \
 
 # Ignite main dependencies
 RUN pip install tensorboardX tensorboard trains tqdm && \
-    # Horovod support is available in >0.4.1 and in nightly releases:
-    pip install --pre pytorch-ignite
+    pip install pytorch-ignite
 
 # Checkout Ignite examples only
 RUN mkdir -p pytorch-ignite-examples && \
diff --git a/docker/hvd/Dockerfile.hvd-base b/docker/hvd/Dockerfile.hvd-base
index 5e2b5058811f..73eff78ec49c 100644
--- a/docker/hvd/Dockerfile.hvd-base
+++ b/docker/hvd/Dockerfile.hvd-base
@@ -25,8 +25,7 @@ RUN apt-get update && \
 
 # Ignite main dependencies
 RUN pip install tensorboardX tensorboard trains tqdm && \
-    # Horovod support is available in >0.4.1 and in nightly releases:
-    pip install --pre pytorch-ignite
+    pip install pytorch-ignite
 
 # Checkout Ignite examples only
 RUN mkdir -p pytorch-ignite-examples && \
diff --git a/docker/hvd/build_all.sh b/docker/hvd/build_all.sh
new file mode 100644
index 000000000000..97fa0b203233
--- /dev/null
+++ b/docker/hvd/build_all.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+echo "Build all Horovod flavoured PyTorch-Ignite images"
+
+# Start script from ignite docker folder
+if [ ! -d hvd ]; then
+    echo "Can not find 'hvd' folder"
+    echo "Usage: sh hvd/build_all.sh"
+    exit 1
+fi
+
+curr_dir=$PWD
+cd $curr_dir/hvd
+
+set -xeu
+
+image_name="hvd-base"
+
+docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+for image_name in "hvd-vision" "hvd-nlp" "hvd-apex" "hvd-apex-vision" "hvd-apex-nlp"
+do
+
+    docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+    docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+done
+
+cd $curr_dir
\ No newline at end of file
diff --git a/docker/main/build_all.sh b/docker/main/build_all.sh
new file mode 100644
index 000000000000..b8041ae2e210
--- /dev/null
+++ b/docker/main/build_all.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+echo "Build all PyTorch-Ignite images"
+
+# Start script from ignite docker folder
+if [ ! -d main ]; then
+    echo "Can not find 'main' folder"
+    echo "Usage: sh main/build_all.sh"
+    exit 1
+fi
+
+curr_dir=$PWD
+cd $curr_dir/main
+
+set -xeu
+
+image_name="base"
+
+docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+for image_name in "vision" "nlp" "apex" "apex-vision" "apex-nlp"
+do
+
+    docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+    docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+done
+
+cd $curr_dir
\ No newline at end of file
diff --git a/docker/msdp/build_all.sh b/docker/msdp/build_all.sh
new file mode 100644
index 000000000000..8a0ee1ae6387
--- /dev/null
+++ b/docker/msdp/build_all.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+echo "Build all MS DeepSpeed flavoured PyTorch-Ignite images"
+
+# Start script from ignite docker folder
+if [ ! -d msdp ]; then
+    echo "Can not find 'msdp' folder"
+    echo "Usage: sh msdp/build_all.sh"
+    exit 1
+fi
+
+curr_dir=$PWD
+cd $curr_dir/msdp
+
+set -xeu
+
+image_name="msdp-apex-base"
+
+docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+image_name="msdp-apex-vision"
+docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+image_name="msdp-apex-nlp"
+docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+cd $curr_dir
\ No newline at end of file
diff --git a/docker/push_all.sh b/docker/push_all.sh
new file mode 100644
index 000000000000..971654587ec2
--- /dev/null
+++ b/docker/push_all.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+echo "Push all PyTorch-Ignite docker images"
+
+if [ -z $DOCKER_USER ]; then
+    echo "Can not find DOCKER_USER env variable"
+    echo "Please, export DOCKER_USER=<username> before calling this script"
+    exit 1
+fi
+
+if [ -z $DOCKER_TOKEN ]; then
+    echo "Can not find DOCKER_TOKEN env variable"
+    echo "Please, export DOCKER_TOKEN=<token> before calling this script"
+    exit 1
+fi
+
+set -eu
+
+echo $DOCKER_TOKEN | docker login --username=$DOCKER_USER --password-stdin
+
+set -xeu
+
+image_name="base"
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+
+for image_name in "base" "vision" "nlp" "apex" "apex-vision" "apex-nlp"
+do
+
+    docker push pytorchignite/${image_name}:latest
+    docker push pytorchignite/${image_name}:${image_tag}
+
+done
+
+
+image_name="hvd-base"
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+
+for image_name in "hvd-base" "hvd-vision" "hvd-nlp" "hvd-apex" "hvd-apex-vision" "hvd-apex-nlp"
+do
+
+    docker push pytorchignite/${image_name}:latest
+    docker push pytorchignite/${image_name}:${image_tag}
+
+done
+
+image_name="msdp-apex-base"
+image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
+
+for image_name in "msdp-apex-base" "msdp-apex-vision" "msdp-apex-nlp"
+do
+
+    docker push pytorchignite/${image_name}:latest
+    docker push pytorchignite/${image_name}:${image_tag}
+
+done

From 24b3465867e13230757654cde92e3ce4c4697a4c Mon Sep 17 00:00:00 2001
From: vfdev-5 <vfdev.5@gmail.com>
Date: Tue, 22 Sep 2020 17:17:25 +0200
Subject: [PATCH 2/2] Updated scripts according to the review

---
 docker/msdp/build_all.sh | 14 +++++++-------
 docker/push_all.sh       |  5 +++++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/docker/msdp/build_all.sh b/docker/msdp/build_all.sh
index 8a0ee1ae6387..81a3b158bb89 100644
--- a/docker/msdp/build_all.sh
+++ b/docker/msdp/build_all.sh
@@ -20,12 +20,12 @@ docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
 image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
 docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
 
-image_name="msdp-apex-vision"
-docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
-docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+for image_name in "msdp-apex-vision" "msdp-apex-nlp"
+do
 
-image_name="msdp-apex-nlp"
-docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
-docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+    docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
+    docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
+
+done
 
-cd $curr_dir
\ No newline at end of file
+cd $curr_dir
diff --git a/docker/push_all.sh b/docker/push_all.sh
index 971654587ec2..7a441b5315a3 100644
--- a/docker/push_all.sh
+++ b/docker/push_all.sh
@@ -53,3 +53,8 @@ do
     docker push pytorchignite/${image_name}:${image_tag}
 
 done
+
+# If use locally, mind to clean dangling images
+# docker images | grep 'pytorchignite\|<none>' | awk '{print $3}' | xargs docker rmi -f
+# or
+# docker image prune