Skip to content

Commit 936ef53

Browse files
vfdev-5vfdev-5
authored
Updated hvd images + scripts (#1306)
* Updated hvd images - added scripts to auto build and push images * Updated scripts according to the review
1 parent 0c41778 commit 936ef53

File tree

6 files changed

+155
-4
lines changed

6 files changed

+155
-4
lines changed

‎docker/hvd/Dockerfile.hvd-apex‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,7 @@ RUN apt-get update && \
4242

4343
# Ignite main dependencies
4444
RUN pip install tensorboardX tensorboard trains tqdm && \
45-
# Horovod support is available in >0.4.1 and in nightly releases:
46-
pip install --pre pytorch-ignite
45+
pip install pytorch-ignite
4746

4847
# Checkout Ignite examples only
4948
RUN mkdir -p pytorch-ignite-examples && \

‎docker/hvd/Dockerfile.hvd-base‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@ RUN apt-get update && \
2525

2626
# Ignite main dependencies
2727
RUN pip install tensorboardX tensorboard trains tqdm && \
28-
# Horovod support is available in >0.4.1 and in nightly releases:
29-
pip install --pre pytorch-ignite
28+
pip install pytorch-ignite
3029

3130
# Checkout Ignite examples only
3231
RUN mkdir -p pytorch-ignite-examples && \

‎docker/hvd/build_all.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
echo "Build all Horovod flavoured PyTorch-Ignite images"
4+
5+
# Start script from ignite docker folder
6+
if [ ! -d hvd ]; then
7+
echo "Can not find 'hvd' folder"
8+
echo "Usage: sh hvd/build_all.sh"
9+
exit 1
10+
fi
11+
12+
curr_dir=$PWD
13+
cd $curr_dir/hvd
14+
15+
set -xeu
16+
17+
image_name="hvd-base"
18+
19+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
20+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
21+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
22+
23+
for image_name in "hvd-vision" "hvd-nlp" "hvd-apex" "hvd-apex-vision" "hvd-apex-nlp"
24+
do
25+
26+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
27+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
28+
29+
done
30+
31+
cd $curr_dir

‎docker/main/build_all.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
echo "Build all PyTorch-Ignite images"
4+
5+
# Start script from ignite docker folder
6+
if [ ! -d main ]; then
7+
echo "Can not find 'main' folder"
8+
echo "Usage: sh main/build_all.sh"
9+
exit 1
10+
fi
11+
12+
curr_dir=$PWD
13+
cd $curr_dir/main
14+
15+
set -xeu
16+
17+
image_name="base"
18+
19+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
20+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
21+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
22+
23+
for image_name in "vision" "nlp" "apex" "apex-vision" "apex-nlp"
24+
do
25+
26+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
27+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
28+
29+
done
30+
31+
cd $curr_dir

‎docker/msdp/build_all.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
echo "Build all MS DeepSpeed flavoured PyTorch-Ignite images"
4+
5+
# Start script from ignite docker folder
6+
if [ ! -d msdp ]; then
7+
echo "Can not find 'msdp' folder"
8+
echo "Usage: sh msdp/build_all.sh"
9+
exit 1
10+
fi
11+
12+
curr_dir=$PWD
13+
cd $curr_dir/msdp
14+
15+
set -xeu
16+
17+
image_name="msdp-apex-base"
18+
19+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
20+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
21+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
22+
23+
for image_name in "msdp-apex-vision" "msdp-apex-nlp"
24+
do
25+
26+
docker build -t pytorchignite/${image_name}:latest -f Dockerfile.${image_name} .
27+
docker tag pytorchignite/${image_name}:latest pytorchignite/${image_name}:${image_tag}
28+
29+
done
30+
31+
cd $curr_dir

‎docker/push_all.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
echo "Push all PyTorch-Ignite docker images"
4+
5+
if [ -z $DOCKER_USER ]; then
6+
echo "Can not find DOCKER_USER env variable"
7+
echo "Please, export DOCKER_USER=<username> before calling this script"
8+
exit 1
9+
fi
10+
11+
if [ -z $DOCKER_TOKEN ]; then
12+
echo "Can not find DOCKER_TOKEN env variable"
13+
echo "Please, export DOCKER_TOKEN=<token> before calling this script"
14+
exit 1
15+
fi
16+
17+
set -eu
18+
19+
echo $DOCKER_TOKEN | docker login --username=$DOCKER_USER --password-stdin
20+
21+
set -xeu
22+
23+
image_name="base"
24+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
25+
26+
for image_name in "base" "vision" "nlp" "apex" "apex-vision" "apex-nlp"
27+
do
28+
29+
docker push pytorchignite/${image_name}:latest
30+
docker push pytorchignite/${image_name}:${image_tag}
31+
32+
done
33+
34+
35+
image_name="hvd-base"
36+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
37+
38+
for image_name in "hvd-base" "hvd-vision" "hvd-nlp" "hvd-apex" "hvd-apex-vision" "hvd-apex-nlp"
39+
do
40+
41+
docker push pytorchignite/${image_name}:latest
42+
docker push pytorchignite/${image_name}:${image_tag}
43+
44+
done
45+
46+
image_name="msdp-apex-base"
47+
image_tag=`docker run --rm -it pytorchignite/${image_name}:latest -c 'python -c "import torch; import ignite; print(torch.__version__ + \"-\" + ignite.__version__, end=\"\")"'`
48+
49+
for image_name in "msdp-apex-base" "msdp-apex-vision" "msdp-apex-nlp"
50+
do
51+
52+
docker push pytorchignite/${image_name}:latest
53+
docker push pytorchignite/${image_name}:${image_tag}
54+
55+
done
56+
57+
# If use locally, mind to clean dangling images
58+
# docker images | grep 'pytorchignite\|<none>' | awk '{print $3}' | xargs docker rmi -f
59+
# or
60+
# docker image prune

0 commit comments

Comments
 (0)