-
Notifications
You must be signed in to change notification settings - Fork 0
Home
Vibhatha Lakmal Abeykoon edited this page Mar 13, 2021
·
1 revision
Welcome to the PytorchExamples wiki!
cd ..
564 ls
565 rm -rf pytorch/
566 git clone https://github.com/pytorch/pytorch.git
567 ls
568 git clone --recursive https://github.com/pytorch/pytorch/tree/1.6
569 conda install torch
570 conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
571 conda deactivate
572 conda create -n ENVTORCHGPU python=3.7
573 conda activate ENVTORCHGPU
574 conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
575 ls
576 vim mpi_torch_example.py
577 mpirun -n 4 python mpi_torch_example.py
578 pip install mpi4py
579 mpirun -n 4 python mpi_torch_example.py
580 vim mpi_torch_example.py
581 mpirun -n 4 python mpi_torch_example.py
582 vim mpi_torch_example.py
583 mpirun -n 4 python mpi_torch_example.py
584 vim mpi_torch_example.py
585 mpirun -n 4 python mpi_torch_example.py
586 pwd
587 ls
588 python mpi_nccl_example.py
589 mpirun -n 4 python mpi_nccl_example.py
590 ./alloc_romeo
591 ssh r-003
592 conda env list
593 module load conda
594 module load anaconda
595 module load cuda/10.0
596 module load cudnn/10.0-v7.4.1
597 module load gcc/6.4.0
598 export CUDNN_INCLUDE_DIR=/opt/cudnn-10.0-linux-x64-v7.4.1/cuda/include
599 export CUDNN_LIB_DIR=/opt/cudnn-10.0-linux-x64-v7.4.1/cuda/lib64
600 conda env list
601 conda activate ENVTORCHGPU
602 source activate ENVTORCHGPU
603 cd sandbox/
604 ls
605 cd mpi_torch/
606 ls
607 clear
608 ls
609 mpirun -n 4 python mpi_nccl_generic.py
610 nvidia-smi
611 clear
612 cd sandbox/
613 ls
614 cd pyt
615 cd pytorch
616 ls
617 cd pytorch/
618 ls
619 cd ..
620 clear
621 ls
622 cd ..
623 ls
624 cd pytorchplayground/
625 ls
626 cd ..
627 ls
628 module load anaconda
629 module load cuda/10.0
630 module load cudnn/10.0-v7.4.1
631 module load gcc/6.4.0
632 export CUDNN_INCLUDE_DIR=/opt/cudnn-10.0-linux-x64-v7.4.1/cuda/include
633 export CUDNN_LIB_DIR=/opt/cudnn-10.0-linux-x64-v7.4.1/cuda/lib64
634 clear
635 conda env list
636 conda activate ENVTORCH
637 source activate ENVTORCH
638 python3
639 module unload anaconda
640 conda deactivate
641 module avail
642 module load anaconda/2-4.3.0
643 conda activate ENVTORCH
644 python
645 conda deactivate
646 conda activate py36
647 python3
648 ls
649 cd pytorchplayground/
650 ls
651 cd dist_pytorch.py
652 ls
653 cat dist_pytorch.py
654 mpirun -n 4 python dist_pytorch.py
655 mpirun --help
656 vim ~/.bashrc
657 source ~/.bashrc
658 mpirun -n 4 python dist_pytorch.py
659 conda deactivate
660 clear
661 cd
662 ls
663 mkdir source
664 cd source/
665 mkdir pytorch
666 cd pytorch/
667 mkdir 1.6
668 cd 1.6/
669 clear
670 ls
671 wget https://github.com/pytorch/pytorch/archive/v1.6.0.zip
672 clear
673 ls
674 unzip v1.6.0.zip
675 clear
676 ls
677 cd pytorch-1.6.0/
678 clear
679 ls
680 conda activate ENVTORCH
681 python3
682 pip uninstall torch
683 conda deactivate
684 conda activate ENVTORCH37
685 python3
686 clear
687 export CMAKE_PREFIX_PATH="$(dirname $(which conda))/../"
688 gcc --version
689 dirname $(which conda)
690 export CMAKE_PREFIX_PATH="/opt/gcc-6.4.0;$CMAKE_PREFIX_PATH"
691 echo $CMAKE_PREFIX_PATH
692 ls /opt/gcc-6.4.0;/opt/Anaconda2-4.3.0/bin/../
693 ls /opt/gcc-6.4.0;/opt/Anaconda2-4.3.0/bin/
694 ls /opt/gcc-6.4.0;/opt/Anaconda2-4.3.0/bin/../
695 ls /opt/gcc-6.4.0;/opt/Anaconda2-4.3.0/bin/../lib
696 clear
697 ls
698 pip list
699 ls ~/.conda/envs/py36/lib/python3.6/site-packages/cmake/data/bin/*
700 ls ~/.conda/envs/ENVTORCH37/lib/python3.6/site-packages/cmake/data/bin/*
701 ls ~/.conda/envs/ENVTORCH37/lib/python3.7/site-packages/cmake/data/bin/*
702 chmod +x ~/.conda/envs/ENVTORCH37/lib/python3.7/site-packages/cmake/data/bin/*
703 ls ~/.conda/envs/ENVTORCH37/lib/python3.7/site-packages/cmake/data/bin/*
704 clear
705 ls
706 BUILD_TEST=0 python setup.py install
707 git submodules update --init --recursive
708 git submodule update --init --recursive
709 ls
710 cd ..
711 ls
712 mkdir source
713 cd source/
714 clear
715 git clone https://github.com/pytorch/pytorch.git
716 git submodule sync
717 s
718 ls
719 cd pytorch/
720 git submodule sync
721 git submodule update --init --recursive
722 git checkout 1.6
723 ls
724 BUILD_TEST=0 python setup.py install
725 gcc --version
726 module avail
727 module unload gcc/8.2.0
728 BUILD_TEST=0 python setup.py install
729 export CXX=/opt/gcc-6.4.0/bin/c++
730 BUILD_TEST=0 python setup.py install
731 cmake3 --version
732 BUILD_TEST=0 python setup.py install
733 BUILD_TEST=0 python setup.py clean
734 BUILD_TEST=0 python setup.py install
735 git branch
736 BUILD_TEST=0 python setup.py clean
737 ls /opt/gcc-6.4.0/bin/
738 BUILD_TEST=0 python setup.py install
739 export CC=/opt/gcc-6.4.0/bin/gcc
740 BUILD_TEST=0 python setup.py install
741 cd
742 conda deactivate
743 conda create -n ENVTORCHGPU python=3.7
744 conda create -n ENVTORCHGLOO python=3.7
745 conda activate ENVTORCHGLOO
746 clear
747 pip3 install mpi4py
748 cd
749 cd sandbox/mpi_torch/
750 ls
751 cp mpi_nccl_example.py mpi_gloo_example.py
752 vim mpi_gloo_example.py
753 mpirun -n 4 python3 mpi_gloo_example.py
754 conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
755 mpirun -n 4 python3 mpi_gloo_example.py
756 lscpu
757 mpirun -n 48 python3 mpi_gloo_example.py
758 mpirun -n 24 python3 mpi_gloo_example.py
759 mpirun -n 8 python3 mpi_gloo_example.py
760 vim mpi_gloo_example.py
761 cp mpi_gloo_example_cpu.py
762 cp mpi_gloo_example.py mpi_gloo_example_cpu.py
763 vim mpi_gloo_example_cpu.py
764 mpirun -n 24 python3 mpi_gloo_example_cpu.py
765 exit
766 mynodes
767 sinfo
768 squeue
769 ls
770 cat alloc_romeo
771 squeue
772 sinfo
773 screen -x victor_alloc_1
774 salloc -N 13
775 clear
776 squeue
777 screen -ls
778 screen -S victor_alloc
779 squeue
780 clear
781 mynodes
782 ssh v-002
783 module avail
784 gcc --version
785 ls
786 cd source/
787 ls
788 cd pytorch/
789 ls
790 cd source/
791 clear
792 ls
793 cd pytorch/
794 clear
795 ls
796 BUILD_TEST=0 python setup.py install
797 conda activate ENVTORCH
798 module load conda
799 module load anaconda
800 module avail
801 python3 --version
802 ./alloc_romeo
803 ssh r-003
804 mynodes
805 ssh j-071
806 conda activate ENVTORCHGLOO
807 module load anaconda
808 conda activate ENVTORCHGLOO
809 source activate ENVTORCHGLOO
810 python3
811 clear
812 mynodes
813 cd sandbox/mpi_torch/
814 ls
815 vim hostfile
816 lscpu
817 mpirun -n 192 --hostfile hostfile python3 mpi_gloo_example_cpu.py
818 ssh j-072
819 ssh j-073
820 ssh j-074
821 vim hostfile
822 clear
823 ls
824 mpirun -n 144 --hostfile hostfile python3 mpi_gloo_example_cpu.py
825 which python3
826 mpirun -n 144 --hostfile hostfile ~/.conda/envs/ENVTORCHGLOO/bin/python3 mpi_gloo_example_cpu.py
827 lscpu
828 vim hostfile
829 mpirun -n 12 --hostfile hostfile ~/.conda/envs/ENVTORCHGLOO/bin/python3 mpi_gloo_example_cpu.py
830 vim mpi_gloo_example_cpu.py
831 mpirun -n 12 --hostfile hostfile ~/.conda/envs/ENVTORCHGLOO/bin/python3 mpi_gloo_example_cpu.py
832 clear
833 cd
834 cd source/
835 ls
836 cd pytorch/
837 ls
838 cd source/
839 clear
840 ls
841 cd pytorch/
842 BUILD_TEST=0 python setup.py clean
843 conda deactivate
844 conda activate ENVTORCH
845 python3
846 clear
847 c
848 ls
849 clear
850 cls
851 clear
852 ls ~/.conda/envs/py36/lib/python3.6/site-packages/cmake/data/bin/*
853 ls ~/.conda/envs/ENVTORCH/lib/python3.7/site-packages/cmake/data/bin/*
854 ls ~/.conda/envs/ENVTORCH/lib/python3.6/site-packages/cmake/data/bin/*
855 chmod +x ~/.conda/envs/ENVTORCH/lib/python3.6/site-packages/cmake/data/bin/*
856 BUILD_TEST=0 python setup.py install
857 module load gcc/6.4.0
858 module load gcc
859 gcc --version
860 export CXX=/opt/gcc-10.1.0/bin/c++
861 export CC=/opt/gcc-10.1.0/bin/gcc
862 BUILD_TEST=0 python setup.py install
863 BUILD_TEST=0 USE_CUDA=0 python setup.py install
864 BUILD_TEST=0 USE_CUDA=0 python setup.py clean
865 BUILD_TEST=0 USE_CUDA=0 python setup.py install
866 conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses
867 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
868 ls
869 USE_CUDA=0 python setup.py install
870 USE_CUDA=0 python setup.py clean
871 USE_CUDA=0 python setup.py install
872 cd
873 conda deactivate
874 clear
875 ls
876 cd sandbox/
877 mkdir horovod
878 ls
879 cd horovod/
880 ls
881 cd pytorch/
882 ls
883 cat install.sh
884 source ENV/bin/activate
885 vim pytorch_spark_mnist.py
886 ls
887 vim pytorch_mnist.py
888 mpirun -n 24 python3 pytorch_mnist.py
889 ls
890 cat install.sh
891 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip3 install --no-cache-dir git+https://github.com/horovod/horovod.git
892 mpirun --help
893 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip3 install horovod[pytorch]
894 echo $LD_LIBRARY_PATH
895 which mpicc
896 DCMAKE_C_COMPILER=mpicc DCMAKE_CXX_COMPILER=mpiicpc HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip3 install horovod[pytorch]
897 clear
898 ls
899 cd
900 deactivate
901 conda activate ENVTORCH
902 conda install openmpi-mpicc
903 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip3 install horovod[pytorch]
904 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
905 pip
906 conda install pip
907 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
908 python3
909 pip uninstall torch
910 HOROVOD_WITH_MPI=1 HOROVOD_CPU_OPERATIONS=MPI HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
911 pip3 install torch
912 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
913 pip list
914 pip uninstall torch
915 pip3 install torch==1.6.0
916 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
917 ls
918 cd sandbox/
919 ls
920 cd horovod/
921 ls
922 cd pytorch/
923 clear
924 ls
925 cat pytorch_mnist.py
926 cat run_horovod_pytorch.sh
927 sh run_horovod_pytorch.sh
928 pip install torchvision
929 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
930 sh run_horovod_pytorch.sh
931 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
932 pip uninstall horovod
933 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
934 sh run_horovod_pytorch.sh
935 HOROVOD_WITH_MPI=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
936 pip uninstlal horovod
937 pip uninstall horovod
938 HOROVOD_WITH_MPI=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
939 sh run_horovod_pytorch.sh
940 conda create -n ENVTORCHHVD python=3.7
941 conda activate ENVTORCHHVD
942 pip install torchvision
943 HOROVOD_WITH_MPI=1 HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
944 sh run_horovod_pytorch.sh
945 history
946 ls
947 vim pytorch_mnist.py
948 pwd
949 ls
950 pwd
951 ls
952 cat run_horovod_pytorch.sh
953 ssh j-071
954 cd sandbox/horovod/pytorch/
955 ls
956 cat run_horovod_pytorch.sh
957 module avail
958 module load anaconda
959 conda env list
960 conda activate ENVTORCHHVD
961 sourca activate ENVTORCHHVD
962 source activate ENVTORCHHVD
963 clear
964 ls
965 sh run_horovod_pytorch.sh
966 cat run_horovod_pytorch.sh
967 horovodrun --mpi -np 16 python3 pytorch_mnist.py
968 pip3 install requests
969 horovodrun --mpi -np 16 python3 pytorch_mnist.py
970 ls
971 ls data-dir/
972 ls work-dir/
973 ls data-0
974 ls data-0/MNIST/
975 ls data-0/MNIST/raw/
976 ls -lh data-0/MNIST/raw/
977 ls -lh data-1/MNIST/raw/
978 ls -lh data-1/MNIST/processed/
979 horovodrun --mpi -np 16 python3 pytorch_mnist.py
980 ls
981 rm -rf data-*
982 horovodrun --mpi -np 16 python3 pytorch_mnist.py
983 ls
984 horovodrun --mpi -np 16 python3 pytorch_mnist.py
985 rm -rf data-
986 ls
987 rm -rf data-*
988 horovodrun --mpi -np 4 python3 pytorch_mnist.py
989 ssh j-071
990 ssh j-072
991 ssh j-073
992 ps aux | grep python
993 ssh j-074
994 ssh v-002
995 module avail
996 gcc --version
997 module load anaconda
998 cd source/
999 ls
1000 cd pytorch/
1001 ls
1002 cd source/
1003 git branch
1004 ls
1005 cd pytorch/
1006 clear
1007 ls
1008 git branch
1009 python setup.py clean
1010 export CXXFLAGS="${CXXFLAGS} -D__STDC_FORMAT_MACROS")
1011 export CXXFLAGS="${CXXFLAGS} -D__STDC_FORMAT_MACROS"
1012 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1013 conda envlist
1014 conda env list
1015 source activate ENVTORCHSOURCE37
1016 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1017 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1018 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1019 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1020 git checkout master
1021 l
1022 ls
1023 cd ..
1024 ls
1025 rm -rf pytorch/
1026 git clone git clone --recursive https://github.com/pytorch/pytorch
1027 git clone --recursive https://github.com/pytorch/pytorch
1028 git submodule sync
1029 cd pytorch/
1030 git submodule sync
1031 git submodule update --init --recursive
1032 export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
1033 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1034 export USE_CUDA=0
1035 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1036 export USE_CUDA=0 USE_CUDNN=0 USE_MKLDNN=0
1037 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1038 git branch
1039 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1040 BUILD_TEST=0 BUILD_BINARY=0 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1041 python3
1042 cd ..
1043 ls
1044 cd ..
1045 ls
1046 python3
1047 git checkout 1.7
1048 cd source/pytorch/
1049 ls
1050 git checkout 1.6
1051 BUILD_TEST=0 BUILD_BINARY=0 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1052 git submodule sync
1053 git submodule update --init --recursive
1054 BUILD_TEST=0 BUILD_BINARY=0 USE_TENSORPIPE=0 USE_MPI=1 python setup.py clean
1055 BUILD_TEST=0 BUILD_BINARY=0 USE_TENSORPIPE=0 USE_MPI=1 python setup.py install
1056 BUILD_TEST=0 BUILD_BINARY=0 USE_MPI=1 python setup.py clean
1057 BUILD_TEST=0 BUILD_BINARY=0 USE_MPI=1 python setup.py install
1058 cd ../..
1059 python3
1060 history
(ENVTORCHSOURCE37) [vlabeyko@j-071 pytorch]$