Skip to content

Commit a10253f

Browse files
PaliCPaliC
committed
Update on "Add dynamic ABI to Multipy"
[ghstack-poisoned]
1 parent fff8f0a commit a10253f

File tree

3 files changed

+88
-28
lines changed

3 files changed

+88
-28
lines changed

‎.github/workflows/package_tests.yaml‎

Lines changed: 84 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,34 +13,94 @@ jobs:
1313
unittest:
1414
strategy:
1515
matrix:
16-
python-version: [3.7, 3.8, 3.9]
17-
platform: [ubuntu-18.04]
18-
include:
19-
- python-version: 3.9
20-
platform: macos-latest
16+
python-version: [3.8]
17+
platform: [linux.2xlarge]
2118
fail-fast: false
2219
runs-on: ${{ matrix.platform }}
2320
steps:
24-
- name: Setup Python ${{ matrix.python-version }}
25-
uses: actions/setup-python@v2
26-
with:
27-
python-version: ${{ matrix.python-version }}
28-
architecture: x64
21+
- name: Update pip
22+
run: |
23+
sudo yum update -y
24+
sudo yum -y install git python3-pip
25+
sudo pip3 install --upgrade pip
26+
- name: Setup conda
27+
run: |
28+
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
29+
bash ~/miniconda.sh -b -p $HOME/miniconda
30+
- name: setup Path
31+
run: |
32+
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
33+
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
34+
2935
- name: Checkout MultiPy
3036
uses: actions/checkout@v2
31-
- name: Install dependencies
32-
run: |
33-
set -eux
34-
pip install coverage codecov
35-
pip install -r requirements.txt
36-
- name: Run tests
37-
run: coverage run -m unittest discover --verbose --start-directory multipy/test/package/ --pattern "test_*"
38-
- name: Coverage
39-
env:
40-
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
37+
with:
38+
submodules: true
39+
- name: Setup SSH (Click me for login details)
40+
uses: ./.github/actions/setup-ssh
41+
with:
42+
github-secret: ${{ secrets.GITHUB_TOKEN }}
43+
- name: Install C++ toolchain
4144
run: |
42-
set -eux
43-
coverage report -m
44-
coverage xml
45+
sudo yum -y install clang llvm
46+
export CC=clang
47+
export CXX=clang++
48+
sudo yum -y install xz-devel bzip2-devel libnsl2-devel readline-devel expat-devel gdbm-devel glibc-devel gmp-devel libffi-devel libGL-devel libX11-devel ncurses-devel openssl-devel sqlite-devel tcl-devel tix-devel tk-devel
49+
sudo yum -y install lzma
50+
sudo yum -y install uuid
51+
sudo yum -y install openmpi-devel
52+
sudo yum -y install zlib-devel
53+
- name: create conda env
54+
run: |
55+
conda create --name multipy_runtime_env python=${{ matrix.python-version }}
56+
conda info
4557
46-
codecov
58+
- name: Install python/pytorch dependencies
59+
shell: bash -l {0}
60+
env:
61+
PYTHON_VERSION: ${{ matrix.python-version }}
62+
run: |
63+
conda run -n multipy_runtime_env python -m pip install astunparse numpy ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses pytest
64+
- name: gen examples
65+
shell: bash -l {0}
66+
env:
67+
PYTHON_VERSION: ${{ matrix.python-version }}
68+
run: |
69+
# A minor hack to get the CI working as conda doesn't have torch,
70+
# fortunately we can remove this once we have a dynamically linked torch
71+
cd multipy/runtime/example
72+
conda create --name example_env python=${{ matrix.python-version }}
73+
conda run -n example_env python -m pip install torch torchvision torchaudio pathlib
74+
conda run -n example_env python generate_examples.py
75+
- name: Build pytorch with ABI=0
76+
shell: bash -l {0}
77+
env:
78+
PYTHON_VERSION: ${{ matrix.python-version }}
79+
run: |
80+
export GLIBCXX_USE_CXX11_ABI=0
81+
export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
82+
export TORCH_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
83+
cd multipy/runtime/third-party/pytorch
84+
export USE_DEPLOY=1
85+
conda run -n multipy_runtime_env python setup.py develop
86+
- name: Build multipy runtime
87+
shell: bash -l {0}
88+
env:
89+
PYTHON_VERSION: ${{ matrix.python-version }}
90+
run: |
91+
cd multipy
92+
cd runtime
93+
mkdir build
94+
cd build
95+
pwd
96+
conda run -n multipy_runtime_env cmake ..
97+
conda run -n multipy_runtime_env cmake --build . --config Release
98+
- name: test_deploy
99+
shell: bash -l {0}
100+
env:
101+
PYTHON_VERSION: ${{ matrix.python-version }}
102+
run: |
103+
cd multipy/runtime
104+
cd build
105+
conda run -n multipy_runtime_env cmake --install . --prefix "."
106+
./test_deploy

‎.github/workflows/runtime_nightly_abi_1.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,9 @@ jobs:
7373
env:
7474
PYTHON_VERSION: ${{ matrix.python-version }}
7575
run: |
76-
# by default Pytorch ABI is set to 1
76+
export GLIBCXX_USE_CXX11_ABI=1
77+
export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1"
78+
export TORCH_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=1"
7779
cd multipy/runtime/third-party/pytorch
7880
export USE_DEPLOY=1
7981
conda run -n multipy_runtime_env python setup.py develop

‎multipy/runtime/CMakeLists.txt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,10 @@ option(ABI_EQUALS_1 "Set ABI value to 1, by default it is set to 0. Pytorch by d
2323

2424
if(ABI_EQUALS_1)
2525
set(ABI_VALUE 1)
26-
message(STATUS "ABI VALUE 1 is - ${ABI_VALUE}")
2726
else()
2827
set(ABI_VALUE 0)
29-
message(STATUS "ABI VALUE 0 is - ${ABI_VALUE}")
3028
endif()
31-
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
29+
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=${ABI_VALUE})
3230

3331
# we do not want to have torch_deployinterpreter linked against libstdc++ or libc because
3432
# when loading it with RTLD_DEEPBIND it will resolve std::cout/stdout to the copy in libc++/libc instead of the

0 commit comments

Comments
 (0)