Skip to content

Commit d89b988

Browse files
ssaini4ssaini4
authored
Merge pull request #25 from CentML/add-ec2-runner
Add EC2 github runners
2 parents ad38061 + 580ac80 commit d89b988

File tree

1 file changed

+298
-0
lines changed

1 file changed

+298
-0
lines changed

‎.github/workflows/whl-build-ec2.yaml‎

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
name: CI
2+
on:
3+
workflow_dispatch:
4+
5+
concurrency:
6+
group: ${{ github.workflow }}-${{ github.head_ref }}
7+
cancel-in-progress: true
8+
9+
jobs:
10+
create-habitat-index-html:
11+
needs: [whl-build-cu111, whl-build-cu113, whl-build-cu116, whl-build-cu117]
12+
runs-on:
13+
group: organization/t4
14+
labels: [self-hosted]
15+
steps:
16+
- name: Configure AWS Credentials
17+
uses: aws-actions/configure-aws-credentials@v1
18+
with:
19+
aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }}
20+
aws-region: ${{ secrets.HABITAT_AWS_REGION }}
21+
aws-secret-access-key: ${{ secrets.HABITAT_AWS_SECRET_ACCESS_KEY }}
22+
23+
- name: Create index.html
24+
run: |
25+
python3 /home/builduser/create_index.py habitat -f wheels-cu102 wheels-cu111 wheels-cu113 wheels-cu116 wheels-cu117
26+
27+
- name: Upload to S3
28+
run: |
29+
aws s3 cp index.html s3://centml-releases/habitat/
30+
31+
start-runner:
32+
name: Start self-hosted EC2 runner
33+
runs-on: ubuntu-latest
34+
outputs:
35+
label: ${{ steps.start-ec2-runner.outputs.label }}
36+
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
37+
steps:
38+
- name: Configure AWS credentials
39+
uses: aws-actions/configure-aws-credentials@v1
40+
with:
41+
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
42+
aws-region: us-east-1
43+
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
44+
- name: Start EC2 runner
45+
id: start-ec2-runner
46+
uses: CentML/GPUClusterSwitchGithubAction@v1.3.5
47+
with:
48+
mode: start
49+
github-token: ${{ secrets.CENTML_PAT }}
50+
t4: true
51+
a10g: false
52+
v100: false
53+
54+
whl-build-cu111:
55+
runs-on:
56+
group: organization/t4
57+
labels: [cu111]
58+
steps:
59+
- name: Configure AWS Credentials
60+
uses: aws-actions/configure-aws-credentials@v1
61+
with:
62+
aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }}
63+
aws-region: ${{ secrets.HABITAT_AWS_REGION }}
64+
aws-secret-access-key: ${{ secrets.HABITAT_AWS_SECRET_ACCESS_KEY }}
65+
66+
- name: Display host information
67+
run: |
68+
uname -a
69+
hostname
70+
pwd
71+
id
72+
ls -la
73+
74+
- name: Fetch repository
75+
uses: actions/checkout@v3
76+
77+
- name: Build Python3.7 wheel
78+
run: |
79+
/home/builduser/build_habitat.sh python3.7 py37
80+
81+
- name: Build Python3.8 wheel
82+
run: |
83+
/home/builduser/build_habitat.sh python3.8 py38
84+
85+
- name: Build Python3.9 wheel
86+
run: |
87+
/home/builduser/build_habitat.sh python3.9 py39
88+
89+
- name: Build Python3.10 wheel
90+
run: |
91+
/home/builduser/build_habitat.sh python3.10 py310
92+
93+
- name: Upload wheels to S3
94+
run: |
95+
aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu111/ --recursive --exclude "*" --include "*.whl"
96+
97+
- name: Create index.html
98+
run: |
99+
file_list=(analyzer/dist/*.whl)
100+
formatted_file_list=$(basename -a ${file_list[@]})
101+
python3 /home/builduser/create_index.py habitat_cu111 -f $formatted_file_list
102+
103+
- name: Upload index.html to S3
104+
run: |
105+
aws s3 cp index.html s3://centml-releases/habitat/wheels-cu111/
106+
107+
whl-build-cu113:
108+
runs-on:
109+
group: organization/t4
110+
labels: [cu113]
111+
steps:
112+
- name: Configure AWS Credentials
113+
uses: aws-actions/configure-aws-credentials@v1
114+
with:
115+
aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }}
116+
aws-region: ${{ secrets.HABITAT_AWS_REGION }}
117+
aws-secret-access-key: ${{ secrets.HABITAT_AWS_SECRET_ACCESS_KEY }}
118+
119+
- name: Display host information
120+
run: |
121+
uname -a
122+
hostname
123+
pwd
124+
id
125+
ls -la
126+
127+
- name: Fetch repository
128+
uses: actions/checkout@v3
129+
130+
- name: Build Python3.7 wheel
131+
run: |
132+
/home/builduser/build_habitat.sh python3.7 py37
133+
134+
- name: Build Python3.8 wheel
135+
run: |
136+
/home/builduser/build_habitat.sh python3.8 py38
137+
138+
- name: Build Python3.9 wheel
139+
run: |
140+
/home/builduser/build_habitat.sh python3.9 py39
141+
142+
- name: Build Python3.10 wheel
143+
run: |
144+
/home/builduser/build_habitat.sh python3.10 py310
145+
146+
- name: Upload wheels to S3
147+
run: |
148+
aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu113/ --recursive --exclude "*" --include "*.whl"
149+
150+
- name: Create index.html
151+
run: |
152+
file_list=(analyzer/dist/*.whl)
153+
formatted_file_list=$(basename -a ${file_list[@]})
154+
python3 /home/builduser/create_index.py habitat_cu113 -f $formatted_file_list
155+
156+
- name: Upload index.html to S3
157+
run: |
158+
aws s3 cp index.html s3://centml-releases/habitat/wheels-cu113/
159+
160+
whl-build-cu116:
161+
runs-on:
162+
group: organization/t4
163+
labels: [cu116]
164+
steps:
165+
- name: Configure AWS Credentials
166+
uses: aws-actions/configure-aws-credentials@v1
167+
with:
168+
aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }}
169+
aws-region: ${{ secrets.HABITAT_AWS_REGION }}
170+
aws-secret-access-key: ${{ secrets.HABITAT_AWS_SECRET_ACCESS_KEY }}
171+
172+
- name: Display host information
173+
run: |
174+
uname -a
175+
hostname
176+
pwd
177+
id
178+
ls -la
179+
180+
- name: Fetch repository
181+
uses: actions/checkout@v3
182+
183+
- name: Build Python3.7 wheel
184+
run: |
185+
/home/builduser/build_habitat.sh python3.7 py37
186+
187+
- name: Build Python3.8 wheel
188+
run: |
189+
/home/builduser/build_habitat.sh python3.8 py38
190+
191+
- name: Build Python3.9 wheel
192+
run: |
193+
/home/builduser/build_habitat.sh python3.9 py39
194+
195+
- name: Build Python3.10 wheel
196+
run: |
197+
/home/builduser/build_habitat.sh python3.10 py310
198+
199+
- name: Upload wheel artifact
200+
uses: actions/upload-artifact@v3
201+
with:
202+
name: wheels-cu116
203+
path: analyzer/dist/*.whl
204+
205+
- name: Upload wheels to S3
206+
run: |
207+
aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu116/ --recursive --exclude "*" --include "*.whl"
208+
209+
- name: Create index.html
210+
run: |
211+
file_list=(analyzer/dist/*.whl)
212+
formatted_file_list=$(basename -a ${file_list[@]})
213+
python3 /home/builduser/create_index.py habitat_cu116 -f $formatted_file_list
214+
215+
- name: Upload index.html to S3
216+
run: |
217+
aws s3 cp index.html s3://centml-releases/habitat/wheels-cu116/
218+
219+
whl-build-cu117:
220+
runs-on:
221+
group: organization/t4
222+
labels: [cu117]
223+
steps:
224+
- name: Configure AWS Credentials
225+
uses: aws-actions/configure-aws-credentials@v1
226+
with:
227+
aws-access-key-id: ${{ secrets.HABITAT_AWS_ACCESS_KEY }}
228+
aws-region: ${{ secrets.HABITAT_AWS_REGION }}
229+
aws-secret-access-key: ${{ secrets.HABITAT_AWS_SECRET_ACCESS_KEY }}
230+
231+
- name: Display host information
232+
run: |
233+
uname -a
234+
hostname
235+
pwd
236+
id
237+
ls -la
238+
239+
- name: Fetch repository
240+
uses: actions/checkout@v3
241+
242+
- name: Build Python3.7 wheel
243+
run: |
244+
/home/builduser/build_habitat.sh python3.7 py37
245+
246+
- name: Build Python3.8 wheel
247+
run: |
248+
/home/builduser/build_habitat.sh python3.8 py38
249+
250+
- name: Build Python3.9 wheel
251+
run: |
252+
/home/builduser/build_habitat.sh python3.9 py39
253+
254+
- name: Build Python3.10 wheel
255+
run: |
256+
/home/builduser/build_habitat.sh python3.10 py310
257+
258+
- name: Upload wheels to S3
259+
run: |
260+
aws s3 cp analyzer/dist/ s3://centml-releases/habitat/wheels-cu117/ --recursive --exclude "*" --include "*.whl"
261+
262+
- name: Create index.html
263+
run: |
264+
file_list=(analyzer/dist/*.whl)
265+
formatted_file_list=$(basename -a ${file_list[@]})
266+
python3 /home/builduser/create_index.py habitat_cu117 -f $formatted_file_list
267+
268+
- name: Upload index.html to S3
269+
run: |
270+
aws s3 cp index.html s3://centml-releases/habitat/wheels-cu117/
271+
272+
stop-runner:
273+
name: Stop self-hosted EC2 runner
274+
needs:
275+
- start-runner # required to get output from the start-runner job
276+
- whl-build-cu117 # required to wait when the main job is done
277+
- whl-build-cu116
278+
- whl-build-cu113
279+
- whl-build-cu111
280+
- create-habitat-index-html
281+
runs-on: ubuntu-latest
282+
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
283+
steps:
284+
- name: Configure AWS credentials
285+
uses: aws-actions/configure-aws-credentials@v1
286+
with:
287+
aws-access-key-id: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
288+
aws-region: us-east-1
289+
aws-secret-access-key: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
290+
- name: Stop EC2 runner
291+
id: stop-ec2-runner
292+
uses: CentML/GPUClusterSwitchGithubAction@v1.3.5
293+
with:
294+
mode: stop
295+
github-token: ${{ secrets.CENTML_PAT }}
296+
t4: true # required to match the list above - otherwise the runners will not stop
297+
a10g: false # required to match the list above - otherwise the runners will not stop
298+
v100: false

0 commit comments

Comments
 (0)