Skip to content

Commit 4db4794

Browse files
clee2000clee2000
authored
Add workflow to test against nightly (#3626)
Add a workflow that tests against nightly The hope is that it can easily be changed to test against the test channel that contains the RC when a release is happening, but there is not currently a release happening so it tests against the nightly Currently only runs on main branch
1 parent 9dc8613 commit 4db4794

File tree

4 files changed

+246
-173
lines changed

4 files changed

+246
-173
lines changed

‎.github/workflows/_build-tutorials-base.yml‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
name: Build tutorials
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
USE_NIGHTLY:
7+
description: "Use nightly builds inside build.sh"
8+
required: false
9+
type: number
10+
default: 0
11+
UPLOAD:
12+
description: "Upload built docs to PR preview and main site"
13+
required: false
14+
type: number
15+
default: 0
16+
17+
jobs:
18+
worker:
19+
name: pytorch_tutorial_build_worker
20+
strategy:
21+
matrix:
22+
include:
23+
- { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" }
24+
- { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
25+
- { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
26+
- { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
27+
- { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
28+
- { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
29+
- { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
30+
- { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
31+
- { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
32+
- { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
33+
- { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
34+
- { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
35+
- { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
36+
- { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
37+
- { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" }
38+
fail-fast: false
39+
runs-on: ${{ matrix.runner }}
40+
steps:
41+
- name: Setup SSH (Click me for login details)
42+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
43+
with:
44+
github-secret: ${{ secrets.GITHUB_TOKEN }}
45+
instructions: |
46+
All testing is done inside the container, to start an interactive session run:
47+
docker exec -it $(docker container ps --format '{{.ID}}') bash
48+
49+
- name: Checkout Tutorials
50+
uses: actions/checkout@v3
51+
with:
52+
fetch-depth: 0
53+
54+
- name: Setup Linux
55+
uses: pytorch/pytorch/.github/actions/setup-linux@main
56+
57+
- name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
58+
uses: pytorch/test-infra/.github/actions/setup-nvidia@main
59+
60+
- name: Calculate/build docker image
61+
id: calculate-docker-image
62+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
63+
with:
64+
docker-image-name: tutorials
65+
66+
- name: Pull docker image
67+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
68+
with:
69+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
70+
71+
- name: Build
72+
shell: bash
73+
env:
74+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
75+
NUM_WORKERS: ${{ matrix.num_shards }}
76+
WORKER_ID: ${{ matrix.shard }}
77+
COMMIT_ID: ${{ github.sha }}
78+
JOB_TYPE: worker
79+
COMMIT_SOURCE: ${{ github.ref }}
80+
USE_NIGHTLY: ${{ inputs.USE_NIGHTLY }}
81+
UPLOAD: ${{ inputs.UPLOAD }}
82+
run: |
83+
set -ex
84+
85+
chmod +x ".jenkins/build.sh"
86+
87+
container_name=$(docker run \
88+
${GPU_FLAG:-} \
89+
-e WORKER_ID \
90+
-e NUM_WORKERS \
91+
-e COMMIT_ID \
92+
-e JOB_TYPE \
93+
-e COMMIT_SOURCE \
94+
-e USE_NIGHTLY \
95+
-e UPLOAD \
96+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
97+
--tty \
98+
--detach \
99+
--shm-size=2gb \
100+
--name="${container_name}" \
101+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
102+
-w /var/lib/workspace \
103+
"${DOCKER_IMAGE}"
104+
)
105+
106+
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
107+
108+
- name: Teardown Linux
109+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
110+
if: always()
111+
112+
manager:
113+
name: pytorch_tutorial_build_manager
114+
needs: worker
115+
runs-on: [self-hosted, linux.2xlarge]
116+
if: ${{ inputs.UPLOAD == 1 }}
117+
environment: ${{ github.ref == 'refs/heads/main' && 'pytorchbot-env' || '' }}
118+
steps:
119+
- name: Setup SSH (Click me for login details)
120+
uses: pytorch/test-infra/.github/actions/setup-ssh@main
121+
with:
122+
github-secret: ${{ secrets.GITHUB_TOKEN }}
123+
instructions: |
124+
All testing is done inside the container, to start an interactive session run:
125+
docker exec -it $(docker container ps --format '{{.ID}}') bash
126+
127+
- name: Checkout Tutorials
128+
uses: actions/checkout@v3
129+
with:
130+
fetch-depth: 0
131+
132+
- name: Setup Linux
133+
uses: pytorch/pytorch/.github/actions/setup-linux@main
134+
135+
- name: Calculate/build docker image
136+
id: calculate-docker-image
137+
uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
138+
with:
139+
docker-image-name: tutorials
140+
141+
- name: Pull docker image
142+
uses: pytorch/test-infra/.github/actions/pull-docker-image@main
143+
with:
144+
docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}
145+
146+
- name: Build
147+
shell: bash
148+
env:
149+
DOCKER_IMAGE: ${{ steps.calculate-docker-image.outputs.docker-image }}
150+
NUM_WORKERS: 15
151+
WORKER_ID: 0
152+
COMMIT_ID: ${{ github.sha }}
153+
JOB_TYPE: manager
154+
COMMIT_SOURCE: ${{ github.ref }}
155+
GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.PYTORCHBOT_TOKEN }}
156+
USE_NIGHTLY: ${{ inputs.USE_NIGHTLY }}
157+
run: |
158+
set -ex
159+
160+
chmod +x ".jenkins/build.sh"
161+
162+
container_name=$(docker run \
163+
${GPU_FLAG:-} \
164+
-e WORKER_ID \
165+
-e NUM_WORKERS \
166+
-e COMMIT_ID \
167+
-e JOB_TYPE \
168+
-e COMMIT_SOURCE \
169+
-e GITHUB_PYTORCHBOT_TOKEN \
170+
-e USE_NIGHTLY \
171+
--env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
172+
--tty \
173+
--detach \
174+
--name="${container_name}" \
175+
-v "${GITHUB_WORKSPACE}:/var/lib/workspace" \
176+
-w /var/lib/workspace \
177+
"${DOCKER_IMAGE}"
178+
)
179+
180+
docker exec -u ci-user -t "${container_name}" sh -c ".jenkins/build.sh"
181+
182+
- name: Upload docs preview
183+
uses: seemethere/upload-artifact-s3@v5
184+
if: ${{ github.event_name == 'pull_request' }}
185+
with:
186+
retention-days: 14
187+
s3-bucket: doc-previews
188+
if-no-files-found: error
189+
path: docs
190+
s3-prefix: pytorch/tutorials/${{ github.event.pull_request.number }}
191+
192+
- name: Teardown Linux
193+
uses: pytorch/test-infra/.github/actions/teardown-linux@main
194+
if: always()

‎.github/workflows/build-tutorials-nightly.yml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Build tutorials (nightly/test)
2+
# This is a workflow to build tutorials using nightly or the test/release
3+
# candidate builds for pytorch libraries. It downloads torch and other torch
4+
# related libraries from the nightly or test channel and checks that the
5+
# tutorials can run. This workflow will not upload the built docs anywhere in
6+
# order to prevent polluting the official documentation.
7+
8+
# During releases, this workflow should be run on PRs to verify that the
9+
# tutorials work with the test/rc builds before the official release is made.
10+
# When there is no release candidate, this workflow should only be run on the
11+
# main branch since nightly can be unstable and we do not want to block PRs due
12+
# to failures in this workflow.
13+
14+
# To change the channel between nightly and test/rc, change the index used to
15+
# download the binaries in .jenkins/build.sh.
16+
on:
17+
# Only main branch for now. Uncomment the below line to enable it on PRs.
18+
# pull_request:
19+
20+
# Comment out the below line to disable on the main branch
21+
push:
22+
branches:
23+
- main
24+
25+
concurrency:
26+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
build:
31+
uses: ./.github/workflows/_build-tutorials-base.yml
32+
secrets: inherit
33+
with:
34+
USE_NIGHTLY: 1
35+
UPLOAD: 0

0 commit comments

Comments
 (0)