-
Notifications
You must be signed in to change notification settings - Fork 14
/
tox.ini
177 lines (151 loc) · 6.93 KB
/
tox.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
; For all tests that run in Github Actions, see
; .github/workflows/cpu-ci.yml for the workflow definition.
[tox]
envlist = python3.8,test-gpu,test-cpu
[testenv]
commands =
pip install --upgrade pip
pip install .
[testenv:test-cpu]
; Runs in: Github Actions
; Runs all CPU-based tests. NOTE: if you are using an M1 mac, this will fail. You need to
; change the tensorflow dependency to `tensorflow-macos` in requirements-test-cpu.txt.
deps = -rrequirements-test-cpu.txt
commands = python -m pytest --cov-report term --cov=merlin -rxs tests/unit
[testenv:test-gpu]
sitepackages=true
; Runs in: Internal Jenkins
; Runs GPU-based tests.
; The jenkins jobs run on an image based on merlin-hugectr. This will include all cudf configuration
; and other gpu-specific libraries that we can enxpect will always exist. Thus, we don't need
; to install requirements.txt yet. As we get better at python environment isolation, we will
; need to add some back.
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
deps =
-rrequirements.txt
-rrequirements-dev.txt
-rrequirements-gpu.txt
commands =
python -m pytest --cov-report term --cov merlin -rxs tests/unit
[testenv:test-gpu-not-visible]
sitepackages=true
; Runs in: Internal Jenkins
; Runs GPU-based tests.
; The jenkins jobs run on an image based on merlin-hugectr. This will include all cudf configuration
; and other gpu-specific libraries that we can enxpect will always exist. Thus, we don't need
; to install requirements.txt yet. As we get better at python environment isolation, we will
; need to add some back.
setenv =
TF_GPU_ALLOCATOR=cuda_malloc_async
CUDA_VISIBLE_DEVICES=
deps =
-rrequirements.txt
-rrequirements-dev.txt
commands =
python -m pytest --cov-report term --cov merlin -rxs tests/unit
[testenv:test-nvtabular-cpu]
passenv=GIT_COMMIT
sitepackages=true
allowlist_externals = git
deps =
-rrequirements.txt
-rrequirements-dev.txt
commands =
; the GIT_COMMIT env is the current commit of the core repo
; NOTE!!!! We must clean this up in the jenkins configuration with `rm -rf "nvtabular-$GIT_COMMIT"`
git clone --depth 1 --branch {posargs:main} https://github.com/NVIDIA-Merlin/NVTabular.git nvtabular-{env:GIT_COMMIT}
python -m pip install --upgrade "./nvtabular-{env:GIT_COMMIT}"
python -m pip install --upgrade -r "./nvtabular-{env:GIT_COMMIT}/requirements/test.txt"
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{posargs:main}
python -m pip install .
; this runs the tests then removes the models repo directory whether the tests work or fail
python -m pytest nvtabular-{env:GIT_COMMIT}/tests/unit
[testenv:test-models-cpu]
passenv=GIT_COMMIT
sitepackages=true
allowlist_externals = git
deps =
-rrequirements.txt
-rrequirements-dev.txt
commands =
; the GIT_COMMIT env is the current commit of the core repo
; NOTE!!!! We must clean this up in the jenkins configuration with `rm -rf "nvtabular-$GIT_COMMIT"`
git clone --depth 1 --branch {posargs:main} https://github.com/NVIDIA-Merlin/models.git models-{env:GIT_COMMIT}
python -m pip install --upgrade ./models-{env:GIT_COMMIT}[dev,implicit,lightfm,tensorflow,torch,xgboost]
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{posargs:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{posargs:main}
python -m pip install .
python -m pip install tensorflow==2.9.2
python -m pip install protobuf==3.20.3
; this runs the tests then removes the models repo directory whether the tests work or fail
python -m pytest -m "not notebook" models-{env:GIT_COMMIT}/tests/unit
[testenv:test-systems-cpu]
passenv=GIT_COMMIT
sitepackages=true
allowlist_externals = git
deps =
-rrequirements.txt
-rrequirements-dev.txt
commands =
; the GIT_COMMIT env is the current commit of the core repo
; NOTE!!!! We must clean this up afterward with `rm -rf "systems-$GIT_COMMIT"`
git clone --depth 1 --branch {posargs:main} https://github.com/NVIDIA-Merlin/systems.git systems-{env:GIT_COMMIT}
python -m pip install --upgrade ./systems-{env:GIT_COMMIT}[test-cpu]
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/NVTabular.git@{posargs:main}
python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{posargs:main}
python -m pip install .
; this runs the tests then removes the systems repo directory whether the tests work or fail
python -m pytest -m "not notebook" systems-{env:GIT_COMMIT}/tests/unit
[testenv:test-dataloader-cpu]
passenv=GIT_COMMIT
sitepackages=true
allowlist_externals = git
deps =
-rrequirements.txt
-rrequirements-dev.txt
commands =
; the GIT_COMMIT env is the current commit of the core repo
; NOTE!!!! We must clean this up afterward with `rm -rf "dataloader-$GIT_COMMIT"`
git clone --depth 1 --branch {posargs:main} https://github.com/NVIDIA-Merlin/dataloader.git dataloader-{env:GIT_COMMIT}
python -m pip install --upgrade ./dataloader-{env:GIT_COMMIT}[tensorflow]
python -m pip install --upgrade -r ./dataloader-{env:GIT_COMMIT}/requirements/dev.txt
python -m pip install .
; this runs the tests then removes the dataloader repo directory whether the tests work or fail
python -m pytest dataloader-{env:GIT_COMMIT}/tests/unit
[testenv:test-merlin]
; Runs in: Internal Jenkins
; This runs the end-to-end tests from the NVIDIA-Merlin/Merlin repo on the jenkins machine.
; We will check out `Merlin` from github and execute the notebooks using the current PR of core.
passenv=GIT_COMMIT
sitepackages=true
allowlist_externals = git
deps =
-rrequirements-gpu.txt
commands =
; the GIT_COMMIT env is the current commit of the core repo
; NOTE!!!! We must clean this up in the jenkins configuration with `rm -rf "Merlin-$GIT_COMMIT"`
git clone --depth 1 --branch {posargs:main} https://github.com/NVIDIA-Merlin/Merlin.git Merlin-{env:GIT_COMMIT}
; the downstream packages must be installed in reverse dependency
; order so that they overwrite any other versions installed by packages
; earlier on the list
python -m pip install .
; this runs the tests then removes the Merlin repo directory whether the tests work or fail
python -m pytest Merlin-{env:GIT_COMMIT}/tests/unit
[testenv:docs]
; Runs in: Github Actions
; Generates documentation with sphinx. There are other steps in the Github Actions workflow
; to publish the documentation on release.
changedir = {toxinidir}
deps = -rrequirements-docs.txt
commands =
pip install --upgrade "numpy~=1.23.5"
python -m sphinx.cmd.build -P -b html docs/source docs/build/html
[testenv:docs-multi]
; Run the multi-version build that is shown on GitHub Pages.
changedir = {toxinidir}
deps = -rrequirements-docs.txt
commands =
pip install --upgrade "numpy~=1.23.5"
sphinx-multiversion --dump-metadata docs/source docs/build/html | jq "keys"
sphinx-multiversion docs/source docs/build/html