From 97df4a4ebaceb27503a4796b3454f08a325e31c3 Mon Sep 17 00:00:00 2001 From: "yuxuan.zhuang@dbb.su.se" Date: Wed, 1 May 2024 09:07:01 +0200 Subject: [PATCH 1/9] limit numpy thread --- .github/workflows/gh-ci.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index d705012a4d6..8a4bcda940a 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -105,6 +105,12 @@ jobs: micromamba list pip list + - name: Limit NumPy threads + run: | + echo "OPENBLAS_NUM_THREADS=1" >> $GITHUB_ENV + echo "GOTO_NUM_THREADS=1" >> $GITHUB_ENV + echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV + - name: run_tests if: contains(matrix.name, 'asv_check') != true run: | From 21435cfc533ce23fd0380ff6382f5e79a29db9a6 Mon Sep 17 00:00:00 2001 From: "yuxuan.zhuang@dbb.su.se" Date: Wed, 1 May 2024 09:33:06 +0200 Subject: [PATCH 2/9] limit numpy thread with prefix --- .github/workflows/gh-ci.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index 8a4bcda940a..65b99ad4fad 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -105,21 +105,16 @@ jobs: micromamba list pip list - - name: Limit NumPy threads - run: | - echo "OPENBLAS_NUM_THREADS=1" >> $GITHUB_ENV - echo "GOTO_NUM_THREADS=1" >> $GITHUB_ENV - echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV - - name: run_tests if: contains(matrix.name, 'asv_check') != true run: | + NUMPY_FLAGS="OPENBLAS_NUM_THREADS=1 GOTO_NUM_THREADS=1 OMP_NUM_THREADS=1" PYTEST_FLAGS="--disable-pytest-warnings --durations=50" if [ ${{ matrix.codecov }} = "true" ]; then PYTEST_FLAGS="${PYTEST_FLAGS} --cov-config=.coveragerc --cov=MDAnalysis --cov-report=xml" fi echo $PYTEST_FLAGS - pytest -n auto --timeout=200 testsuite/MDAnalysisTests $PYTEST_FLAGS + $NUMPY_FLAGS pytest -n auto --timeout=200 testsuite/MDAnalysisTests $PYTEST_FLAGS - name: run_asv if: contains(matrix.name, 'asv_check') From 12adcb1427ef30553c3249f0cec1d5c7351a3014 Mon Sep 17 00:00:00 2001 From: "yuxuan.zhuang@dbb.su.se" Date: Wed, 1 May 2024 09:47:46 +0200 Subject: [PATCH 3/9] export instead --- .github/workflows/gh-ci.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index 65b99ad4fad..ef1a37a547a 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -108,13 +108,15 @@ jobs: - name: run_tests if: contains(matrix.name, 'asv_check') != true run: | - NUMPY_FLAGS="OPENBLAS_NUM_THREADS=1 GOTO_NUM_THREADS=1 OMP_NUM_THREADS=1" + export OPENBLAS_NUM_THREADS=1 + export GOTO_NUM_THREADS=1 + export OMP_NUM_THREADS=1 PYTEST_FLAGS="--disable-pytest-warnings --durations=50" if [ ${{ matrix.codecov }} = "true" ]; then PYTEST_FLAGS="${PYTEST_FLAGS} --cov-config=.coveragerc --cov=MDAnalysis --cov-report=xml" fi echo $PYTEST_FLAGS - $NUMPY_FLAGS pytest -n auto --timeout=200 testsuite/MDAnalysisTests $PYTEST_FLAGS + pytest -n auto --timeout=200 testsuite/MDAnalysisTests $PYTEST_FLAGS - name: run_asv if: contains(matrix.name, 'asv_check') From 06f8d342359598a962ecf3fc228f88e932e28df7 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 2 May 2024 14:58:05 -0700 Subject: [PATCH 4/9] reduce computation in encore --- .../MDAnalysisTests/analysis/test_encore.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/testsuite/MDAnalysisTests/analysis/test_encore.py b/testsuite/MDAnalysisTests/analysis/test_encore.py index bc07c21af73..d2cd9c63c9e 100644 --- a/testsuite/MDAnalysisTests/analysis/test_encore.py +++ b/testsuite/MDAnalysisTests/analysis/test_encore.py @@ -125,10 +125,10 @@ def test_triangular_matrix(self): reason="Not yet supported on Windows.") def test_parallel_calculation(self): - arguments = [tuple([i]) for i in np.arange(0,100)] + arguments = [tuple([i]) for i in np.arange(0,10)] parallel_calculation = encore.utils.ParallelCalculation(function=function, - n_jobs=4, + n_jobs=2, args=arguments) results = parallel_calculation.run() @@ -142,12 +142,12 @@ def test_rmsd_matrix_with_superimposition(self, ens1): conf_dist_matrix = encore.confdistmatrix.conformational_distance_matrix( ens1, encore.confdistmatrix.set_rmsd_matrix_elements, - select="name CA", + select="name CA and resnum 1:3", pairwise_align=True, weights='mass', n_jobs=1) - reference = rms.RMSD(ens1, select="name CA") + reference = rms.RMSD(ens1, select="name CA and resnum 1:3") reference.run() err_msg = ( "Calculated RMSD values differ from " @@ -159,7 +159,7 @@ def test_rmsd_matrix_with_superimposition_custom_weights(self, ens1): conf_dist_matrix = encore.confdistmatrix.conformational_distance_matrix( ens1, encore.confdistmatrix.set_rmsd_matrix_elements, - select="name CA", + select="name CA and resnum 1:3", pairwise_align=True, weights='mass', n_jobs=1) @@ -167,16 +167,17 @@ def test_rmsd_matrix_with_superimposition_custom_weights(self, ens1): conf_dist_matrix_custom = encore.confdistmatrix.conformational_distance_matrix( ens1, encore.confdistmatrix.set_rmsd_matrix_elements, - select="name CA", + select="name CA and resnum 1:3", pairwise_align=True, - weights=(ens1.select_atoms('name CA').masses, ens1.select_atoms('name CA').masses), + weights=(ens1.select_atoms("name CA and resnum 1:3").masses, + ens1.select_atoms("name CA and resnum 1:3").masses), n_jobs=1) for i in range(conf_dist_matrix_custom.size): assert_allclose(conf_dist_matrix_custom[0, i], conf_dist_matrix[0, i], rtol=0, atol=1.5e-7) def test_rmsd_matrix_without_superimposition(self, ens1): - selection_string = "name CA" + selection_string = "name CA and resnum 1:3" selection = ens1.select_atoms(selection_string) reference_rmsd = [] coordinates = ens1.trajectory.timeseries(selection, order='fac') From 33ba4003536d194164971bf7d835a395f50b41fc Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 2 May 2024 15:40:36 -0700 Subject: [PATCH 5/9] limit thread in azure pipeline --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 41424cf50de..81f38fdeba1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -128,6 +128,9 @@ jobs: displayName: 'Check installed packages' - powershell: | cd testsuite + $env:OPENBLAS_NUM_THREADS=1 + $env:GOTO_NUM_THREADS=1 + $env:OMP_NUM_THREADS=1 pytest MDAnalysisTests --disable-pytest-warnings -n auto --timeout=200 -rsx --cov=MDAnalysis displayName: 'Run MDAnalysis Test Suite' - script: | From 757d773ff79c5ff7b3ecc1f2aee427bb03d5a7ed Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Thu, 2 May 2024 22:55:13 -0700 Subject: [PATCH 6/9] use 2 nworkers --- .github/workflows/gh-ci.yaml | 4 ++++ azure-pipelines.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index ef1a37a547a..f0de37ffb19 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -111,6 +111,10 @@ jobs: export OPENBLAS_NUM_THREADS=1 export GOTO_NUM_THREADS=1 export OMP_NUM_THREADS=1 + export MKL_NUM_THREADS=1 + # limit to 2 workers to avoid overloading the CI + export PYTEST_XDIST_AUTO_NUM_WORKERS=2 + PYTEST_FLAGS="--disable-pytest-warnings --durations=50" if [ ${{ matrix.codecov }} = "true" ]; then PYTEST_FLAGS="${PYTEST_FLAGS} --cov-config=.coveragerc --cov=MDAnalysis --cov-report=xml" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 81f38fdeba1..a663c1d01d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -131,6 +131,10 @@ jobs: $env:OPENBLAS_NUM_THREADS=1 $env:GOTO_NUM_THREADS=1 $env:OMP_NUM_THREADS=1 + $env:MKL_NUM_THREADS=1 + # limit to 2 workers to avoid overloading the CI + $env:PYTEST_XDIST_AUTO_NUM_WORKERS=2 + pytest MDAnalysisTests --disable-pytest-warnings -n auto --timeout=200 -rsx --cov=MDAnalysis displayName: 'Run MDAnalysis Test Suite' - script: | From e3192e7a99e5fd21cd2c83d90d27ee652f0a9526 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Fri, 3 May 2024 00:18:11 -0700 Subject: [PATCH 7/9] limit azure to 1 core --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a663c1d01d7..16d303bb196 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,7 +133,7 @@ jobs: $env:OMP_NUM_THREADS=1 $env:MKL_NUM_THREADS=1 # limit to 2 workers to avoid overloading the CI - $env:PYTEST_XDIST_AUTO_NUM_WORKERS=2 + $env:PYTEST_XDIST_AUTO_NUM_WORKERS=1 pytest MDAnalysisTests --disable-pytest-warnings -n auto --timeout=200 -rsx --cov=MDAnalysis displayName: 'Run MDAnalysis Test Suite' From 747bfb8ed0efb02606da28dc3beb96365e3e731e Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Fri, 3 May 2024 01:14:31 -0700 Subject: [PATCH 8/9] reduce parallel test time --- .../parallelism/test_multiprocessing.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/testsuite/MDAnalysisTests/parallelism/test_multiprocessing.py b/testsuite/MDAnalysisTests/parallelism/test_multiprocessing.py index 6b29aec6a57..31f5825f93f 100644 --- a/testsuite/MDAnalysisTests/parallelism/test_multiprocessing.py +++ b/testsuite/MDAnalysisTests/parallelism/test_multiprocessing.py @@ -81,8 +81,8 @@ ), (NCDF,), (np.arange(150).reshape(5, 10, 3).astype(np.float64),), - (GRO, [GRO, GRO, GRO, GRO, GRO]), - (PDB, [PDB, PDB, PDB, PDB, PDB]), + (GRO, [GRO, GRO]), + (PDB, [PDB, PDB]), (GRO, [XTC, XTC]), (TRC_PDB_VAC, TRC_TRAJ1_VAC), (TRC_PDB_VAC, [TRC_TRAJ1_VAC, TRC_TRAJ2_VAC]), @@ -121,11 +121,11 @@ def test_multiprocess_COG(u): ag = u.atoms[2:5] ref = np.array([cog(u, ag, i) - for i in range(3)]) + for i in range(2)]) p = multiprocessing.Pool(2) res = np.array([p.apply(cog, args=(u, ag, i)) - for i in range(3)]) + for i in range(2)]) p.close() assert_equal(ref, res) @@ -198,9 +198,9 @@ def test_creating_multiple_universe_without_offset(temp_xtc, ncopies=3): ('memory', np.arange(60).reshape(2, 10, 3).astype(np.float64), dict()), ('TRC', TRC_TRAJ1_VAC, dict()), ('CHAIN', [TRC_TRAJ1_VAC, TRC_TRAJ2_VAC], dict()), - ('CHAIN', [GRO, GRO, GRO], dict()), - ('CHAIN', [PDB, PDB, PDB], dict()), - ('CHAIN', [XTC, XTC, XTC], dict()), + ('CHAIN', [GRO, GRO], dict()), + ('CHAIN', [PDB, PDB], dict()), + ('CHAIN', [XTC, XTC], dict()), ]) def ref_reader(request): fmt_name, filename, extras = request.param From 555520af7dd0a00b5d4e24dd0ac813bbc3e51049 Mon Sep 17 00:00:00 2001 From: Yuxuan Zhuang Date: Mon, 9 Sep 2024 16:08:11 -0700 Subject: [PATCH 9/9] restore gh-ci --- .github/workflows/gh-ci.yaml | 7 ------- azure-pipelines.yml | 7 ------- 2 files changed, 14 deletions(-) diff --git a/.github/workflows/gh-ci.yaml b/.github/workflows/gh-ci.yaml index 9b9cb8b7579..471e3bd20de 100644 --- a/.github/workflows/gh-ci.yaml +++ b/.github/workflows/gh-ci.yaml @@ -108,13 +108,6 @@ jobs: - name: run_tests if: contains(matrix.name, 'asv_check') != true run: | - export OPENBLAS_NUM_THREADS=1 - export GOTO_NUM_THREADS=1 - export OMP_NUM_THREADS=1 - export MKL_NUM_THREADS=1 - # limit to 2 workers to avoid overloading the CI - export PYTEST_XDIST_AUTO_NUM_WORKERS=2 - PYTEST_FLAGS="--disable-pytest-warnings --durations=50" if [ ${{ matrix.codecov }} = "true" ]; then PYTEST_FLAGS="${PYTEST_FLAGS} --cov-config=.coveragerc --cov=MDAnalysis --cov-report=xml" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ca3e744d45b..cace2be35ba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -128,13 +128,6 @@ jobs: displayName: 'Check installed packages' - powershell: | cd testsuite - $env:OPENBLAS_NUM_THREADS=1 - $env:GOTO_NUM_THREADS=1 - $env:OMP_NUM_THREADS=1 - $env:MKL_NUM_THREADS=1 - # limit to 2 workers to avoid overloading the CI - $env:PYTEST_XDIST_AUTO_NUM_WORKERS=1 - pytest MDAnalysisTests --disable-pytest-warnings -n auto --timeout=200 -rsx --cov=MDAnalysis displayName: 'Run MDAnalysis Test Suite' - script: |