From c746e9dc41e4c1e643e726886eef78621f07886d Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Fri, 23 Dec 2022 23:32:22 +0100 Subject: [PATCH 01/17] BUG: SUR reader reported wrong `height_scale_factor` in channel info --- SurfaceTopography/ChangeLog.md | 1 + SurfaceTopography/IO/SUR.py | 1 + test/IO/test_io.py | 9 ++++++--- test/IO/test_sur.py | 3 +++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/SurfaceTopography/ChangeLog.md b/SurfaceTopography/ChangeLog.md index 79950f63d..e2be51a58 100644 --- a/SurfaceTopography/ChangeLog.md +++ b/SurfaceTopography/ChangeLog.md @@ -4,6 +4,7 @@ Change log for SurfaceTopography v1.2.6 (not yet released) ------------------------- +- BUG: SUR reader reported wrong `height_scale_factor` in channel info - BUILD: Yet another fix for version discovery when installing from source package diff --git a/SurfaceTopography/IO/SUR.py b/SurfaceTopography/IO/SUR.py index c05dee8a6..75ad45b85 100644 --- a/SurfaceTopography/IO/SUR.py +++ b/SurfaceTopography/IO/SUR.py @@ -179,6 +179,7 @@ def channels(self): dim=2, nb_grid_pts=self._nb_grid_pts, physical_sizes=self._physical_sizes, + height_scale_factor=float(self._header['height_scale_factor']), uniform=True, unit=self._unit)] diff --git a/test/IO/test_io.py b/test/IO/test_io.py index 33f5aa59a..00ba64139 100644 --- a/test/IO/test_io.py +++ b/test/IO/test_io.py @@ -98,6 +98,7 @@ def _convert_filelist(filelist): 'example.vk3', 'example.vk4', 'example.vk6', + 'example.sur', 'mitutoyo_mock.xlsx', 'mitutoyo_nonuniform_mock.xlsx', 'example_ps.tiff', @@ -339,7 +340,7 @@ def test_nb_grid_pts_and_physical_sizes_are_tuples_or_none(fn): @pytest.mark.parametrize('fn', text_example_file_list + text_example_without_size_file_list + binary_example_file_list + binary_without_stream_support_example_file_list) -def test_reader_topography_same(fn): +def test_channel_info_and_topography_have_same_metadata(fn): """ Tests that properties like physical sizes, units and nb_grid_pts are the same in the ChannelInfo and the loaded topography. @@ -366,8 +367,10 @@ def test_reader_topography_same(fn): if channel.physical_sizes is not None: assert channel.physical_sizes == topography.physical_sizes - if channel.height_scale_factor is not None and hasattr(topography, 'scale_factor'): - assert channel.height_scale_factor == topography.scale_factor + if channel.height_scale_factor is not None: + assert channel.height_scale_factor == topography.height_scale_factor + else: + assert not hasattr(topography, 'height_scale_factor') if channel.is_periodic is not None: assert isinstance(channel.is_periodic, (bool, np.bool_)) diff --git a/test/IO/test_sur.py b/test/IO/test_sur.py index c302d5945..7fc4f0277 100644 --- a/test/IO/test_sur.py +++ b/test/IO/test_sur.py @@ -69,3 +69,6 @@ def test_sur_metadata(file_format_examples): assert t.unit == 'mm' np.testing.assert_almost_equal(t.rms_height_from_area(), 0.00029098752636393403) + + print(r.channels[0].height_scale_factor) + print(t.height_scale_factor) \ No newline at end of file From dec9ce099d40c77b5ecc1062dbfc814179944ea9 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Fri, 23 Dec 2022 23:53:44 +0100 Subject: [PATCH 02/17] CI: Updated flake8 --- .github/workflows/flake8.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 155b59065..2c458d3ad 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -25,11 +25,11 @@ jobs: - name: Install dependencies run: | sudo apt-get update -qy - sudo apt-get install -y python3-dev python3-pip libfftw3-dev libopenblas-dev + sudo apt-get install -y python3-dev python3-pip libfftw3-dev libopenblas-dev meson - name: Build extension module run: | - python3 -m pip install --no-binary numpy ".[test]" + python3 -m pip install ".[test]" - name: Flake8 via pytest run: | From 9159623651bbbf7fec560f980a70630a69236cee Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Fri, 23 Dec 2022 23:59:51 +0100 Subject: [PATCH 03/17] CI: Need to fetch tags (also for Flake8) --- .github/workflows/flake8.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index 2c458d3ad..a1f4b9192 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -15,7 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Set up python3 ${{ matrix.python-version }} uses: actions/setup-python@v2 From 92ee1049bd84696ae0b10c2492c74e5f47bd2ddd Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sat, 24 Dec 2022 08:26:04 +0100 Subject: [PATCH 04/17] BUILD: Restrict numpy to < 1.24.0, temporarily addresses #266 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b4a8d81a2..125782cfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.24.0"] build-backend = "mesonpy" [project] From 2a19ce51f6000b24f970cfcbc81673ef844b12f2 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 12:50:48 +0100 Subject: [PATCH 05/17] MAINT: Removed pointers to data type with numpy (e.g. `np.complex128` is now `complex`) - this has been deprecated in numpy 1.20.0 --- SurfaceTopography/Generation.py | 8 ++++---- SurfaceTopography/Nonuniform/PowerSpectrum.py | 2 +- SurfaceTopography/Uniform/Filtering.py | 4 ++-- pyproject.toml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/SurfaceTopography/Generation.py b/SurfaceTopography/Generation.py index afeee1e03..d35c402ae 100644 --- a/SurfaceTopography/Generation.py +++ b/SurfaceTopography/Generation.py @@ -263,13 +263,13 @@ def fourier_synthesis(nb_grid_pts, physical_sizes, # Create in-memory or memory-mapped arrays as storage buffers if rfn is None: - rarr = np.empty(nb_grid_pts, dtype=np.float64) + rarr = np.empty(nb_grid_pts, dtype=float) else: - rarr = np.memmap(rfn, np.float64, 'w+', shape=nb_grid_pts) + rarr = np.memmap(rfn, float, 'w+', shape=nb_grid_pts) if kfn is None: - karr = np.empty(kshape, dtype=np.complex128) + karr = np.empty(kshape, dtype=complex) else: - karr = np.memmap(kfn, np.complex128, 'w+', shape=kshape) + karr = np.memmap(kfn, complex, 'w+', shape=kshape) qy = 2 * np.pi * np.arange(kny) / sy for x in range(nx): diff --git a/SurfaceTopography/Nonuniform/PowerSpectrum.py b/SurfaceTopography/Nonuniform/PowerSpectrum.py index ac59ab93a..fdedf8d1c 100644 --- a/SurfaceTopography/Nonuniform/PowerSpectrum.py +++ b/SurfaceTopography/Nonuniform/PowerSpectrum.py @@ -191,7 +191,7 @@ def power_spectrum(self, reliable=True, algorithm='fft', wavevectors=None, nb_in L = x[-1] - x[0] if wavevectors is None: wavevectors = 2 * np.pi * np.arange(int(L / np.diff(x).min())) / L - y_q = np.zeros_like(wavevectors, dtype=np.complex128) + y_q = np.zeros_like(wavevectors, dtype=complex) for x1, x2, y1, y2 in zip(x[:-1], x[1:], y[:-1], y[1:]): dx = x2 - x1 if dx > 0: diff --git a/SurfaceTopography/Uniform/Filtering.py b/SurfaceTopography/Uniform/Filtering.py index 31a72d7e1..7d4fc7783 100644 --- a/SurfaceTopography/Uniform/Filtering.py +++ b/SurfaceTopography/Uniform/Filtering.py @@ -199,11 +199,11 @@ def heights(self): nx, ny = self.parent_topography.nb_grid_pts sx, sy = self.parent_topography.physical_sizes - qx = np.arange(0, nx, dtype=np.float64).reshape(-1, 1) + qx = np.arange(0, nx, dtype=float).reshape(-1, 1) qx = np.where(qx <= nx // 2, qx / sx, (qx - nx) / sx) qx *= 2 * np.pi - qy = np.arange(0, ny // 2 + 1, dtype=np.float64).reshape(1, -1) + qy = np.arange(0, ny // 2 + 1, dtype=float).reshape(1, -1) qy *= 2 * np.pi / sy if self.is_filter_isotropic: diff --git a/pyproject.toml b/pyproject.toml index 125782cfa..b4a8d81a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.24.0"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy"] build-backend = "mesonpy" [project] From 396164d2d0d7cf1379cbae7cfbedda46399fc289 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 13:18:09 +0100 Subject: [PATCH 06/17] BUILD: Fix for igor (closes #266) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b4a8d81a2..fef3647f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dynamic = [ "version" ] dependencies = [ "defusedxml", "h5py", - "igor", + "git+https://github.com/AurelienJaquier/igor.git@numpy-update", "matplotlib>=1.0.0", "muFFT>=0.24.0", "NuMPI>=0.3.1", From a1711a3b9048bf92a641941377725d0ccf34747a Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 13:22:34 +0100 Subject: [PATCH 07/17] BUILD: Limit numpy version (temporarily fixes #266) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fef3647f7..cba3dfad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.20.0"] build-backend = "mesonpy" [project] @@ -19,7 +19,7 @@ dynamic = [ "version" ] dependencies = [ "defusedxml", "h5py", - "git+https://github.com/AurelienJaquier/igor.git@numpy-update", + "igor", "matplotlib>=1.0.0", "muFFT>=0.24.0", "NuMPI>=0.3.1", From 48fe5846bf346f67acc6e03322c54bdafea10ed4 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 14:00:48 +0100 Subject: [PATCH 08/17] BUILD: Limit numpy version (temporarily fixes #266) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cba3dfad0..f149b23f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.20.0"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.23.0"] build-backend = "mesonpy" [project] From abacdff1200f24b98b3fd10df3adba25cba3b4c1 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 14:27:10 +0100 Subject: [PATCH 09/17] BUILD: Limit numpy version (temporarily fixes #266) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f149b23f6..4f1c10336 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.23.0"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<=1.23.5"] build-backend = "mesonpy" [project] From 2490697695fa94e4e8a1cc496188bcf0c31a7224 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 14:28:20 +0100 Subject: [PATCH 10/17] BUILD: Limit numpy version (temporarily fixes #266) --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4f1c10336..554138b15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<=1.23.5"] +requires = ["meson>=0.46.0", "meson-python>=0.11.0", "ninja", "numpy<1.24.0"] build-backend = "mesonpy" [project] @@ -31,7 +31,7 @@ dependencies = [ "requests", "scipy", "tiffile", - "numpy" + "numpy<1.24.0" ] [project.optional-dependencies] From 85ab490981468671c00177f2345b42725017048b Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 14:29:01 +0100 Subject: [PATCH 11/17] DOC: Added doc string on igor dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 554138b15..9fc0f5342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dynamic = [ "version" ] dependencies = [ "defusedxml", "h5py", - "igor", + "igor", # FIXME! Igor no longer works with numpy 1.24.0 or later "matplotlib>=1.0.0", "muFFT>=0.24.0", "NuMPI>=0.3.1", From 5dd7685d48682625a53366e8d959d6f605de29a0 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 14:53:49 +0100 Subject: [PATCH 12/17] TST: Removed debug prints --- test/IO/test_sur.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/IO/test_sur.py b/test/IO/test_sur.py index 7fc4f0277..c302d5945 100644 --- a/test/IO/test_sur.py +++ b/test/IO/test_sur.py @@ -69,6 +69,3 @@ def test_sur_metadata(file_format_examples): assert t.unit == 'mm' np.testing.assert_almost_equal(t.rms_height_from_area(), 0.00029098752636393403) - - print(r.channels[0].height_scale_factor) - print(t.height_scale_factor) \ No newline at end of file From 64ed3c2351f324320d34fdbde072db5c606e36fb Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 16:20:02 +0100 Subject: [PATCH 13/17] CI: Increased timeout --- .github/workflows/test-code-functionality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-code-functionality.yml b/.github/workflows/test-code-functionality.yml index 630b21c30..657464351 100644 --- a/.github/workflows/test-code-functionality.yml +++ b/.github/workflows/test-code-functionality.yml @@ -14,7 +14,7 @@ on: jobs: tests: runs-on: ubuntu-20.04 - timeout-minutes: 30 + timeout-minutes: 45 strategy: matrix: From 83f50cd5b9ac0d3e301d4388e2960f5635fac5e0 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 16:43:40 +0100 Subject: [PATCH 14/17] CI: Switched to setup-python v4 --- .github/workflows/flake8.yml | 2 +- .github/workflows/test-code-functionality.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml index a1f4b9192..5e86db3d5 100644 --- a/.github/workflows/flake8.yml +++ b/.github/workflows/flake8.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - name: Set up python3 ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/test-code-functionality.yml b/.github/workflows/test-code-functionality.yml index 657464351..73c6bd9a8 100644 --- a/.github/workflows/test-code-functionality.yml +++ b/.github/workflows/test-code-functionality.yml @@ -14,7 +14,7 @@ on: jobs: tests: runs-on: ubuntu-20.04 - timeout-minutes: 45 + timeout-minutes: 30 strategy: matrix: @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 - name: Set up python3 ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From c7dfebfbb2280ac7bcb9178cbdb38af0afccd0fb Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Sun, 25 Dec 2022 16:46:24 +0100 Subject: [PATCH 15/17] TST: Disabled MPI parallel NC I/O test --- test/IO/test_nc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/IO/test_nc.py b/test/IO/test_nc.py index 580798973..003c31882 100644 --- a/test/IO/test_nc.py +++ b/test/IO/test_nc.py @@ -39,6 +39,9 @@ from .test_io import binary_example_file_list, explicit_physical_sizes +@pytest.mark.skipif( + MPI.COMM_WORLD.Get_size() > 1, + reason="FIXME! This tests often stalls (randomly) on multiple MPI processes; disabling for now") def test_save_and_load(maxcomm): nb_grid_pts = (128, 128) size = (3, 3) From 6cc1d6d5ca8ed9ded6c7ab87d70ed824132bb6f3 Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Thu, 29 Dec 2022 10:28:04 +0100 Subject: [PATCH 16/17] DOC: Updated ChangeLog.md --- SurfaceTopography/ChangeLog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SurfaceTopography/ChangeLog.md b/SurfaceTopography/ChangeLog.md index e2be51a58..f26628f5b 100644 --- a/SurfaceTopography/ChangeLog.md +++ b/SurfaceTopography/ChangeLog.md @@ -1,8 +1,8 @@ Change log for SurfaceTopography ============================= -v1.2.6 (not yet released) -------------------------- +v1.2.6 (29Dec22) +---------------- - BUG: SUR reader reported wrong `height_scale_factor` in channel info - BUILD: Yet another fix for version discovery when installing from source From 12fb51d82472d90e5c98c07bb8d8827ded50f88b Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Thu, 29 Dec 2022 10:28:18 +0100 Subject: [PATCH 17/17] DOC: Updated ChangeLog.md --- SurfaceTopography/ChangeLog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SurfaceTopography/ChangeLog.md b/SurfaceTopography/ChangeLog.md index f26628f5b..2501cfbea 100644 --- a/SurfaceTopography/ChangeLog.md +++ b/SurfaceTopography/ChangeLog.md @@ -1,5 +1,5 @@ Change log for SurfaceTopography -============================= +================================ v1.2.6 (29Dec22) ----------------