From 11211336fde56c4e0d14f3adc4e8ed496cef58e9 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Thu, 22 Aug 2024 16:46:16 -0700 Subject: [PATCH 1/7] change the algorithm to remove the flyback to incorporate multiple frames per probe position --- python/stempy/io/sparse_array.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index cec4a04b..dd557e58 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -192,21 +192,20 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): scan_positions_group = f['electron_events/scan_positions'] scan_shape = [scan_positions_group.attrs[x] for x in ['Nx', 'Ny']] frame_shape = [frames.attrs[x] for x in ['Nx', 'Ny']] - + if keep_flyback: data = frames[()] # load the full data set scan_positions = scan_positions_group[()] else: - # Generate the original scan indices from the scan_shape - orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) - # Remove the indices of the last column - crop_indices = np.delete(orig_indices, orig_indices[scan_shape[0]-1::scan_shape[0]]) - # Load only the data needed - data = frames[crop_indices] - # Reduce the column shape by 1 - scan_shape[0] = scan_shape[0] - 1 - # Create the proper scan_positions without the flyback column - scan_positions = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + num = frames.shape[0] // scan_positions_group.shape[0] # number of frames per probe position + data = np.empty(((scan_shape[0]-1) * scan_shape[1] * num), dtype=object) + new_num_cols = scan_shape[0]-1 # number of columns without flyback + for ii in range(scan_shape[1]): + start = ii*new_num_cols*4 # start of cropped data + end = (ii+1)*new_num_cols*4 + start2 = ii*new_num_cols*4+4*ii # start of uncropped data + end2 = (ii+1)*new_num_cols*4+4*ii + data[start:end] = frames[start2:end2] # Load any metadata metadata = {} From 7c8a77174b6ef28943454a702a02f63104e9220c Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Thu, 22 Aug 2024 17:49:10 -0700 Subject: [PATCH 2/7] fix scan_shape --- python/stempy/io/sparse_array.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index dd557e58..7415940c 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -201,17 +201,19 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): data = np.empty(((scan_shape[0]-1) * scan_shape[1] * num), dtype=object) new_num_cols = scan_shape[0]-1 # number of columns without flyback for ii in range(scan_shape[1]): - start = ii*new_num_cols*4 # start of cropped data - end = (ii+1)*new_num_cols*4 - start2 = ii*new_num_cols*4+4*ii # start of uncropped data - end2 = (ii+1)*new_num_cols*4+4*ii + start = ii*new_num_cols*num # start of cropped data + end = (ii+1)*new_num_cols*num + start2 = ii*new_num_cols*num + num*ii # start of uncropped data + end2 = (ii+1)*new_num_cols*num + num*ii data[start:end] = frames[start2:end2] - + scan_shape = (scan_shape[0]-1, scan_shape[1]) # update scan shape + # Load any metadata metadata = {} if 'metadata' in f: load_h5_to_dict(f['metadata'], metadata) + # reverse the scan shape to match expected shape scan_shape = scan_shape[::-1] if version >= 3: From 62b9478f7af4b92967bab6b54782ebdbbd37283d Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 16 Sep 2024 10:58:19 -0700 Subject: [PATCH 3/7] Add scan_positions --- python/stempy/io/sparse_array.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index 7415940c..a4ea2632 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -207,7 +207,9 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): end2 = (ii+1)*new_num_cols*num + num*ii data[start:end] = frames[start2:end2] scan_shape = (scan_shape[0]-1, scan_shape[1]) # update scan shape - + # Create the proper scan_positions without the flyback column + scan_positions = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)], scan_shape) + # Load any metadata metadata = {} if 'metadata' in f: From d9e53ad73d1fa0aa4aca601d2123ebcc3b30471d Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 16 Sep 2024 10:59:22 -0700 Subject: [PATCH 4/7] Change logic for number of frames per probe position to match a similar line later. Its best to keep things consistent. --- python/stempy/io/sparse_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index a4ea2632..e4f5c37b 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -197,7 +197,7 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): data = frames[()] # load the full data set scan_positions = scan_positions_group[()] else: - num = frames.shape[0] // scan_positions_group.shape[0] # number of frames per probe position + num = frames.shape[0] // np.prod(scan_shape, dtype=int) # number of frames per probe position data = np.empty(((scan_shape[0]-1) * scan_shape[1] * num), dtype=object) new_num_cols = scan_shape[0]-1 # number of columns without flyback for ii in range(scan_shape[1]): From 4931a602eaeefafdaa94b1c7a26ca9047569d824 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 16 Sep 2024 11:51:35 -0700 Subject: [PATCH 5/7] use multi-frame data set for flyback test --- tests/test_sparse_array.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index e088884c..92bd79f0 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -710,11 +710,15 @@ def compare_with_sparse(full, sparse): assert np.array_equal(m_array[[False, True], 0][0], position_one) -def test_keep_flyback(electron_data_small): - flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - assert flyback.scan_shape[1] == 50 - no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=False) - assert no_flyback.scan_shape[1] == 49 +def test_keep_flyback(cropped_multi_frames_v3): + # Test keeping the flyback + flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=True) + assert flyback.scan_shape[1] == 20 + assert flyback.num_frames_per_scan = 2 + # Test removing the flyback + no_flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=False) + assert no_flyback.scan_shape[1] == 19 + assert flyback.num_frames_per_scan = 2 # Test binning until this number TEST_BINNING_UNTIL = 33 From 91dca0bcfd946d52479d634c82731cdf51b009e1 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 16 Sep 2024 12:17:53 -0700 Subject: [PATCH 6/7] fix typos in test --- tests/test_sparse_array.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index 92bd79f0..8fc6102f 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -714,11 +714,11 @@ def test_keep_flyback(cropped_multi_frames_v3): # Test keeping the flyback flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=True) assert flyback.scan_shape[1] == 20 - assert flyback.num_frames_per_scan = 2 + assert flyback.num_frames_per_scan == 2 # Test removing the flyback no_flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=False) assert no_flyback.scan_shape[1] == 19 - assert flyback.num_frames_per_scan = 2 + assert flyback.num_frames_per_scan == 2 # Test binning until this number TEST_BINNING_UNTIL = 33 From 733950178b4e30b2ed5d9129c7b343eb4ed908d3 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 16 Sep 2024 12:27:56 -0700 Subject: [PATCH 7/7] add fixutre without flyback --- tests/conftest.py | 5 +++++ tests/test_sparse_array.py | 12 +++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d8cc8f25..bcd86b6d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -109,6 +109,11 @@ def cropped_multi_frames_v2(cropped_multi_frames_data_v2): def cropped_multi_frames_v3(cropped_multi_frames_data_v3): return SparseArray.from_hdf5(cropped_multi_frames_data_v3, dtype=np.uint16) +@pytest.fixture +def cropped_multi_frames_v3_noflyback(cropped_multi_frames_data_v3): + return SparseArray.from_hdf5(cropped_multi_frames_data_v3, + dtype=np.uint16, keep_flyback=False) + @pytest.fixture def simulate_sparse_array(): diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index 8fc6102f..87fa36e0 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -710,15 +710,13 @@ def compare_with_sparse(full, sparse): assert np.array_equal(m_array[[False, True], 0][0], position_one) -def test_keep_flyback(cropped_multi_frames_v3): +def test_keep_flyback(cropped_multi_frames_v3, cropped_multi_frames_v3_noflyback): # Test keeping the flyback - flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=True) - assert flyback.scan_shape[1] == 20 - assert flyback.num_frames_per_scan == 2 + assert cropped_multi_frames_v3.scan_shape[1] == 20 + assert cropped_multi_frames_v3.num_frames_per_scan == 2 # Test removing the flyback - no_flyback = SparseArray.from_hdf5(cropped_multi_frames_v3, keep_flyback=False) - assert no_flyback.scan_shape[1] == 19 - assert flyback.num_frames_per_scan == 2 + assert cropped_multi_frames_v3_noflyback.scan_shape[1] == 19 + assert cropped_multi_frames_v3_noflyback.num_frames_per_scan == 2 # Test binning until this number TEST_BINNING_UNTIL = 33