From 08733345efacc53cb0a2af2a17c6142b3dbc1a5b Mon Sep 17 00:00:00 2001 From: davidparks21 Date: Tue, 5 Dec 2023 15:06:29 -0800 Subject: [PATCH 1/9] Fix bug in common utils and remove old files that are causing issues with import --- .../test_data/maxwell-metadata.expected.json | 175 ------------------ .../data/test_data/maxwell-metadata.old.json | 174 ----------------- src/braingeneers/utils/common_utils.py | 24 ++- src/braingeneers/utils/common_utils_test.py | 50 +++++ 4 files changed, 61 insertions(+), 362 deletions(-) delete mode 100644 braingeneers/data/test_data/maxwell-metadata.expected.json delete mode 100644 braingeneers/data/test_data/maxwell-metadata.old.json create mode 100644 src/braingeneers/utils/common_utils_test.py diff --git a/braingeneers/data/test_data/maxwell-metadata.expected.json b/braingeneers/data/test_data/maxwell-metadata.expected.json deleted file mode 100644 index dc20144..0000000 --- a/braingeneers/data/test_data/maxwell-metadata.expected.json +++ /dev/null @@ -1,175 +0,0 @@ -{ - "uuid": "2023-08-12-e-GABA_v0", - "timestamp": "2023-10-05T18:10:02", - "hardware": "Maxwell", - "maxwell_chip_id": "20325, 20247, 20402", - "notes": { - "purpose": "GABA dose response sweep. Gaba added X:55-X+1:~15 for x=2,4,... Started at .5uM, doubled each time (last was 14 not 16)", - "biology": { - "sample_type": "organoid", - "aggregation_date": "23-06-22", - "plating_date": "23-07-17", - "species": "mouse", - "cell_line": "e14", - "genotype": "", - "culture_media": "gfcdm", - "organoid_tracker_sr_no": "", - "organoid modification": "", - "organoid modification date": "" - }, - "stimulation": { - "optogenetic": false, - "electrical": false, - "pharmacological": "GABA" - }, - "automated_feeding": false, - "automated_imaging": false, - "comments": "" - }, - "ephys_experiments": { - "data_GABA_BL_20325": { - "name": "data_GABA_BL_20325", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 819, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:01:19;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6001200, - "path": "shared/data_GABA_BL_20325.nwb", - "timestamp": "2023-08-12 T15:01:19;" - } - ], - "data_format": "NeurodataWithoutBorders" - }, - "Trace_20230812_17_00_14_GABA_BL_20247": { - "name": "Trace_20230812_17_00_14_GABA_BL_20247", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 944, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T17:00:41;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6242800, - "path": "shared/Trace_20230812_17_00_14_GABA_BL_20247.nwb", - "timestamp": "2023-08-12 T17:00:41;" - } - ], - "data_format": "NeurodataWithoutBorders" - }, - "Trace_20230812_15_55_43_GABA_BL_20402": { - "name": "Trace_20230812_15_55_43_GABA_BL_20402", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 971, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:58:02;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6108800, - "path": "shared/Trace_20230812_15_55_43_GABA_BL_20402.nwb", - "timestamp": "2023-08-12 T15:58:02;" - } - ], - "data_format": "NeurodataWithoutBorders" - }, - "Trace_20230812_16_03_20_GABA_DR2_20402": { - "name": "Trace_20230812_16_03_20_GABA_DR2_20402", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 971, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T16:04:39;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16962400, - "path": "shared/Trace_20230812_16_03_20_GABA_DR2_20402.nwb", - "timestamp": "2023-08-12 T16:04:39;" - } - ], - "data_format": "NeurodataWithoutBorders" - }, - "Trace_20230812_15_18_49_GABA_DR_20325": { - "name": "Trace_20230812_15_18_49_GABA_DR_20325", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 819, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:19:25;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16888000, - "path": "shared/Trace_20230812_15_18_49_GABA_DR_20325.nwb", - "timestamp": "2023-08-12 T15:19:25;" - } - ], - "data_format": "NeurodataWithoutBorders" - }, - "Trace_20230812_17_27_36_GABA_DR_20247": { - "name": "Trace_20230812_17_27_36_GABA_DR_20247", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 944, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T17:27:59;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16890400, - "path": "shared/Trace_20230812_17_27_36_GABA_DR_20247.nwb", - "timestamp": "2023-08-12 T17:27:59;" - } - ], - "data_format": "NeurodataWithoutBorders" - } - } -} \ No newline at end of file diff --git a/braingeneers/data/test_data/maxwell-metadata.old.json b/braingeneers/data/test_data/maxwell-metadata.old.json deleted file mode 100644 index aa77584..0000000 --- a/braingeneers/data/test_data/maxwell-metadata.old.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "uuid": "2023-08-12-e-GABA_v0", - "timestamp": "", - "maxwell_chip_id": "20325, 20247, 20402", - "notes": { - "purpose": "GABA dose response sweep. Gaba added X:55-X+1:~15 for x=2,4,... Started at .5uM, doubled each time (last was 14 not 16)", - "biology": { - "sample_type": "organoid", - "aggregation_date": "23-06-22", - "plating_date": "23-07-17", - "species": "mouse", - "cell_line": "e14", - "genotype": "", - "culture_media": "gfcdm", - "organoid_tracker_sr_no": "", - "organoid modification": "", - "organoid modification date": "" - }, - "stimulation": { - "optogenetic": false, - "electrical": false, - "pharmacological": "GABA" - }, - "automated_feeding": false, - "automated_imaging": false, - "comments": "" - }, - "ephys_experiments": { - "data_GABA_BL_20325": { - "name": "data_GABA_BL_20325", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 819, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:01:19;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6001200, - "path": "original/data/data_GABA_BL_20325.raw.h5", - "timestamp": "2023-08-12 T15:01:19;" - } - ] - }, - "Trace_20230812_17_00_14_GABA_BL_20247": { - "name": "Trace_20230812_17_00_14_GABA_BL_20247", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 944, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T17:00:41;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6242800, - "path": "original/data/Trace_20230812_17_00_14_GABA_BL_20247.raw.h5", - "timestamp": "2023-08-12 T17:00:41;" - } - ] - }, - "Trace_20230812_15_55_43_GABA_BL_20402": { - "name": "Trace_20230812_15_55_43_GABA_BL_20402", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 971, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:58:02;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 6108800, - "path": "original/data/Trace_20230812_15_55_43_GABA_BL_20402.raw.h5", - "timestamp": "2023-08-12 T15:58:02;" - } - ] - }, - "Trace_20230812_16_03_20_GABA_DR2_20402": { - "name": "Trace_20230812_16_03_20_GABA_DR2_20402", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 971, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T16:04:39;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16962400, - "path": "original/data/Trace_20230812_16_03_20_GABA_DR2_20402.raw.h5", - "timestamp": "2023-08-12 T16:04:39;" - } - ] - }, - "Trace_20230812_15_18_49_GABA_DR_20325": { - "name": "Trace_20230812_15_18_49_GABA_DR_20325", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 819, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T15:19:25;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16888000, - "path": "original/data/Trace_20230812_15_18_49_GABA_DR_20325.raw.h5", - "timestamp": "2023-08-12 T15:19:25;" - } - ] - }, - "Trace_20230812_17_27_36_GABA_DR_20247": { - "name": "Trace_20230812_17_27_36_GABA_DR_20247", - "hardware": "Maxwell", - "maxwell_chip_number": "N/A", - "maxwell_tracker_sr_no": "xxxxxx", - "channels": [], - "notes": "", - "num_channels": 944, - "num_current_input_channels": 0, - "num_voltage_channels": 32, - "offset": 0, - "sample_rate": 20000, - "voltage_scaling_factor": 1, - "timestamp": "2023-08-12 T17:27:59;", - "units": "\u00b5V", - "version": 20190530, - "blocks": [ - { - "num_frames": 16890400, - "path": "original/data/Trace_20230812_17_27_36_GABA_DR_20247.raw.h5", - "timestamp": "2023-08-12 T17:27:59;" - } - ] - } - } -} \ No newline at end of file diff --git a/src/braingeneers/utils/common_utils.py b/src/braingeneers/utils/common_utils.py index 14a9f45..353f7d9 100644 --- a/src/braingeneers/utils/common_utils.py +++ b/src/braingeneers/utils/common_utils.py @@ -99,27 +99,25 @@ def file_list(filepath: str) -> List[Tuple[str, str, int]]: """ Returns a list of files, last modified time, and size on local or S3 in descending order of last modified time - :param filepath: Local or S3 file path to list, example: "local/dir/" or "s3://braingeneers/ephys/ + :param filepath: Local or S3 file path to list, example: "local/dir/" or "s3://bucket/prefix/" :return: A list of tuples of [('fileA', 'last_modified_A', size), ('fileB', 'last_modified_B', size), ...] """ + files_and_details = [] + if filepath.startswith('s3://'): s3_client = _lazy_init_s3_client() o = urllib.parse.urlparse(filepath) response = s3_client.list_objects(Bucket=o.netloc, Prefix=o.path[1:]) - if 'Contents' not in response: - if raise_on_missing: - raise FileNotFoundError(filepath) - else: - return [(o.path[1:].split('/')[-1], 'Missing')] - - files_and_details = [ - (f['Key'].split('/')[-1], str(f['LastModified']), int(f['Size'])) - for f in sorted(response['Contents'], key=lambda x: x['LastModified'], reverse=True) - ] + if 'Contents' in response: + files_and_details = [ + (f['Key'].split('/')[-1], str(f['LastModified']), int(f['Size'])) + for f in sorted(response['Contents'], key=lambda x: x['LastModified'], reverse=True) + ] else: - files = sorted(pathlib.Path(filepath).iterdir(), key=os.path.getmtime, reverse=True) - files_and_details = [(f.name, str(f.stat().st_mtime), f.stat().st_size) for f in files] + if os.path.exists(filepath): + files = sorted(pathlib.Path(filepath).iterdir(), key=os.path.getmtime, reverse=True) + files_and_details = [(f.name, str(f.stat().st_mtime), f.stat().st_size) for f in files] return files_and_details diff --git a/src/braingeneers/utils/common_utils_test.py b/src/braingeneers/utils/common_utils_test.py new file mode 100644 index 0000000..4dfac49 --- /dev/null +++ b/src/braingeneers/utils/common_utils_test.py @@ -0,0 +1,50 @@ +import unittest +from unittest.mock import patch, MagicMock +import file_utils +import os +import tempfile +from pathlib import Path + + +class TestFileListFunction(unittest.TestCase): + + @patch('file_utils._lazy_init_s3_client') + def test_s3_files_exist(self, mock_s3_client): + # Mock S3 client response + mock_response = { + 'Contents': [ + {'Key': 'file1.txt', 'LastModified': '2023-01-01', 'Size': 123}, + {'Key': 'file2.txt', 'LastModified': '2023-01-02', 'Size': 456} + ] + } + mock_s3_client.return_value.list_objects.return_value = mock_response + + result = file_utils.file_list('s3://test-bucket/') + expected = [('file1.txt', '2023-01-01', 123), ('file2.txt', '2023-01-02', 456)] + self.assertEqual(result, expected) + + @patch('file_utils._lazy_init_s3_client') + def test_s3_no_files(self, mock_s3_client): + # Mock S3 client response for no files + mock_s3_client.return_value.list_objects.return_value = {} + result = file_utils.file_list('s3://test-bucket/') + self.assertEqual(result, []) + + def test_local_files_exist(self): + with tempfile.TemporaryDirectory() as temp_dir: + # Create temporary files + Path(temp_dir, 'tempfile1.txt').touch() + Path(temp_dir, 'tempfile2.txt').touch() + + result = file_utils.file_list(temp_dir) + # The result should contain two files with their details + self.assertEqual(len(result), 2) + + def test_local_no_files(self): + with tempfile.TemporaryDirectory() as temp_dir: + result = file_utils.file_list(temp_dir) + self.assertEqual(result, []) + + +if __name__ == '__main__': + unittest.main() From 00a08ba74c13cefa3ce281731949de7cf9fc8604 Mon Sep 17 00:00:00 2001 From: DailyDreaming Date: Wed, 6 Dec 2023 15:26:06 -0800 Subject: [PATCH 2/9] Add a version stub. --- .gitignore | 2 +- src/braingeneers/_version.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 src/braingeneers/_version.py diff --git a/.gitignore b/.gitignore index 2a464b6..f6925c5 100644 --- a/.gitignore +++ b/.gitignore @@ -159,7 +159,7 @@ cython_debug/ #.idea/ # Version file -**/_version.py +#**/_version.py .ipynb_checkpoints .coverage diff --git a/src/braingeneers/_version.py b/src/braingeneers/_version.py new file mode 100644 index 0000000..614db6e --- /dev/null +++ b/src/braingeneers/_version.py @@ -0,0 +1,16 @@ +# file generated by setuptools_scm +# don't change, don't track in version control +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple, Union + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '0.1.1.dev8' +__version_tuple__ = version_tuple = (0, 1, 1, 'dev8') From 9e9dbe7ff0845de1d8d39de748ed35492fda1c1d Mon Sep 17 00:00:00 2001 From: DailyDreaming Date: Wed, 6 Dec 2023 15:26:45 -0800 Subject: [PATCH 3/9] Go back to ignoring version.py stub. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f6925c5..2a464b6 100644 --- a/.gitignore +++ b/.gitignore @@ -159,7 +159,7 @@ cython_debug/ #.idea/ # Version file -#**/_version.py +**/_version.py .ipynb_checkpoints .coverage From 1741c1cc35b7032476f23ac086207bd6b402487c Mon Sep 17 00:00:00 2001 From: Lon Blauvelt Date: Wed, 6 Dec 2023 17:24:04 -0800 Subject: [PATCH 4/9] Update _version.py --- src/braingeneers/_version.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/braingeneers/_version.py b/src/braingeneers/_version.py index 614db6e..4c402ed 100644 --- a/src/braingeneers/_version.py +++ b/src/braingeneers/_version.py @@ -1,16 +1,2 @@ -# file generated by setuptools_scm -# don't change, don't track in version control -TYPE_CHECKING = False -if TYPE_CHECKING: - from typing import Tuple, Union - VERSION_TUPLE = Tuple[Union[int, str], ...] -else: - VERSION_TUPLE = object - -version: str -__version__: str -__version_tuple__: VERSION_TUPLE -version_tuple: VERSION_TUPLE - __version__ = version = '0.1.1.dev8' __version_tuple__ = version_tuple = (0, 1, 1, 'dev8') From cd3f45cc6b845d9eef140f1a8110d7a721afcfa1 Mon Sep 17 00:00:00 2001 From: Lon Blauvelt Date: Wed, 6 Dec 2023 17:24:40 -0800 Subject: [PATCH 5/9] Update _version.py --- src/braingeneers/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/braingeneers/_version.py b/src/braingeneers/_version.py index 4c402ed..2d07c8e 100644 --- a/src/braingeneers/_version.py +++ b/src/braingeneers/_version.py @@ -1,2 +1,2 @@ -__version__ = version = '0.1.1.dev8' -__version_tuple__ = version_tuple = (0, 1, 1, 'dev8') +__version__ = version = '0.0.0.dev0' +__version_tuple__ = version_tuple = (0, 0, 0, 'dev0') From 7e2ae51302689428dc723d0dde3360e49d9e2064 Mon Sep 17 00:00:00 2001 From: davidparks21 Date: Sat, 9 Dec 2023 14:29:19 -0800 Subject: [PATCH 6/9] Unit tests --- src/braingeneers/utils/common_utils.py | 1 + src/braingeneers/utils/common_utils_test.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/braingeneers/utils/common_utils.py b/src/braingeneers/utils/common_utils.py index 353f7d9..83b2d87 100644 --- a/src/braingeneers/utils/common_utils.py +++ b/src/braingeneers/utils/common_utils.py @@ -10,6 +10,7 @@ import multiprocessing import posixpath import itertools +import pathlib _s3_client = None # S3 client for boto3, lazy initialization performed in _lazy_init_s3_client() diff --git a/src/braingeneers/utils/common_utils_test.py b/src/braingeneers/utils/common_utils_test.py index 4dfac49..3876fbe 100644 --- a/src/braingeneers/utils/common_utils_test.py +++ b/src/braingeneers/utils/common_utils_test.py @@ -1,6 +1,6 @@ import unittest from unittest.mock import patch, MagicMock -import file_utils +import common_utils # Updated import statement import os import tempfile from pathlib import Path @@ -8,7 +8,7 @@ class TestFileListFunction(unittest.TestCase): - @patch('file_utils._lazy_init_s3_client') + @patch('common_utils._lazy_init_s3_client') # Updated to common_utils def test_s3_files_exist(self, mock_s3_client): # Mock S3 client response mock_response = { @@ -19,15 +19,15 @@ def test_s3_files_exist(self, mock_s3_client): } mock_s3_client.return_value.list_objects.return_value = mock_response - result = file_utils.file_list('s3://test-bucket/') - expected = [('file1.txt', '2023-01-01', 123), ('file2.txt', '2023-01-02', 456)] + result = common_utils.file_list('s3://test-bucket/') # Updated to common_utils + expected = [('file2.txt', '2023-01-02', 456), ('file1.txt', '2023-01-01', 123)] self.assertEqual(result, expected) - @patch('file_utils._lazy_init_s3_client') + @patch('common_utils._lazy_init_s3_client') # Updated to common_utils def test_s3_no_files(self, mock_s3_client): # Mock S3 client response for no files mock_s3_client.return_value.list_objects.return_value = {} - result = file_utils.file_list('s3://test-bucket/') + result = common_utils.file_list('s3://test-bucket/') # Updated to common_utils self.assertEqual(result, []) def test_local_files_exist(self): @@ -36,13 +36,13 @@ def test_local_files_exist(self): Path(temp_dir, 'tempfile1.txt').touch() Path(temp_dir, 'tempfile2.txt').touch() - result = file_utils.file_list(temp_dir) + result = common_utils.file_list(temp_dir) # Updated to common_utils # The result should contain two files with their details self.assertEqual(len(result), 2) def test_local_no_files(self): with tempfile.TemporaryDirectory() as temp_dir: - result = file_utils.file_list(temp_dir) + result = common_utils.file_list(temp_dir) # Updated to common_utils self.assertEqual(result, []) From 76e2cd5b2468afa630aa110f21456f0fa800d095 Mon Sep 17 00:00:00 2001 From: DailyDreaming Date: Thu, 14 Dec 2023 12:39:41 -0800 Subject: [PATCH 7/9] Repair old test files. --- .../test_data/maxwell-metadata.expected.json | 175 ++++++++++++++++++ .../data/test_data/maxwell-metadata.old.json | 174 +++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 braingeneers/data/test_data/maxwell-metadata.expected.json create mode 100644 braingeneers/data/test_data/maxwell-metadata.old.json diff --git a/braingeneers/data/test_data/maxwell-metadata.expected.json b/braingeneers/data/test_data/maxwell-metadata.expected.json new file mode 100644 index 0000000..c39ae72 --- /dev/null +++ b/braingeneers/data/test_data/maxwell-metadata.expected.json @@ -0,0 +1,175 @@ +{ + "uuid": "2023-08-12-e-GABA_v0", + "timestamp": "2023-10-05T18:10:02", + "hardware": "Maxwell", + "maxwell_chip_id": "20325, 20247, 20402", + "notes": { + "purpose": "GABA dose response sweep. Gaba added X:55-X+1:~15 for x=2,4,... Started at .5uM, doubled each time (last was 14 not 16)", + "biology": { + "sample_type": "organoid", + "aggregation_date": "23-06-22", + "plating_date": "23-07-17", + "species": "mouse", + "cell_line": "e14", + "genotype": "", + "culture_media": "gfcdm", + "organoid_tracker_sr_no": "", + "organoid modification": "", + "organoid modification date": "" + }, + "stimulation": { + "optogenetic": false, + "electrical": false, + "pharmacological": "GABA" + }, + "automated_feeding": false, + "automated_imaging": false, + "comments": "" + }, + "ephys_experiments": { + "data_GABA_BL_20325": { + "name": "data_GABA_BL_20325", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 819, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:01:19;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6001200, + "path": "shared/data_GABA_BL_20325.nwb", + "timestamp": "2023-08-12 T15:01:19;" + } + ], + "data_format": "NeurodataWithoutBorders" + }, + "Trace_20230812_17_00_14_GABA_BL_20247": { + "name": "Trace_20230812_17_00_14_GABA_BL_20247", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 944, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T17:00:41;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6242800, + "path": "shared/Trace_20230812_17_00_14_GABA_BL_20247.nwb", + "timestamp": "2023-08-12 T17:00:41;" + } + ], + "data_format": "NeurodataWithoutBorders" + }, + "Trace_20230812_15_55_43_GABA_BL_20402": { + "name": "Trace_20230812_15_55_43_GABA_BL_20402", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 971, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:58:02;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6108800, + "path": "shared/Trace_20230812_15_55_43_GABA_BL_20402.nwb", + "timestamp": "2023-08-12 T15:58:02;" + } + ], + "data_format": "NeurodataWithoutBorders" + }, + "Trace_20230812_16_03_20_GABA_DR2_20402": { + "name": "Trace_20230812_16_03_20_GABA_DR2_20402", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 971, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T16:04:39;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16962400, + "path": "shared/Trace_20230812_16_03_20_GABA_DR2_20402.nwb", + "timestamp": "2023-08-12 T16:04:39;" + } + ], + "data_format": "NeurodataWithoutBorders" + }, + "Trace_20230812_15_18_49_GABA_DR_20325": { + "name": "Trace_20230812_15_18_49_GABA_DR_20325", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 819, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:19:25;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16888000, + "path": "shared/Trace_20230812_15_18_49_GABA_DR_20325.nwb", + "timestamp": "2023-08-12 T15:19:25;" + } + ], + "data_format": "NeurodataWithoutBorders" + }, + "Trace_20230812_17_27_36_GABA_DR_20247": { + "name": "Trace_20230812_17_27_36_GABA_DR_20247", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 944, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T17:27:59;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16890400, + "path": "shared/Trace_20230812_17_27_36_GABA_DR_20247.nwb", + "timestamp": "2023-08-12 T17:27:59;" + } + ], + "data_format": "NeurodataWithoutBorders" + } + } +} diff --git a/braingeneers/data/test_data/maxwell-metadata.old.json b/braingeneers/data/test_data/maxwell-metadata.old.json new file mode 100644 index 0000000..48afc10 --- /dev/null +++ b/braingeneers/data/test_data/maxwell-metadata.old.json @@ -0,0 +1,174 @@ +{ + "uuid": "2023-08-12-e-GABA_v0", + "timestamp": "", + "maxwell_chip_id": "20325, 20247, 20402", + "notes": { + "purpose": "GABA dose response sweep. Gaba added X:55-X+1:~15 for x=2,4,... Started at .5uM, doubled each time (last was 14 not 16)", + "biology": { + "sample_type": "organoid", + "aggregation_date": "23-06-22", + "plating_date": "23-07-17", + "species": "mouse", + "cell_line": "e14", + "genotype": "", + "culture_media": "gfcdm", + "organoid_tracker_sr_no": "", + "organoid modification": "", + "organoid modification date": "" + }, + "stimulation": { + "optogenetic": false, + "electrical": false, + "pharmacological": "GABA" + }, + "automated_feeding": false, + "automated_imaging": false, + "comments": "" + }, + "ephys_experiments": { + "data_GABA_BL_20325": { + "name": "data_GABA_BL_20325", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 819, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:01:19;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6001200, + "path": "original/data/data_GABA_BL_20325.raw.h5", + "timestamp": "2023-08-12 T15:01:19;" + } + ] + }, + "Trace_20230812_17_00_14_GABA_BL_20247": { + "name": "Trace_20230812_17_00_14_GABA_BL_20247", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 944, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T17:00:41;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6242800, + "path": "original/data/Trace_20230812_17_00_14_GABA_BL_20247.raw.h5", + "timestamp": "2023-08-12 T17:00:41;" + } + ] + }, + "Trace_20230812_15_55_43_GABA_BL_20402": { + "name": "Trace_20230812_15_55_43_GABA_BL_20402", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 971, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:58:02;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 6108800, + "path": "original/data/Trace_20230812_15_55_43_GABA_BL_20402.raw.h5", + "timestamp": "2023-08-12 T15:58:02;" + } + ] + }, + "Trace_20230812_16_03_20_GABA_DR2_20402": { + "name": "Trace_20230812_16_03_20_GABA_DR2_20402", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 971, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T16:04:39;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16962400, + "path": "original/data/Trace_20230812_16_03_20_GABA_DR2_20402.raw.h5", + "timestamp": "2023-08-12 T16:04:39;" + } + ] + }, + "Trace_20230812_15_18_49_GABA_DR_20325": { + "name": "Trace_20230812_15_18_49_GABA_DR_20325", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 819, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T15:19:25;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16888000, + "path": "original/data/Trace_20230812_15_18_49_GABA_DR_20325.raw.h5", + "timestamp": "2023-08-12 T15:19:25;" + } + ] + }, + "Trace_20230812_17_27_36_GABA_DR_20247": { + "name": "Trace_20230812_17_27_36_GABA_DR_20247", + "hardware": "Maxwell", + "maxwell_chip_number": "N/A", + "maxwell_tracker_sr_no": "xxxxxx", + "channels": [], + "notes": "", + "num_channels": 944, + "num_current_input_channels": 0, + "num_voltage_channels": 32, + "offset": 0, + "sample_rate": 20000, + "voltage_scaling_factor": 1, + "timestamp": "2023-08-12 T17:27:59;", + "units": "\u00b5V", + "version": 20190530, + "blocks": [ + { + "num_frames": 16890400, + "path": "original/data/Trace_20230812_17_27_36_GABA_DR_20247.raw.h5", + "timestamp": "2023-08-12 T17:27:59;" + } + ] + } + } +} From 42caa515b19bbf9ffa69aa8185213dd363f929bb Mon Sep 17 00:00:00 2001 From: DailyDreaming Date: Thu, 14 Dec 2023 13:26:20 -0800 Subject: [PATCH 8/9] Update. --- src/braingeneers/utils/common_utils_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/braingeneers/utils/common_utils_test.py b/src/braingeneers/utils/common_utils_test.py index 3876fbe..bc33ca4 100644 --- a/src/braingeneers/utils/common_utils_test.py +++ b/src/braingeneers/utils/common_utils_test.py @@ -3,7 +3,6 @@ import common_utils # Updated import statement import os import tempfile -from pathlib import Path class TestFileListFunction(unittest.TestCase): @@ -32,9 +31,9 @@ def test_s3_no_files(self, mock_s3_client): def test_local_files_exist(self): with tempfile.TemporaryDirectory() as temp_dir: - # Create temporary files - Path(temp_dir, 'tempfile1.txt').touch() - Path(temp_dir, 'tempfile2.txt').touch() + for f in ['tempfile1.txt', 'tempfile2.txt']: + with open(os.path.join(temp_dir, f), 'w') as w: + w.write('nothing') result = common_utils.file_list(temp_dir) # Updated to common_utils # The result should contain two files with their details From 79a43c68740ec8ba379b4ecfa16a48d9219b9edf Mon Sep 17 00:00:00 2001 From: DailyDreaming Date: Thu, 14 Dec 2023 13:32:41 -0800 Subject: [PATCH 9/9] Dedent. --- src/braingeneers/utils/common_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/braingeneers/utils/common_utils.py b/src/braingeneers/utils/common_utils.py index 83b2d87..53eb2bb 100644 --- a/src/braingeneers/utils/common_utils.py +++ b/src/braingeneers/utils/common_utils.py @@ -115,10 +115,9 @@ def file_list(filepath: str) -> List[Tuple[str, str, int]]: (f['Key'].split('/')[-1], str(f['LastModified']), int(f['Size'])) for f in sorted(response['Contents'], key=lambda x: x['LastModified'], reverse=True) ] - else: - if os.path.exists(filepath): - files = sorted(pathlib.Path(filepath).iterdir(), key=os.path.getmtime, reverse=True) - files_and_details = [(f.name, str(f.stat().st_mtime), f.stat().st_size) for f in files] + elif os.path.exists(filepath): + files = sorted(pathlib.Path(filepath).iterdir(), key=os.path.getmtime, reverse=True) + files_and_details = [(f.name, str(f.stat().st_mtime), f.stat().st_size) for f in files] return files_and_details