diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d011da68..f18ad887a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Spikesorting - Update calls in v0 pipeline for spikeinterface>=0.99 #893 - Fix method type of `get_spike_times` #904 + - Add helper functions for restricting spikesorting results and linking to probe info #910 - Decoding - Handle dimensions of clusterless `get_ahead_behind_distance` #904 diff --git a/notebooks/10_Spike_SortingV1.ipynb b/notebooks/10_Spike_SortingV1.ipynb index e650f2cbc..e4bbbeb8e 100644 --- a/notebooks/10_Spike_SortingV1.ipynb +++ b/notebooks/10_Spike_SortingV1.ipynb @@ -31,7 +31,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "5778bf96-740c-4e4b-a695-ed4385fc9b58", "metadata": { "tags": [] @@ -75,8 +75,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-03-22 09:25:28,835][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n", - "[2024-03-22 09:25:28,874][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n" + "[2024-04-19 10:57:17,965][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n", + "[2024-04-19 10:57:17,985][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n" ] } ], @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "8d659323", "metadata": {}, "outputs": [], @@ -273,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "a269f6af-eb16-4551-b511-a264368c9490", "metadata": {}, "outputs": [], @@ -291,10 +291,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "5b307631-3cc5-4859-9e95-aeedf6a3de56", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'nwb_file_name': 'minirec20230622_.nwb',\n", + " 'sort_group_id': 0,\n", + " 'preproc_param_name': 'default',\n", + " 'interval_list_name': '01_s1',\n", + " 'team_name': 'My Team',\n", + " 'recording_id': UUID('3450db49-28d5-4942-aa37-7c19126d16db')}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# define and insert a key for each sort group and interval you want to sort\n", "key = {\n", @@ -317,10 +333,107 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "3840f86a-8769-423e-8aeb-4d9ab694f1ef", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[10:57:43][INFO] Spyglass: Writing new NWB file minirec20230622_PTCFX77XOI.nwb\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/hdmf/build/objectmapper.py:668: MissingRequiredBuildWarning: NWBFile 'root' is missing required value for attribute 'source_script_file_name'.\n", + " warnings.warn(msg, MissingRequiredBuildWarning)\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Processed recording.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "
\n", + "

recording_id

\n", + " \n", + "
\n", + "

analysis_file_name

\n", + " name of the file\n", + "
\n", + "

object_id

\n", + " Object ID for the processed recording in NWB file\n", + "
3450db49-28d5-4942-aa37-7c19126d16dbminirec20230622_PTCFX77XOI.nwb15592178-c317-4112-bfa6-b0943542e507
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*recording_id analysis_file_ object_id \n", + "+------------+ +------------+ +------------+\n", + "3450db49-28d5- minirec2023062 15592178-c317-\n", + " (Total: 1)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Assuming 'key' is a dictionary with fields that you want to include in 'ssr_key'\n", "ssr_key = {\n", @@ -336,7 +449,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "1c6c7ea3-9538-4fa9-890b-ee16cc18af31", "metadata": {}, "outputs": [], @@ -362,10 +475,39 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "74415172-f2da-4fd3-ab43-01857d682b0d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[10:57:52][INFO] Spyglass: Using 4 jobs...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "461383a7fb194d79b603244c4e371a98", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "detect_artifact_frames: 0%| | 0/2 [00:00\n", + " .Table{\n", + " border-collapse:collapse;\n", + " }\n", + " .Table th{\n", + " background: #A0A0A0; color: #ffffff; padding:4px; border:#f0e0e0 1px solid;\n", + " font-weight: normal; font-family: monospace; font-size: 100%;\n", + " }\n", + " .Table td{\n", + " padding:4px; border:#f0e0e0 1px solid; font-size:100%;\n", + " }\n", + " .Table tr:nth-child(odd){\n", + " background: #ffffff;\n", + " color: #000000;\n", + " }\n", + " .Table tr:nth-child(even){\n", + " background: #f3f1ff;\n", + " color: #000000;\n", + " }\n", + " /* Tooltip container */\n", + " .djtooltip {\n", + " }\n", + " /* Tooltip text */\n", + " .djtooltip .djtooltiptext {\n", + " visibility: hidden;\n", + " width: 120px;\n", + " background-color: black;\n", + " color: #fff;\n", + " text-align: center;\n", + " padding: 5px 0;\n", + " border-radius: 6px;\n", + " /* Position the tooltip text - see examples below! */\n", + " position: absolute;\n", + " z-index: 1;\n", + " }\n", + " #primary {\n", + " font-weight: bold;\n", + " color: black;\n", + " }\n", + " #nonprimary {\n", + " font-weight: normal;\n", + " color: white;\n", + " }\n", + "\n", + " /* Show the tooltip text when you mouse over the tooltip container */\n", + " .djtooltip:hover .djtooltiptext {\n", + " visibility: visible;\n", + " }\n", + " \n", + " \n", + " Detected artifacts (e.g. large transients from movement).\n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "

artifact_id

\n", + " \n", + "
0058dab4-41c1-42b1-91f4-5773f2ad36cc
01b39d37-3ff8-4907-9da6-9fec9baf87b5
035f0bae-80b3-4ce9-a767-94d336f36283
038ee778-6cf1-4e99-ab80-e354db5170c9
03e9768d-d101-4f56-abf9-5b0e3e1803b7
0490c820-c381-43b6-857e-f463147723ff
04a289c6-9e19-486a-a4cb-7e9638af225a
06dd7922-7042-4023-bebf-da1dacb0b6c7
07036486-e9f5-4dba-8662-7fb5ff2a6711
070ed448-a52d-478e-9102-0d04a6ed0b96
07a65788-bb89-48f3-90ea-4ab1add06eae
0a6611b3-c593-4900-a715-66bb1396940e
\n", + "

...

\n", + "

Total: 151

\n", + " " + ], + "text/plain": [ + "*artifact_id \n", + "+------------+\n", + "0058dab4-41c1-\n", + "01b39d37-3ff8-\n", + "035f0bae-80b3-\n", + "038ee778-6cf1-\n", + "03e9768d-d101-\n", + "0490c820-c381-\n", + "04a289c6-9e19-\n", + "06dd7922-7042-\n", + "07036486-e9f5-\n", + "070ed448-a52d-\n", + "07a65788-bb89-\n", + "0a6611b3-c593-\n", + " ...\n", + " (Total: 151)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sgs.ArtifactDetection()" ] @@ -419,7 +653,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "34246883-9dc4-43c5-a438-009215a3a35e", "metadata": {}, "outputs": [], @@ -452,67 +686,201 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "68856fb6-b5c2-4ee4-b300-43a117e453a1", "metadata": {}, - "outputs": [], - "source": [ - "sgs.SpikeSortingSelection.insert_selection(key)\n", - "sgs.SpikeSortingSelection() & key" - ] - }, - { - "cell_type": "markdown", - "id": "bb343fb7-04d6-48fc-bf67-9919769a7a52", - "metadata": {}, - "source": [ - "Once `SpikeSortingSelection` is populated, let's run `SpikeSorting.populate`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "54ccf059-b1ae-42e8-aede-4af30a61fd2b", - "metadata": {}, - "outputs": [], - "source": [ - "sss_pk = (sgs.SpikeSortingSelection & key).proj()\n", - "\n", - "sgs.SpikeSorting.populate(sss_pk)" - ] - }, - { - "cell_type": "markdown", - "id": "f3d1e621", - "metadata": {}, - "source": [ - "The spike sorting results (spike times of detected units) are saved in an NWB file. We can access this in two ways. First, we can access it via the `fetch_nwb` method, which allows us to directly access the spike times saved in the `units` table of the NWB file. Second, we can access it as a `spikeinterface.NWBSorting` object. This allows us to take advantage of the rich APIs of `spikeinterface` to further analyze the sorting. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3d41d3ab", - "metadata": {}, - "outputs": [], - "source": [ - "sorting_nwb = (sgs.SpikeSorting & key).fetch_nwb()\n", - "sorting_si = sgs.SpikeSorting.get_sorting(key)" - ] - }, - { - "cell_type": "markdown", - "id": "db328eb1", - "metadata": {}, - "source": [ - "Note that the spike times of `fetch_nwb` is in units of seconds aligned with the timestamps of the recording. The spike times of the `spikeinterface.NWBSorting` object is in units of samples (as is generally true for sorting objects in `spikeinterface`)." - ] - }, - { - "cell_type": "markdown", - "id": "55d6c183", - "metadata": {}, - "source": [ + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Processed recording and spike sorting parameters. Use `insert_selection` method to insert rows.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

recording_id

\n", + " \n", + "
\n", + "

sorter

\n", + " \n", + "
\n", + "

sorter_param_name

\n", + " \n", + "
\n", + "

nwb_file_name

\n", + " name of the NWB file\n", + "
\n", + "

interval_list_name

\n", + " descriptive name of this interval list\n", + "
16cbb873-052f-44f3-9f4d-89af3544915e3450db49-28d5-4942-aa37-7c19126d16dbmountainsort4franklab_tetrode_hippocampus_30KHzminirec20230622_.nwbf03513af-bff8-4732-a6ab-e53f0550e7b0
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*sorting_id recording_id sorter sorter_param_n nwb_file_name interval_list_\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "16cbb873-052f- 3450db49-28d5- mountainsort4 franklab_tetro minirec2023062 f03513af-bff8-\n", + " (Total: 1)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sgs.SpikeSortingSelection.insert_selection(key)\n", + "sgs.SpikeSortingSelection() & key" + ] + }, + { + "cell_type": "markdown", + "id": "bb343fb7-04d6-48fc-bf67-9919769a7a52", + "metadata": {}, + "source": [ + "Once `SpikeSortingSelection` is populated, let's run `SpikeSorting.populate`. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "54ccf059-b1ae-42e8-aede-4af30a61fd2b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mountainsort4 use the OLD spikeextractors mapped with NewToOldRecording\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[10:58:17][INFO] Spyglass: Writing new NWB file minirec20230622_PP6Y10VW0V.nwb\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/hdmf/build/objectmapper.py:668: MissingRequiredBuildWarning: NWBFile 'root' is missing required value for attribute 'source_script_file_name'.\n", + " warnings.warn(msg, MissingRequiredBuildWarning)\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/hash.py:39: ResourceWarning: unclosed file <_io.BufferedReader name='/stelmo/nwb/analysis/minirec20230622/minirec20230622_PP6Y10VW0V.nwb'>\n", + " return uuid_from_stream(Path(filepath).open(\"rb\"), init_string=init_string)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/external.py:276: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n", + " if check_hash:\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/tempfile.py:821: ResourceWarning: Implicitly cleaning up \n", + " _warnings.warn(warn_message, ResourceWarning)\n" + ] + } + ], + "source": [ + "sss_pk = (sgs.SpikeSortingSelection & key).proj()\n", + "\n", + "sgs.SpikeSorting.populate(sss_pk)" + ] + }, + { + "cell_type": "markdown", + "id": "f3d1e621", + "metadata": {}, + "source": [ + "The spike sorting results (spike times of detected units) are saved in an NWB file. We can access this in two ways. First, we can access it via the `fetch_nwb` method, which allows us to directly access the spike times saved in the `units` table of the NWB file. Second, we can access it as a `spikeinterface.NWBSorting` object. This allows us to take advantage of the rich APIs of `spikeinterface` to further analyze the sorting. " + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "3d41d3ab", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/hash.py:39: ResourceWarning: unclosed file <_io.BufferedReader name='/stelmo/nwb/analysis/minirec20230622/minirec20230622_PP6Y10VW0V.nwb'>\n", + " return uuid_from_stream(Path(filepath).open(\"rb\"), init_string=init_string)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ], + "source": [ + "sorting_nwb = (sgs.SpikeSorting & key).fetch_nwb()\n", + "sorting_si = sgs.SpikeSorting.get_sorting(key)" + ] + }, + { + "cell_type": "markdown", + "id": "db328eb1", + "metadata": {}, + "source": [ + "Note that the spike times of `fetch_nwb` is in units of seconds aligned with the timestamps of the recording. The spike times of the `spikeinterface.NWBSorting` object is in units of samples (as is generally true for sorting objects in `spikeinterface`)." + ] + }, + { + "cell_type": "markdown", + "id": "55d6c183", + "metadata": {}, + "source": [ "## Automatic Curation" ] }, @@ -529,10 +897,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "6245eec9-3fba-4071-b58b-eec6d9345532", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[10:58:32][INFO] Spyglass: Writing new NWB file minirec20230622_SYPH1SYT75.nwb\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/hdmf/build/objectmapper.py:668: MissingRequiredBuildWarning: NWBFile 'root' is missing required value for attribute 'source_script_file_name'.\n", + " warnings.warn(msg, MissingRequiredBuildWarning)\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/hash.py:39: ResourceWarning: unclosed file <_io.BufferedReader name='/stelmo/nwb/analysis/minirec20230622/minirec20230622_SYPH1SYT75.nwb'>\n", + " return uuid_from_stream(Path(filepath).open(\"rb\"), init_string=init_string)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/external.py:276: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n", + " if check_hash:\n" + ] + }, + { + "data": { + "text/plain": [ + "{'sorting_id': UUID('16cbb873-052f-44f3-9f4d-89af3544915e'),\n", + " 'curation_id': 0,\n", + " 'parent_curation_id': -1,\n", + " 'analysis_file_name': 'minirec20230622_SYPH1SYT75.nwb',\n", + " 'object_id': '3e4f927b-716f-4dd8-9c98-acd132d758fb',\n", + " 'merges_applied': False,\n", + " 'description': 'testing sort'}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sgs.SpikeSortingRecording & key\n", "sgs.CurationV1.insert_curation(\n", @@ -545,10 +944,192 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "5bec5b97-4e9f-4ee9-a6b5-4f05f4726744", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Curation of a SpikeSorting. Use `insert_curation` to insert rows.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

curation_id

\n", + " \n", + "
\n", + "

parent_curation_id

\n", + " \n", + "
\n", + "

analysis_file_name

\n", + " name of the file\n", + "
\n", + "

object_id

\n", + " \n", + "
\n", + "

merges_applied

\n", + " \n", + "
\n", + "

description

\n", + " \n", + "
021fb85a-992f-4360-99c7-e2da32c5b9cb0-1BS2820231107_8Z8CLG184Z.nwb37ee7365-028f-46e1-8351-1cd402a7b36c0testing sort
021fb85a-992f-4360-99c7-e2da32c5b9cb10BS2820231107_HPIQR9LZWU.nwb538032a5-5d29-4cb8-b0a2-7224fee6d8ce0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb20BS2820231107_SVW8YK84IP.nwbed440315-7302-4217-be15-087c7efeda7e0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb30BS2820231107_7CWR2JR68B.nwb0d8be667-2831-4e99-8c9b-54102de48e850after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb40BS2820231107_1PCRTB2UZ2.nwb9f9e9a1e-9be3-405c-9c66-4bf6dc54d4d90after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb50BS2820231107_4NPZ4YTASV.nwb89170a28-487a-4787-83dd-18009c4467000after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb60BS2820231107_MMSIJ8YQ54.nwbc9fb8c88-6449-4d9a-a40a-cd10dcdc193f0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb70BS2820231107_LZJWQPP1YW.nwbf078e3bb-92fc-4e7f-b3a8-32936a90e0570after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb80BS2820231107_RJ7DLUKOIG.nwbc311fbfb-cd3d-4d92-b535-b5da3d4a6ec30after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb90BS2820231107_6ZJP5NRCX9.nwba54ee3f8-851a-4dca-bb46-7673e28074620after metric curation
03dc29a5-febe-4a59-ab61-21a25dea36250-1j1620210710_EOE1VZ4YAX.nwb52889e86-c249-4916-9576-a9ccf7f48dbe0
061ba57b-d2cb-4052-b375-42ba13684e410-1BS2820231107_S21IIVRCZA.nwb5d71500d-1065-4610-b3a6-746821d0f4380testing sort
\n", + "

...

\n", + "

Total: 626

\n", + " " + ], + "text/plain": [ + "*sorting_id *curation_id parent_curatio analysis_file_ object_id merges_applied description \n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "021fb85a-992f- 0 -1 BS2820231107_8 37ee7365-028f- 0 testing sort \n", + "021fb85a-992f- 1 0 BS2820231107_H 538032a5-5d29- 0 after metric c\n", + "021fb85a-992f- 2 0 BS2820231107_S ed440315-7302- 0 after metric c\n", + "021fb85a-992f- 3 0 BS2820231107_7 0d8be667-2831- 0 after metric c\n", + "021fb85a-992f- 4 0 BS2820231107_1 9f9e9a1e-9be3- 0 after metric c\n", + "021fb85a-992f- 5 0 BS2820231107_4 89170a28-487a- 0 after metric c\n", + "021fb85a-992f- 6 0 BS2820231107_M c9fb8c88-6449- 0 after metric c\n", + "021fb85a-992f- 7 0 BS2820231107_L f078e3bb-92fc- 0 after metric c\n", + "021fb85a-992f- 8 0 BS2820231107_R c311fbfb-cd3d- 0 after metric c\n", + "021fb85a-992f- 9 0 BS2820231107_6 a54ee3f8-851a- 0 after metric c\n", + "03dc29a5-febe- 0 -1 j1620210710_EO 52889e86-c249- 0 \n", + "061ba57b-d2cb- 0 -1 BS2820231107_S 5d71500d-1065- 0 testing sort \n", + " ...\n", + " (Total: 626)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sgs.CurationV1()" ] @@ -563,7 +1144,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "7207abda-ea84-43af-97d4-e5be3464d28d", "metadata": {}, "outputs": [], @@ -581,10 +1162,110 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "14c2eacc-cc45-4e61-9919-04785a721079", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Spike sorting and parameters for metric curation. Use `insert_selection` to insert a row into this table.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

metric_curation_id

\n", + " \n", + "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

curation_id

\n", + " \n", + "
\n", + "

waveform_param_name

\n", + " name of waveform extraction parameters\n", + "
\n", + "

metric_param_name

\n", + " \n", + "
\n", + "

metric_curation_param_name

\n", + " \n", + "
5bd75cd5-cc2e-41dd-9056-5d62fa46021a16cbb873-052f-44f3-9f4d-89af3544915e0default_not_whitenedfranklab_defaultdefault
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*metric_curati sorting_id curation_id waveform_param metric_param_n metric_curatio\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "5bd75cd5-cc2e- 16cbb873-052f- 0 default_not_wh franklab_defau default \n", + " (Total: 1)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sgs.MetricCurationSelection.insert_selection(key)\n", "sgs.MetricCurationSelection() & key" @@ -592,12 +1273,100 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "id": "d22f5725-4fd1-42ea-a1d4-590bd1353d46", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Results of applying curation based on quality metrics. To do additional curation, insert another row in `CurationV1`\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "
\n", + "

metric_curation_id

\n", + " \n", + "
\n", + "

analysis_file_name

\n", + " name of the file\n", + "
\n", + "

object_id

\n", + " Object ID for the metrics in NWB file\n", + "
5bd75cd5-cc2e-41dd-9056-5d62fa46021aminirec20230622_PVSMM7XHHJ.nwb01b58a59-1b49-4bd1-a204-16fb09d67b2a
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*metric_curati analysis_file_ object_id \n", + "+------------+ +------------+ +------------+\n", + "5bd75cd5-cc2e- minirec2023062 01b58a59-1b49-\n", + " (Total: 1)" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "sgs.MetricCuration.populate()\n", + "sgs.MetricCuration.populate(key)\n", "sgs.MetricCuration() & key" ] }, @@ -611,10 +1380,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "id": "544ba8c0-560e-471b-9eaf-5924f6051faa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[11:08:29][INFO] Spyglass: Writing new NWB file minirec20230622_ZCMODPF1NM.nwb\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/hdmf/build/objectmapper.py:668: MissingRequiredBuildWarning: NWBFile 'root' is missing required value for attribute 'source_script_file_name'.\n", + " warnings.warn(msg, MissingRequiredBuildWarning)\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/hash.py:39: ResourceWarning: unclosed file <_io.BufferedReader name='/stelmo/nwb/analysis/minirec20230622/minirec20230622_ZCMODPF1NM.nwb'>\n", + " return uuid_from_stream(Path(filepath).open(\"rb\"), init_string=init_string)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/home/sambray/mambaforge-pypy3/envs/spyglass/lib/python3.9/site-packages/datajoint/external.py:276: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n", + " if check_hash:\n" + ] + }, + { + "data": { + "text/plain": [ + "{'sorting_id': UUID('16cbb873-052f-44f3-9f4d-89af3544915e'),\n", + " 'curation_id': 1,\n", + " 'parent_curation_id': 0,\n", + " 'analysis_file_name': 'minirec20230622_ZCMODPF1NM.nwb',\n", + " 'object_id': 'c43cd7ab-e5bd-4528-a0e5-0ca7c337a72d',\n", + " 'merges_applied': False,\n", + " 'description': 'after metric curation'}" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "key = {\n", " \"metric_curation_id\": (\n", @@ -639,10 +1439,192 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "id": "f7c6bfd9-5985-41e1-bf37-8c8874b59191", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Curation of a SpikeSorting. Use `insert_curation` to insert rows.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

curation_id

\n", + " \n", + "
\n", + "

parent_curation_id

\n", + " \n", + "
\n", + "

analysis_file_name

\n", + " name of the file\n", + "
\n", + "

object_id

\n", + " \n", + "
\n", + "

merges_applied

\n", + " \n", + "
\n", + "

description

\n", + " \n", + "
021fb85a-992f-4360-99c7-e2da32c5b9cb0-1BS2820231107_8Z8CLG184Z.nwb37ee7365-028f-46e1-8351-1cd402a7b36c0testing sort
021fb85a-992f-4360-99c7-e2da32c5b9cb10BS2820231107_HPIQR9LZWU.nwb538032a5-5d29-4cb8-b0a2-7224fee6d8ce0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb20BS2820231107_SVW8YK84IP.nwbed440315-7302-4217-be15-087c7efeda7e0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb30BS2820231107_7CWR2JR68B.nwb0d8be667-2831-4e99-8c9b-54102de48e850after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb40BS2820231107_1PCRTB2UZ2.nwb9f9e9a1e-9be3-405c-9c66-4bf6dc54d4d90after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb50BS2820231107_4NPZ4YTASV.nwb89170a28-487a-4787-83dd-18009c4467000after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb60BS2820231107_MMSIJ8YQ54.nwbc9fb8c88-6449-4d9a-a40a-cd10dcdc193f0after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb70BS2820231107_LZJWQPP1YW.nwbf078e3bb-92fc-4e7f-b3a8-32936a90e0570after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb80BS2820231107_RJ7DLUKOIG.nwbc311fbfb-cd3d-4d92-b535-b5da3d4a6ec30after metric curation
021fb85a-992f-4360-99c7-e2da32c5b9cb90BS2820231107_6ZJP5NRCX9.nwba54ee3f8-851a-4dca-bb46-7673e28074620after metric curation
03dc29a5-febe-4a59-ab61-21a25dea36250-1j1620210710_EOE1VZ4YAX.nwb52889e86-c249-4916-9576-a9ccf7f48dbe0
061ba57b-d2cb-4052-b375-42ba13684e410-1BS2820231107_S21IIVRCZA.nwb5d71500d-1065-4610-b3a6-746821d0f4380testing sort
\n", + "

...

\n", + "

Total: 627

\n", + " " + ], + "text/plain": [ + "*sorting_id *curation_id parent_curatio analysis_file_ object_id merges_applied description \n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "021fb85a-992f- 0 -1 BS2820231107_8 37ee7365-028f- 0 testing sort \n", + "021fb85a-992f- 1 0 BS2820231107_H 538032a5-5d29- 0 after metric c\n", + "021fb85a-992f- 2 0 BS2820231107_S ed440315-7302- 0 after metric c\n", + "021fb85a-992f- 3 0 BS2820231107_7 0d8be667-2831- 0 after metric c\n", + "021fb85a-992f- 4 0 BS2820231107_1 9f9e9a1e-9be3- 0 after metric c\n", + "021fb85a-992f- 5 0 BS2820231107_4 89170a28-487a- 0 after metric c\n", + "021fb85a-992f- 6 0 BS2820231107_M c9fb8c88-6449- 0 after metric c\n", + "021fb85a-992f- 7 0 BS2820231107_L f078e3bb-92fc- 0 after metric c\n", + "021fb85a-992f- 8 0 BS2820231107_R c311fbfb-cd3d- 0 after metric c\n", + "021fb85a-992f- 9 0 BS2820231107_6 a54ee3f8-851a- 0 after metric c\n", + "03dc29a5-febe- 0 -1 j1620210710_EO 52889e86-c249- 0 \n", + "061ba57b-d2cb- 0 -1 BS2820231107_S 5d71500d-1065- 0 testing sort \n", + " ...\n", + " (Total: 627)" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sgs.CurationV1()" ] @@ -652,7 +1634,7 @@ "id": "a627274b", "metadata": {}, "source": [ - "## Manual Curation" + "## Manual Curation (Optional)" ] }, { @@ -797,15 +1779,124 @@ "id": "9ff6aff5-7020-40d6-832f-006d66d54a7e", "metadata": {}, "source": [ - "We now insert the curated spike sorting to a `Merge` table for feeding into downstream processing pipelines.\n" + "## Downstream usage (Merge table)\n", + "\n", + "Regardless of Curation method used, to make use of spikeorting results in downstream pipelines like Decoding, we will need to insert it into the `SpikeSortingOutput` merge table. " ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "id": "511ecb19-7d8d-4db6-be71-c0ed66e2b0f2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " Output of spike sorting pipelines.\n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

merge_id

\n", + " \n", + "
\n", + "

source

\n", + " \n", + "
0001a1ab-7c2b-1085-2062-53c0338ffe22CuratedSpikeSorting
000c5d0b-1c4c-55d1-ccf6-5808f57152d3CuratedSpikeSorting
0015e01d-0dc0-ca2c-1f5c-2178fa2c7f1eCuratedSpikeSorting
001628b1-0af1-7c74-a211-0e5c158ba10fCuratedSpikeSorting
001783f0-c5da-98c2-5b2a-63f1334c0a43CuratedSpikeSorting
0020b039-6a2d-1d68-6585-4866fb7ea266CuratedSpikeSorting
002be77b-38a6-fff8-cb48-a81e20ccb51bCuratedSpikeSorting
002da11c-2d16-a6dc-0468-980674ca12b0CuratedSpikeSorting
003bf29a-fa09-05be-5cac-b7ea70a48c0cCuratedSpikeSorting
003cabf2-c471-972a-4b18-63d4ab7e1b8bCuratedSpikeSorting
004d99c6-1b2e-1696-fc85-e78ac5cc7e6bCuratedSpikeSorting
004faf9a-72cb-4416-ae13-3f85d538604fCuratedSpikeSorting
\n", + "

...

\n", + "

Total: 8684

\n", + " " + ], + "text/plain": [ + "*merge_id source \n", + "+------------+ +------------+\n", + "0001a1ab-7c2b- CuratedSpikeSo\n", + "000c5d0b-1c4c- CuratedSpikeSo\n", + "0015e01d-0dc0- CuratedSpikeSo\n", + "001628b1-0af1- CuratedSpikeSo\n", + "001783f0-c5da- CuratedSpikeSo\n", + "0020b039-6a2d- CuratedSpikeSo\n", + "002be77b-38a6- CuratedSpikeSo\n", + "002da11c-2d16- CuratedSpikeSo\n", + "003bf29a-fa09- CuratedSpikeSo\n", + "003cabf2-c471- CuratedSpikeSo\n", + "004d99c6-1b2e- CuratedSpikeSo\n", + "004faf9a-72cb- CuratedSpikeSo\n", + " ...\n", + " (Total: 8684)" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput\n", "\n", @@ -814,23 +1905,313 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "id": "b20c2c9e-0c97-4669-b45d-4b1c50fd2fcc", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "*merge_id *source *sorting_id *curation_id *nwb_file_name *sort_group_id *sort_interval *preproc_param *team_name *sorter *sorter_params *artifact_remo\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +-----------+ +--------+ +------------+ +------------+\n", + "d76584f8-0969- CurationV1 03dc29a5-febe- 0 None 0 None None None None None None \n", + "33d71671-63e5- CurationV1 090377fb-72b7- 0 None 0 None None None None None None \n", + "dfa87e8e-c5cf- CurationV1 0cf93833-6a14- 0 None 0 None None None None None None \n", + "a6cc0a23-7e29- CurationV1 110e27f6-5ffa- 0 None 0 None None None None None None \n", + "7f8841a6-5e27- CurationV1 16cbb873-052f- 1 None 0 None None None None None None \n", + "91e8e8d8-1568- CurationV1 21bea0ea-3084- 0 None 0 None None None None None None \n", + "218c17c7-8a4c- CurationV1 21bea0ea-3084- 1 None 0 None None None None None None \n", + "25823222-85ed- CurationV1 2484ee5d-0819- 0 None 0 None None None None None None \n", + "5ae79d97-6a99- CurationV1 3046a016-1613- 0 None 0 None None None None None None \n", + "869072e1-76d6- CurationV1 41a13836-e128- 0 None 0 None None None None None None \n", + "a0771d6c-fc9d- CurationV1 4bc61e94-5bf9- 0 None 0 None None None None None None \n", + "ed70dacb-a637- CurationV1 5d15f94e-d53d- 0 None 0 None None None None None None \n", + " ...\n", + " (Total: 0)\n", + "\n" + ] + } + ], "source": [ - "SpikeSortingOutput.insert([key], part_name=\"CurationV1\")\n", + "# insert the automatic curation spikesorting results\n", + "curation_key = sss_pk.fetch1(\"KEY\")\n", + "curation_key[\"curation_id\"] = 1\n", + "merge_insert_key = (sgs.CurationV1 & curation_key).fetch(\"KEY\", as_dict=True)\n", + "SpikeSortingOutput.insert(merge_insert_key, part_name=\"CurationV1\")\n", "SpikeSortingOutput.merge_view()" ] }, + { + "cell_type": "markdown", + "id": "a8ab3ed2", + "metadata": {}, + "source": [ + "Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's" + ] + }, { "cell_type": "code", - "execution_count": null, - "id": "184c3401-8df3-46f0-9dd0-c9fa98395c34", + "execution_count": 6, + "id": "3925b5de", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[13:34:12][WARNING] Spyglass: V0 requires artifact restrict. Ignoring \"restrict_by_artifact\" flag.\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'merge_id': UUID('74c006e8-dcfe-e994-7b40-73f8d9f75b85')}]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "selection_key = {\n", + " \"nwb_file_name\": nwb_file_name2,\n", + " \"sorter\": \"mountainsort4\",\n", + " \"interval_list_name\": \"01_s1\",\n", + " \"curation_id\": 0,\n", + "} # this function can use restrictions from throughout the spikesorting pipeline\n", + "spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids(\n", + " selection_key, as_dict=True\n", + ")\n", + "spikesorting_merge_ids" + ] + }, + { + "cell_type": "markdown", + "id": "007fbb60", + "metadata": {}, + "source": [ + "With the spikesorting merge_ids we want we can also use the method `get_sort_group_info` to get a table linking the merge id to the electrode group it is sourced from. This can be helpful for restricting to just electrodes from a brain area of interest" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "696345db", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

merge_id

\n", + " \n", + "
\n", + "

nwb_file_name

\n", + " name of the NWB file\n", + "
\n", + "

electrode_group_name

\n", + " electrode group name from NWBFile\n", + "
\n", + "

electrode_id

\n", + " the unique number for this electrode\n", + "
\n", + "

curation_id

\n", + " a number correponding to the index of this curation\n", + "
\n", + "

sort_group_id

\n", + " identifier for a group of electrodes\n", + "
\n", + "

sort_interval_name

\n", + " name for this interval\n", + "
\n", + "

preproc_params_name

\n", + " \n", + "
\n", + "

team_name

\n", + " \n", + "
\n", + "

sorter

\n", + " \n", + "
\n", + "

sorter_params_name

\n", + " \n", + "
\n", + "

artifact_removed_interval_list_name

\n", + " \n", + "
\n", + "

region_id

\n", + " \n", + "
\n", + "

probe_id

\n", + " \n", + "
\n", + "

probe_shank

\n", + " shank number within probe\n", + "
\n", + "

probe_electrode

\n", + " electrode\n", + "
\n", + "

name

\n", + " unique label for each contact\n", + "
\n", + "

original_reference_electrode

\n", + " the configured reference electrode for this electrode\n", + "
\n", + "

x

\n", + " the x coordinate of the electrode position in the brain\n", + "
\n", + "

y

\n", + " the y coordinate of the electrode position in the brain\n", + "
\n", + "

z

\n", + " the z coordinate of the electrode position in the brain\n", + "
\n", + "

filtering

\n", + " description of the signal filtering\n", + "
\n", + "

impedance

\n", + " electrode impedance\n", + "
\n", + "

bad_channel

\n", + " if electrode is \"good\" or \"bad\" as observed during recording\n", + "
\n", + "

x_warped

\n", + " x coordinate of electrode position warped to common template brain\n", + "
\n", + "

y_warped

\n", + " y coordinate of electrode position warped to common template brain\n", + "
\n", + "

z_warped

\n", + " z coordinate of electrode position warped to common template brain\n", + "
\n", + "

contacts

\n", + " label of electrode contacts used for a bipolar signal - current workaround\n", + "
\n", + "

analysis_file_name

\n", + " name of the file\n", + "
\n", + "

units_object_id

\n", + " \n", + "
\n", + "

region_name

\n", + " the name of the brain region\n", + "
\n", + "

subregion_name

\n", + " subregion name\n", + "
\n", + "

subsubregion_name

\n", + " subregion within subregion\n", + "
662f3e35-c81e-546c-69c3-b3a2f5ed2776minirec20230622_.nwb001001_s1_first9default_hippocampusMy Teammountainsort4hippocampus_tutorialminirec20230622_.nwb_01_s1_first9_0_default_hippocampus_none_artifact_removed_valid_times35tetrode_12.500000.00.00.0None0.0False0.00.00.0minirec20230622_RXRSAFCGVJ.nwbcorpus callosum and associated subcortical white matter (cc-ec-cing-dwm)NoneNone
\n", + " \n", + "

Total: 1

\n", + " " + ], + "text/plain": [ + "*merge_id *nwb_file_name *electrode_gro *electrode_id *curation_id *sort_group_id *sort_interval *preproc_param *team_name *sorter *sorter_params *artifact_remo *region_id probe_id probe_shank probe_electrod name original_refer x y z filtering impedance bad_channel x_warped y_warped z_warped contacts analysis_file_ units_object_i region_name subregion_name subsubregion_n\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +-----------+ +------------+ +------------+ +------------+ +-----------+ +------------+ +------------+ +------------+ +------+ +------------+ +-----+ +-----+ +-----+ +-----------+ +-----------+ +------------+ +----------+ +----------+ +----------+ +----------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "662f3e35-c81e- minirec2023062 0 0 1 0 01_s1_first9 default_hippoc My Team mountainsort4 hippocampus_tu minirec2023062 35 tetrode_12.5 0 0 0 0 0.0 0.0 0.0 None 0.0 False 0.0 0.0 0.0 minirec2023062 corpus callosu None None \n", + " (Total: 1)" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "SpikeSortingOutput.CurationV1()" + "merge_keys = [{\"merge_id\": str(id)} for id in spikesorting_merge_ids]\n", + "SpikeSortingOutput().get_sort_group_info(merge_keys)" ] } ], diff --git a/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb b/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb index 820da6496..07b3130a5 100644 --- a/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb +++ b/notebooks/40_Extracting_Clusterless_Waveform_Features.ipynb @@ -57,9 +57,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-01-17 22:14:51,194][INFO]: Connecting root@localhost:3306\n", - "[2024-01-17 22:14:51,274][INFO]: Connected root@localhost:3306\n", - "/Users/edeno/Documents/GitHub/spyglass/src/spyglass/data_import/insert_sessions.py:58: UserWarning: Cannot insert data from mediumnwb20230802.nwb: mediumnwb20230802_.nwb is already in Nwbfile table.\n", + "[2024-04-19 10:37:45,302][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n", + "[2024-04-19 10:37:45,330][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n", + "/home/sambray/Documents/spyglass/src/spyglass/data_import/insert_sessions.py:58: UserWarning: Cannot insert data from mediumnwb20230802.nwb: mediumnwb20230802_.nwb is already in Nwbfile table.\n", " warnings.warn(\n" ] } @@ -111,30 +111,30 @@ "name": "stderr", "output_type": "stream", "text": [ - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:55][WARNING] Spyglass: Similar row(s) already inserted.\n" + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:53][WARNING] Spyglass: Similar row(s) already inserted.\n" ] } ], @@ -178,30 +178,39 @@ "name": "stderr", "output_type": "stream", "text": [ - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][WARNING] Spyglass: Similar row(s) already inserted.\n" + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n", + "[10:37:56][WARNING] Spyglass: Similar row(s) already inserted.\n" ] } ], @@ -234,34 +243,72 @@ "execution_count": 5, "metadata": {}, "outputs": [ + { + "data": { + "text/plain": [ + "{'sorter': 'clusterless_thresholder',\n", + " 'sorter_param_name': 'default_clusterless',\n", + " 'sorter_params': {'detect_threshold': 100.0,\n", + " 'method': 'locally_exclusive',\n", + " 'peak_sign': 'neg',\n", + " 'exclude_sweep_ms': 0.1,\n", + " 'local_radius_um': 100,\n", + " 'noise_levels': array([1.]),\n", + " 'random_chunk_kwargs': {},\n", + " 'outputs': 'sorting'}}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "(sgs.SpikeSorterParameters() & {\"sorter\": \"clusterless_thresholder\"}).fetch1()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n", - "[22:14:56][INFO] Spyglass: Similar row(s) already inserted.\n" + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:38][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n", + "[12:03:39][INFO] Spyglass: Similar row(s) already inserted.\n" ] } ], @@ -275,14 +322,15 @@ " \"nwb_file_name\": nwb_copy_file_name,\n", " \"interval_list_name\": str(\n", " (\n", - " sgs.ArtifactDetectionSelection & {\"recording_id\": recording_id}\n", + " sgs.ArtifactDetectionSelection\n", + " & {\"recording_id\": recording_id, \"artifact_param_name\": \"none\"}\n", " ).fetch1(\"artifact_id\")\n", " ),\n", " }\n", " group_keys.append(key)\n", " sgs.SpikeSortingSelection.insert_selection(key)\n", - "\n", - "sgs.SpikeSorting.populate(group_keys)" + "sort_keys = (sgs.SpikeSortingSelection & group_keys).fetch(\"KEY\")\n", + "sgs.SpikeSorting.populate(sort_keys)" ] }, { @@ -294,14 +342,63 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b9e53383606f4cbebdb5f8ccf1b56878", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "detect peaks using locally_exclusive: 0%| | 0/1476 [00:00features_param_name

\n", " a name for this set of parameters\n", " \n", - " 0751a1e1-a406-7f87-ae6f-ce4ffc60621c\n", - "amplitude485a4ddf-332d-35b5-3ad4-0561736c1844\n", - "amplitude4a712103-c223-864f-82e0-6c23de79cc14\n", - "amplitude4a72c253-b3ca-8c13-e615-736a7ebff35c\n", - "amplitude5c53bd33-d57c-fbba-e0fb-55e0bcb85d03\n", - "amplitude614d796c-0b95-6364-aaa0-b6cb1e7bbb83\n", - "amplitude6acb99b8-6a0c-eb83-1141-5f603c5895e0\n", - "amplitude6d039a63-17ad-0b78-4b1e-f02d5f3dbbc5\n", - "amplitude74e10781-1228-4075-0870-af224024ffdc\n", - "amplitude7e3fa66e-727e-1541-819a-b01309bb30ae\n", - "amplitude86897349-ff68-ac72-02eb-739dd88936e6\n", - "amplitude8bbddc0f-d6ae-6260-9400-f884a6e25ae8\n", + " 003bf29a-fa09-05be-5cac-b7ea70a48c0c\n", + "amplitude004faf9a-72cb-4416-ae13-3f85d538604f\n", + "amplitude0061a9df-3954-99d4-d738-fd13ab7119fe\n", + "amplitude00775472-67a6-5836-b68a-d15186ae3b3c\n", + "amplitude00a1861f-bbf0-5e78-3dc6-36551b2657b0\n", + "amplitude00a9f0d0-b682-2b12-6a2b-08e4129291ce\n", + "amplitude00bd2bbf-ccdb-7be3-f1a0-5e337d87a5a4\n", + "amplitude00bdda4f-7059-6c72-c571-a80ad323fda2\n", + "amplitude00db0baa-4ec0-3d20-897a-ea4a067ebbba\n", + "amplitude00e5a16b-c4f2-e8dc-3083-17f542dadc36\n", + "amplitude00f25c1b-d5a3-6ca6-c501-ef0e544f6284\n", + "amplitude012fdb25-bd7e-aedd-c41b-bb7e177ceeb8\n", "amplitude \n", " \n", "

...

\n", - "

Total: 23

\n", + "

Total: 3504

\n", " " ], "text/plain": [ "*spikesorting_ *features_para\n", "+------------+ +------------+\n", - "0751a1e1-a406- amplitude \n", - "485a4ddf-332d- amplitude \n", - "4a712103-c223- amplitude \n", - "4a72c253-b3ca- amplitude \n", - "5c53bd33-d57c- amplitude \n", - "614d796c-0b95- amplitude \n", - "6acb99b8-6a0c- amplitude \n", - "6d039a63-17ad- amplitude \n", - "74e10781-1228- amplitude \n", - "7e3fa66e-727e- amplitude \n", - "86897349-ff68- amplitude \n", - "8bbddc0f-d6ae- amplitude \n", + "003bf29a-fa09- amplitude \n", + "004faf9a-72cb- amplitude \n", + "0061a9df-3954- amplitude \n", + "00775472-67a6- amplitude \n", + "00a1861f-bbf0- amplitude \n", + "00a9f0d0-b682- amplitude \n", + "00bd2bbf-ccdb- amplitude \n", + "00bdda4f-7059- amplitude \n", + "00db0baa-4ec0- amplitude \n", + "00e5a16b-c4f2- amplitude \n", + "00f25c1b-d5a3- amplitude \n", + "012fdb25-bd7e- amplitude \n", " ...\n", - " (Total: 23)" + " (Total: 3504)" ] }, - "execution_count": 8, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -619,59 +726,38 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "First we find the units we need:\n" + "First we find the units we need. We can use the method `SpikeSortingOutput.get_restricted_merge_ids()` to perform the needed joins to find them:\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([UUID('485a4ddf-332d-35b5-3ad4-0561736c1844'),\n", - " UUID('6acb99b8-6a0c-eb83-1141-5f603c5895e0'),\n", - " UUID('f7237e18-4e73-4aee-805b-90735e9147de'),\n", - " UUID('7e3fa66e-727e-1541-819a-b01309bb30ae'),\n", - " UUID('6d039a63-17ad-0b78-4b1e-f02d5f3dbbc5'),\n", - " UUID('e0e9133a-7a4e-1321-a43a-e8afcb2f25da'),\n", - " UUID('9959b614-2318-f597-6651-a3a82124d28a'),\n", - " UUID('c0eb6455-fc41-c200-b62e-e3ca81b9a3f7'),\n", - " UUID('912e250e-56d8-ee33-4525-c844d810971b'),\n", - " UUID('d7d2c97a-0e6e-d1b8-735c-d55dc66a30e1'),\n", - " UUID('abb92dce-4410-8f17-a501-a4104bda0dcf'),\n", - " UUID('74e10781-1228-4075-0870-af224024ffdc'),\n", - " UUID('8bbddc0f-d6ae-6260-9400-f884a6e25ae8'),\n", - " UUID('614d796c-0b95-6364-aaa0-b6cb1e7bbb83'),\n", - " UUID('b332482b-e430-169d-8ac0-0a73ce968ed7'),\n", - " UUID('86897349-ff68-ac72-02eb-739dd88936e6'),\n", - " UUID('4a712103-c223-864f-82e0-6c23de79cc14'),\n", - " UUID('cf858380-e8a3-49de-c2a9-1a277e307a68'),\n", - " UUID('cc4ee561-f974-f8e5-0ea4-83185263ac67'),\n", - " UUID('4a72c253-b3ca-8c13-e615-736a7ebff35c'),\n", - " UUID('b92a94d8-ee1e-2097-a81f-5c1e1556ed24'),\n", - " UUID('5c53bd33-d57c-fbba-e0fb-55e0bcb85d03'),\n", - " UUID('0751a1e1-a406-7f87-ae6f-ce4ffc60621c')], dtype=object)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput\n", - "\n", - "merge_ids = (\n", - " (SpikeSortingOutput.CurationV1 * sgs.SpikeSortingSelection)\n", - " & {\n", + "nwb_copy_file_name = \"mediumnwb20230802_.nwb\"\n", + "from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "key = {\n", + " \"nwb_file_name\": nwb_copy_file_name,\n", + " \"sorter\": \"clusterless_thresholder\",\n", + " \"sorter_param_name\": \"default_clusterless\",\n", + "}\n", + "merge_ids = SpikeSortingOutput().get_restricted_merge_ids(\n", + " {\n", " \"nwb_file_name\": nwb_copy_file_name,\n", " \"sorter\": \"clusterless_thresholder\",\n", " \"sorter_param_name\": \"default_clusterless\",\n", - " }\n", - ").fetch(\"merge_id\")\n", - "merge_ids" + " },\n", + " sources=[\"v1\"],\n", + ")" ] }, { @@ -683,7 +769,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -751,44 +837,44 @@ "

features_param_name

\n", " a name for this set of parameters\n", " \n", - " 0751a1e1-a406-7f87-ae6f-ce4ffc60621c\n", - "amplitude485a4ddf-332d-35b5-3ad4-0561736c1844\n", - "amplitude4a712103-c223-864f-82e0-6c23de79cc14\n", - "amplitude4a72c253-b3ca-8c13-e615-736a7ebff35c\n", - "amplitude5c53bd33-d57c-fbba-e0fb-55e0bcb85d03\n", - "amplitude614d796c-0b95-6364-aaa0-b6cb1e7bbb83\n", - "amplitude6acb99b8-6a0c-eb83-1141-5f603c5895e0\n", - "amplitude6d039a63-17ad-0b78-4b1e-f02d5f3dbbc5\n", - "amplitude74e10781-1228-4075-0870-af224024ffdc\n", - "amplitude7e3fa66e-727e-1541-819a-b01309bb30ae\n", - "amplitude86897349-ff68-ac72-02eb-739dd88936e6\n", - "amplitude8bbddc0f-d6ae-6260-9400-f884a6e25ae8\n", + " 0233e49a-b849-7eab-7434-9c298eea87b8\n", + "amplitude07239cea-7578-5409-692c-18c9d26b4d36\n", + "amplitude08be9775-370d-6492-0b4e-a5db4ce7a128\n", + "amplitude11819f33-11d5-f0f8-2590-ce3d60b76f3a\n", + "amplitude1c2ea289-2e7f-dcda-0464-ce97d3d6a392\n", + "amplitude20f24092-d191-0c58-55c8-d43d453f9fd4\n", + "amplitude2598b48e-49a0-3389-dd15-0230e8d326e4\n", + "amplitude483055a5-9775-27b7-856e-01543bd920aa\n", + "amplitude50ae3f7e-65a8-5fc2-5304-ab534b90fa46\n", + "amplitude50b29d01-2d74-e37e-2842-ad56d833c5f9\n", + "amplitude5e756e76-68be-21b7-7764-cb78d9aa4ef8\n", + "amplitude67f156e1-5da7-9c89-03b1-cc2dba88dacd\n", "amplitude \n", " \n", "

...

\n", - "

Total: 23

\n", + "

Total: 26

\n", " " ], "text/plain": [ "*spikesorting_ *features_para\n", "+------------+ +------------+\n", - "0751a1e1-a406- amplitude \n", - "485a4ddf-332d- amplitude \n", - "4a712103-c223- amplitude \n", - "4a72c253-b3ca- amplitude \n", - "5c53bd33-d57c- amplitude \n", - "614d796c-0b95- amplitude \n", - "6acb99b8-6a0c- amplitude \n", - "6d039a63-17ad- amplitude \n", - "74e10781-1228- amplitude \n", - "7e3fa66e-727e- amplitude \n", - "86897349-ff68- amplitude \n", - "8bbddc0f-d6ae- amplitude \n", + "0233e49a-b849- amplitude \n", + "07239cea-7578- amplitude \n", + "08be9775-370d- amplitude \n", + "11819f33-11d5- amplitude \n", + "1c2ea289-2e7f- amplitude \n", + "20f24092-d191- amplitude \n", + "2598b48e-49a0- amplitude \n", + "483055a5-9775- amplitude \n", + "50ae3f7e-65a8- amplitude \n", + "50b29d01-2d74- amplitude \n", + "5e756e76-68be- amplitude \n", + "67f156e1-5da7- amplitude \n", " ...\n", - " (Total: 23)" + " (Total: 26)" ] }, - "execution_count": 10, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -815,660 +901,9 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/edeno/miniconda3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/waveform_extractor.py:275: UserWarning: Sorting object is not dumpable, which might result in downstream errors for parallel processing. To make the sorting dumpable, use the `sorting.save()` function.\n", - " warn(\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c4f79735339147cf93143b0d329f7b0c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "extract waveforms memmap: 0%| | 0/2 [00:00= 0.1.0\n" - ] - }, { "data": { "text/html": [ @@ -123,15 +114,18 @@ "
\n", " \n", " \n", - " \n", - "\n", + " \n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", - "\n", + "\n", "\n", - "\n", + "\n", + "\n", + "\n", "\n", "\n", "\n", - "\n", - "\n", + "\n", "
\n", - "

sorting_id

\n", - " \n", - "
\n", "

merge_id

\n", " \n", "
\n", "

features_param_name

\n", " a name for this set of parameters\n", "
\n", + "

sorting_id

\n", + " \n", + "
\n", + "

curation_id

\n", + " \n", + "
\n", "

recording_id

\n", " \n", "
\n", @@ -146,132 +140,129 @@ "
\n", "

interval_list_name

\n", " descriptive name of this interval list\n", - "
\n", - "

curation_id

\n", - " \n", "
08a302b6-5505-40fa-b4d5-62162f8eef58485a4ddf-332d-35b5-3ad4-0561736c1844
0233e49a-b849-7eab-7434-9c298eea87b8amplitude449b64e3-db0b-437e-a1b9-0d29928aa2dd85cb4efd-5dd9-4637-8c47-50927da56ecb0d6ec337b-f131-47fa-8d04-f152459539abclusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb45f6b9a1-eef3-46eb-866d-d0999afebda60
0ca508ee-af4c-4a89-8181-d48bd209bfd46acb99b8-6a0c-eb83-1141-5f603c5895e0d4d3d806-13dc-42b9-a149-267fa170aa8f
07239cea-7578-5409-692c-18c9d26b4d36amplitude328da21c-1d9c-41e2-9800-76b3484b707b17abb5a3-cc9a-4a7f-8fbf-ae3bcffad23909b34c86e-f2d0-4c6c-a7b8-302ef30b0fffclusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb686d9951-1c0f-4d5e-9f5c-09e6fd8bdd4c0
209dc048-6fae-4315-b293-c06fff29f947f7237e18-4e73-4aee-805b-90735e9147de24608f0d-ffca-4f56-8dd3-a274b7248b63
08be9775-370d-6492-0b4e-a5db4ce7a128amplitudeaff78f2f-2ba0-412a-95cc-447c3a2f46832056130f-b8c9-46d1-9c27-4287d237f63f0e9ea1b3c-6e7b-4960-a593-0dd6d5ab0990clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb719e8a86-fcf1-4ffc-8c1f-ea912f67ad5d0
21a9a593-f6f3-4b82-99d7-8fc46556eff37e3fa66e-727e-1541-819a-b01309bb30aec96e245d-efef-4ab6-b549-683270857dbb
11819f33-11d5-f0f8-2590-ce3d60b76f3aamplitude2402805a-04f9-4a88-9ccf-071376c8de1971add870-7efe-4e64-b5fc-079c7b6d4a8a08f4b5933-7f9d-4ca1-a262-9a7978630101clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwbd581b117-160e-4311-b096-7781a4de43940
406a20e3-5a9f-4fec-b046-a6561f72461e6d039a63-17ad-0b78-4b1e-f02d5f3dbbc59d5a025a-2b46-47b3-94f4-70d58db68e60
1c2ea289-2e7f-dcda-0464-ce97d3d6a392amplitudef1427e00-2974-4301-b2ac-b4dc29277c5146b8a445-1513-44ce-8a14-d1c9dec80d7400d247564-2302-4ace-9157-c3891eceaf2cclusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb0e848c38-9105-4ea4-b6ba-dbdd5b46a0880
4131c51b-c56d-41fa-b046-46635fc17fd9e0e9133a-7a4e-1321-a43a-e8afcb2f25da56cbb21e-8fe8-4f4a-b2b0-537ad6039543
20f24092-d191-0c58-55c8-d43d453f9fd4amplitude9e332d82-1daf-4e92-bb50-12e4f9430875aec60cb7-017c-42ed-91be-0fb2a5f759480747f4eea-6df3-422b-941e-b5aaad7ec607clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb9ed11db5-c42e-491a-8caf-7d9a37a65f130
4c5a629a-71d9-481d-ab11-a4cb0fc160879959b614-2318-f597-6651-a3a82124d28a65009b63-5830-45b5-9954-cd5341aa8cef
2598b48e-49a0-3389-dd15-0230e8d326e4amplitude3a2c3eed-413a-452a-83c8-0e4648141bdee26863d0-7a77-455c-b687-0af1bd626486034ea9dd3-b728-4bd3-872c-7a4e37fb2ac9clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb2b9fbf14-74a0-4294-a805-26702340aac90
4d629c07-1931-4e1f-a3a8-cbf1b72161e3c0eb6455-fc41-c200-b62e-e3ca81b9a3f7e4daaf56-e40d-41d3-8523-097237d98bbd
483055a5-9775-27b7-856e-01543bd920aaamplitudef07bc0b0-de6b-4424-8ef9-766213aaca269af6681f-2e37-496e-823e-7acbdd436a27073c9e01c-b37c-41a2-8571-0df13c32bf76clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb5c68f0f0-f577-4905-8a09-e4d171d0a22d0
554a9a3c-0461-48be-8435-123eed59c228912e250e-56d8-ee33-4525-c844d810971b3da02b84-1a7f-4f2a-81bf-2e92c4d88e96
50ae3f7e-65a8-5fc2-5304-ab534b90fa46amplitude7f128981-6868-4976-ba20-248655dcac212483d0c7-4cfe-4d6f-8dd6-2e13a8289d94003cc7709-66e7-47ac-a3bd-63add028d9f8clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwbf4b9301f-bc91-455b-9474-c801093f38560
7bb007f2-26d3-463f-b7dc-7bd4d271725ed7d2c97a-0e6e-d1b8-735c-d55dc66a30e18cfc1ccb-8de3-4eee-9e18-f8b8f5c45821
50b29d01-2d74-e37e-2842-ad56d833c5f9amplitudea9b7cec0-1256-49cf-abf0-8c45fd1553791dcecaac-8e0d-4d18-8296-cdb50eef95060d8a8c564-13c7-4fab-9a33-1eac416869daclusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb74270cba-36ee-4afb-ab50-2a6cc948e68c0
80e1f37f-48a7-4087-bd37-7a37b6a2c160abb92dce-4410-8f17-a501-a4104bda0dcf96678676-89dd-42e4-89f6-ce56c618ce83
5e756e76-68be-21b7-7764-cb78d9aa4ef8amplitude3c40ebdc-0b61-4105-9971-e1348bd49bc7552176ab-d870-41c4-8621-07e71f6e9a190fa4faf43-e747-43ca-b8a5-53a02d7938ecclusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwb0f91197e-bebb-4dc6-ad41-5bf89c3eed280
8848c4a8-a2f2-4f3d-82cd-51b13b8bae3c74e10781-1228-4075-0870-af224024ffdc07036486-e9f5-4dba-8662-7fb5ff2a6711
67f156e1-5da7-9c89-03b1-cc2dba88dacdamplitude257c077b-8f3b-4abb-a631-6b8084d6a1ea8f45b210-c8f9-4a27-96c2-9b85f16b3451030895f0f-1eec-481d-b763-edae7667ef00clusterless_thresholderdefault_clusterlessmediumnwb20230802_.nwbe289e03d-32ad-461a-a1cc-c885373431490
22fb2b64-fc3c-44af-a8c1-dacc9010beab
\n", "

...

\n", - "

Total: 23

\n", + "

Total: 26

\n", " " ], "text/plain": [ - "*sorting_id *merge_id *features_para recording_id sorter sorter_param_n nwb_file_name interval_list_ curation_id \n", + "*merge_id *features_para *sorting_id curation_id recording_id sorter sorter_param_n nwb_file_name interval_list_\n", "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", - "08a302b6-5505- 485a4ddf-332d- amplitude 449b64e3-db0b- clusterless_th default_cluste mediumnwb20230 45f6b9a1-eef3- 0 \n", - "0ca508ee-af4c- 6acb99b8-6a0c- amplitude 328da21c-1d9c- clusterless_th default_cluste mediumnwb20230 686d9951-1c0f- 0 \n", - "209dc048-6fae- f7237e18-4e73- amplitude aff78f2f-2ba0- clusterless_th default_cluste mediumnwb20230 719e8a86-fcf1- 0 \n", - "21a9a593-f6f3- 7e3fa66e-727e- amplitude 2402805a-04f9- clusterless_th default_cluste mediumnwb20230 d581b117-160e- 0 \n", - "406a20e3-5a9f- 6d039a63-17ad- amplitude f1427e00-2974- clusterless_th default_cluste mediumnwb20230 0e848c38-9105- 0 \n", - "4131c51b-c56d- e0e9133a-7a4e- amplitude 9e332d82-1daf- clusterless_th default_cluste mediumnwb20230 9ed11db5-c42e- 0 \n", - "4c5a629a-71d9- 9959b614-2318- amplitude 3a2c3eed-413a- clusterless_th default_cluste mediumnwb20230 2b9fbf14-74a0- 0 \n", - "4d629c07-1931- c0eb6455-fc41- amplitude f07bc0b0-de6b- clusterless_th default_cluste mediumnwb20230 5c68f0f0-f577- 0 \n", - "554a9a3c-0461- 912e250e-56d8- amplitude 7f128981-6868- clusterless_th default_cluste mediumnwb20230 f4b9301f-bc91- 0 \n", - "7bb007f2-26d3- d7d2c97a-0e6e- amplitude a9b7cec0-1256- clusterless_th default_cluste mediumnwb20230 74270cba-36ee- 0 \n", - "80e1f37f-48a7- abb92dce-4410- amplitude 3c40ebdc-0b61- clusterless_th default_cluste mediumnwb20230 0f91197e-bebb- 0 \n", - "8848c4a8-a2f2- 74e10781-1228- amplitude 257c077b-8f3b- clusterless_th default_cluste mediumnwb20230 e289e03d-32ad- 0 \n", + "0233e49a-b849- amplitude 85cb4efd-5dd9- 0 d6ec337b-f131- clusterless_th default_cluste mediumnwb20230 d4d3d806-13dc-\n", + "07239cea-7578- amplitude 17abb5a3-cc9a- 0 9b34c86e-f2d0- clusterless_th default_cluste mediumnwb20230 24608f0d-ffca-\n", + "08be9775-370d- amplitude 2056130f-b8c9- 0 e9ea1b3c-6e7b- clusterless_th default_cluste mediumnwb20230 c96e245d-efef-\n", + "11819f33-11d5- amplitude 71add870-7efe- 0 8f4b5933-7f9d- clusterless_th default_cluste mediumnwb20230 9d5a025a-2b46-\n", + "1c2ea289-2e7f- amplitude 46b8a445-1513- 0 0d247564-2302- clusterless_th default_cluste mediumnwb20230 56cbb21e-8fe8-\n", + "20f24092-d191- amplitude aec60cb7-017c- 0 747f4eea-6df3- clusterless_th default_cluste mediumnwb20230 65009b63-5830-\n", + "2598b48e-49a0- amplitude e26863d0-7a77- 0 34ea9dd3-b728- clusterless_th default_cluste mediumnwb20230 e4daaf56-e40d-\n", + "483055a5-9775- amplitude 9af6681f-2e37- 0 73c9e01c-b37c- clusterless_th default_cluste mediumnwb20230 3da02b84-1a7f-\n", + "50ae3f7e-65a8- amplitude 2483d0c7-4cfe- 0 03cc7709-66e7- clusterless_th default_cluste mediumnwb20230 8cfc1ccb-8de3-\n", + "50b29d01-2d74- amplitude 1dcecaac-8e0d- 0 d8a8c564-13c7- clusterless_th default_cluste mediumnwb20230 96678676-89dd-\n", + "5e756e76-68be- amplitude 552176ab-d870- 0 fa4faf43-e747- clusterless_th default_cluste mediumnwb20230 07036486-e9f5-\n", + "67f156e1-5da7- amplitude 8f45b210-c8f9- 0 30895f0f-1eec- clusterless_th default_cluste mediumnwb20230 22fb2b64-fc3c-\n", " ...\n", - " (Total: 23)" + " (Total: 26)" ] }, - "execution_count": 2, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -279,7 +270,10 @@ "source": [ "from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput\n", "import spyglass.spikesorting.v1 as sgs\n", - "from spyglass.decoding.v1.waveform_features import UnitWaveformFeaturesSelection\n", + "from spyglass.decoding.v1.waveform_features import (\n", + " UnitWaveformFeaturesSelection,\n", + " UnitWaveformFeatures,\n", + ")\n", "\n", "\n", "nwb_copy_file_name = \"mediumnwb20230802_.nwb\"\n", @@ -292,15 +286,18 @@ "\n", "feature_key = {\"features_param_name\": \"amplitude\"}\n", "\n", - "(sgs.SpikeSortingSelection & sorter_keys) * SpikeSortingOutput.CurationV1 * (\n", + "(\n", " UnitWaveformFeaturesSelection.proj(merge_id=\"spikesorting_merge_id\")\n", - " & feature_key\n", + " * SpikeSortingOutput.CurationV1\n", + " * sgs.SpikeSortingSelection\n", + ") & SpikeSortingOutput().get_restricted_merge_ids(\n", + " sorter_keys, sources=[\"v1\"], as_dict=True\n", ")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 24, "metadata": {}, "outputs": [ { @@ -368,44 +365,44 @@ "

features_param_name

\n", " a name for this set of parameters\n", " \n", - " 0751a1e1-a406-7f87-ae6f-ce4ffc60621c\n", - "amplitude485a4ddf-332d-35b5-3ad4-0561736c1844\n", - "amplitude4a712103-c223-864f-82e0-6c23de79cc14\n", - "amplitude4a72c253-b3ca-8c13-e615-736a7ebff35c\n", - "amplitude5c53bd33-d57c-fbba-e0fb-55e0bcb85d03\n", - "amplitude614d796c-0b95-6364-aaa0-b6cb1e7bbb83\n", - "amplitude6acb99b8-6a0c-eb83-1141-5f603c5895e0\n", - "amplitude6d039a63-17ad-0b78-4b1e-f02d5f3dbbc5\n", - "amplitude74e10781-1228-4075-0870-af224024ffdc\n", - "amplitude7e3fa66e-727e-1541-819a-b01309bb30ae\n", - "amplitude86897349-ff68-ac72-02eb-739dd88936e6\n", - "amplitude8bbddc0f-d6ae-6260-9400-f884a6e25ae8\n", + " 0233e49a-b849-7eab-7434-9c298eea87b8\n", + "amplitude07239cea-7578-5409-692c-18c9d26b4d36\n", + "amplitude08be9775-370d-6492-0b4e-a5db4ce7a128\n", + "amplitude11819f33-11d5-f0f8-2590-ce3d60b76f3a\n", + "amplitude1c2ea289-2e7f-dcda-0464-ce97d3d6a392\n", + "amplitude20f24092-d191-0c58-55c8-d43d453f9fd4\n", + "amplitude2598b48e-49a0-3389-dd15-0230e8d326e4\n", + "amplitude483055a5-9775-27b7-856e-01543bd920aa\n", + "amplitude50ae3f7e-65a8-5fc2-5304-ab534b90fa46\n", + "amplitude50b29d01-2d74-e37e-2842-ad56d833c5f9\n", + "amplitude5e756e76-68be-21b7-7764-cb78d9aa4ef8\n", + "amplitude67f156e1-5da7-9c89-03b1-cc2dba88dacd\n", "amplitude \n", " \n", "

...

\n", - "

Total: 23

\n", + "

Total: 26

\n", " " ], "text/plain": [ "*spikesorting_ *features_para\n", "+------------+ +------------+\n", - "0751a1e1-a406- amplitude \n", - "485a4ddf-332d- amplitude \n", - "4a712103-c223- amplitude \n", - "4a72c253-b3ca- amplitude \n", - "5c53bd33-d57c- amplitude \n", - "614d796c-0b95- amplitude \n", - "6acb99b8-6a0c- amplitude \n", - "6d039a63-17ad- amplitude \n", - "74e10781-1228- amplitude \n", - "7e3fa66e-727e- amplitude \n", - "86897349-ff68- amplitude \n", - "8bbddc0f-d6ae- amplitude \n", + "0233e49a-b849- amplitude \n", + "07239cea-7578- amplitude \n", + "08be9775-370d- amplitude \n", + "11819f33-11d5- amplitude \n", + "1c2ea289-2e7f- amplitude \n", + "20f24092-d191- amplitude \n", + "2598b48e-49a0- amplitude \n", + "483055a5-9775- amplitude \n", + "50ae3f7e-65a8- amplitude \n", + "50b29d01-2d74- amplitude \n", + "5e756e76-68be- amplitude \n", + "67f156e1-5da7- amplitude \n", " ...\n", - " (Total: 23)" + " (Total: 26)" ] }, - "execution_count": 3, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -413,19 +410,19 @@ "source": [ "from spyglass.decoding.v1.waveform_features import UnitWaveformFeaturesSelection\n", "\n", - "spikesorting_merge_id = (\n", - " (sgs.SpikeSortingSelection & sorter_keys)\n", - " * SpikeSortingOutput.CurationV1\n", - " * (\n", - " UnitWaveformFeaturesSelection.proj(merge_id=\"spikesorting_merge_id\")\n", - " & feature_key\n", - " )\n", - ").fetch(\"merge_id\")\n", + "# find the merge ids that correspond to the sorter key restrictions\n", + "merge_ids = SpikeSortingOutput().get_restricted_merge_ids(\n", + " sorter_keys, sources=[\"v1\"], as_dict=True\n", + ")\n", "\n", - "waveform_selection_keys = [\n", - " {\"spikesorting_merge_id\": merge_id, \"features_param_name\": \"amplitude\"}\n", - " for merge_id in spikesorting_merge_id\n", - "]\n", + "# find the previously populated waveform selection keys that correspond to these sorts\n", + "waveform_selection_keys = (\n", + " UnitWaveformFeaturesSelection().proj(merge_id=\"spikesorting_merge_id\")\n", + " & merge_ids\n", + " & feature_key\n", + ").fetch(as_dict=True)\n", + "for key in waveform_selection_keys:\n", + " key[\"spikesorting_merge_id\"] = key.pop(\"merge_id\")\n", "\n", "UnitWaveformFeaturesSelection & waveform_selection_keys" ] @@ -2959,7 +2956,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/notebooks/42_Decoding_SortedSpikes.ipynb b/notebooks/42_Decoding_SortedSpikes.ipynb index 23e1888fe..66c3de7f0 100644 --- a/notebooks/42_Decoding_SortedSpikes.ipynb +++ b/notebooks/42_Decoding_SortedSpikes.ipynb @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -40,7 +40,7 @@ "## SortedSpikesGroup\n", "\n", "`SortedSpikesGroup` is a child table of `SpikeSortingOutput` in the spikesorting pipeline. It allows us to group the spikesorting results from multiple \n", - "sources (e.g. multiple terode groups or intervals) into a single entry. Here we will group together the spiking of multiple tetrode groups to use for decoding.\n", + "sources (e.g. multiple tetrode groups or intervals) into a single entry. Here we will group together the spiking of multiple tetrode groups to use for decoding.\n", "\n", "\n", "This table allows us filter units by their annotation labels from curation (e.g only include units labeled \"good\", exclude units labeled \"noise\") by defining parameters from `UnitSelectionParams`. When accessing data through `SortedSpikesGroup` the table will include only units with at least one label in `include_labels` and no labels in `exclude_labels`. We can look at those here:\n" @@ -48,18 +48,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[2024-02-02 12:06:04,725][INFO]: Connecting sambray@lmf-db.cin.ucsf.edu:3306\n", - "[2024-02-02 12:06:04,762][INFO]: Connected sambray@lmf-db.cin.ucsf.edu:3306\n", - "[12:06:05][WARNING] Spyglass: Please update position_tools to >= 0.1.0\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -141,10 +132,12 @@ "=BLOB=\n", "=BLOB=exclude_noise\n", "=BLOB=\n", + "=BLOB=MS2220180629\n", + "=BLOB=\n", "=BLOB= \n", " \n", " \n", - "

Total: 3

\n", + "

Total: 4

\n", " " ], "text/plain": [ @@ -153,10 +146,11 @@ "all_units =BLOB= =BLOB= \n", "default_exclus =BLOB= =BLOB= \n", "exclude_noise =BLOB= =BLOB= \n", - " (Total: 3)" + "MS2220180629 =BLOB= =BLOB= \n", + " (Total: 4)" ] }, - "execution_count": 1, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -187,7 +181,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -309,7 +303,7 @@ " (Total: 3)" ] }, - "execution_count": 2, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -326,111 +320,21 @@ " \"curation_id\": 1,\n", "}\n", "# check the set of sorting we'll use\n", - "(sgs.SpikeSortingSelection & sorter_keys) * SpikeSortingOutput.CurationV1" + "(\n", + " sgs.SpikeSortingSelection & sorter_keys\n", + ") * SpikeSortingOutput.CurationV1 & sorter_keys" ] }, { - "cell_type": "code", - "execution_count": 6, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " \n", - " \n", - " \n", - "\n", - "\n", - "
\n", - "

nwb_file_name

\n", - " name of the NWB file\n", - "
\n", - "

unit_filter_params_name

\n", - " \n", - "
\n", - "

sorted_spikes_group_name

\n", - " \n", - "
mediumnwb20230802_.nwball_unitstest_group
\n", - " \n", - "

Total: 1

\n", - " " - ], - "text/plain": [ - "*nwb_file_name *unit_filter_p *sorted_spikes\n", - "+------------+ +------------+ +------------+\n", - "mediumnwb20230 all_units test_group \n", - " (Total: 1)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "from spyglass.decoding.v1.sorted_spikes import SortedSpikesGroup\n", - "\n", - "SortedSpikesGroup()" + "Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -502,33 +406,30 @@ " \n", " \n", " mediumnwb20230802_.nwb\n", - "all_units\n", - "test_groupmediumnwb20230802_.nwb\n", "default_exclusion\n", "test_group \n", " \n", " \n", - "

Total: 2

\n", + "

Total: 1

\n", " " ], "text/plain": [ "*nwb_file_name *unit_filter_p *sorted_spikes\n", "+------------+ +------------+ +------------+\n", - "mediumnwb20230 all_units test_group \n", "mediumnwb20230 default_exclus test_group \n", - " (Total: 2)" + " (Total: 1)" ] }, - "execution_count": 7, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# get the merge_ids for the selected sorting\n", - "spikesorting_merge_ids = (\n", - " (sgs.SpikeSortingSelection & sorter_keys) * SpikeSortingOutput.CurationV1\n", - ").fetch(\"merge_id\")\n", + "spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids(\n", + " sorter_keys, restrict_by_artifact=False\n", + ")\n", "\n", "# create a new sorted spikes group\n", "unit_filter_params_name = \"default_exclusion\"\n", @@ -1536,7 +1437,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/notebooks/py_scripts/10_Spike_SortingV1.py b/notebooks/py_scripts/10_Spike_SortingV1.py index 96ee444ff..d74c2b4d2 100644 --- a/notebooks/py_scripts/10_Spike_SortingV1.py +++ b/notebooks/py_scripts/10_Spike_SortingV1.py @@ -199,7 +199,7 @@ sgs.MetricCurationSelection.insert_selection(key) sgs.MetricCurationSelection() & key -sgs.MetricCuration.populate() +sgs.MetricCuration.populate(key) sgs.MetricCuration() & key # to do another round of curation, fetch the relevant info and insert back into CurationV1 using `insert_curation` @@ -227,7 +227,7 @@ sgs.CurationV1() -# ## Manual Curation +# ## Manual Curation (Optional) # Next we will do manual curation. this is done with figurl. to incorporate info from other stages of processing (e.g. metrics) we have to store that with kachery cloud and get curation uri referring to it. it can be done with `generate_curation_uri`. # @@ -297,8 +297,9 @@ sgs.CurationV1() -# We now insert the curated spike sorting to a `Merge` table for feeding into downstream processing pipelines. +# ## Downstream usage (Merge table) # +# Regardless of Curation method used, to make use of spikeorting results in downstream pipelines like Decoding, we will need to insert it into the `SpikeSortingOutput` merge table. # + from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput @@ -306,7 +307,27 @@ SpikeSortingOutput() # - -SpikeSortingOutput.insert([key], part_name="CurationV1") +# insert the automatic curation spikesorting results +curation_key = sss_pk.fetch1("KEY") +curation_key["curation_id"] = 1 +merge_insert_key = (sgs.CurationV1 & curation_key).fetch("KEY", as_dict=True) +SpikeSortingOutput.insert(merge_insert_key, part_name="CurationV1") SpikeSortingOutput.merge_view() -SpikeSortingOutput.CurationV1() +# Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's + +selection_key = { + "nwb_file_name": nwb_file_name2, + "sorter": "mountainsort4", + "interval_list_name": "01_s1", + "curation_id": 0, +} # this function can use restrictions from throughout the spikesorting pipeline +spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids( + selection_key, as_dict=True +) +spikesorting_merge_ids + +# With the spikesorting merge_ids we want we can also use the method `get_sort_group_info` to get a table linking the merge id to the electrode group it is sourced from. This can be helpful for restricting to just electrodes from a brain area of interest + +merge_keys = [{"merge_id": str(id)} for id in spikesorting_merge_ids] +SpikeSortingOutput().get_sort_group_info(merge_keys) diff --git a/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py b/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py new file mode 100644 index 000000000..ad17a7c6f --- /dev/null +++ b/notebooks/py_scripts/40_Extracting_Clusterless_Waveform_Features.py @@ -0,0 +1,348 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.15.2 +# kernelspec: +# display_name: spyglass +# language: python +# name: python3 +# --- + +# _Developer Note:_ if you may make a PR in the future, be sure to copy this +# notebook, and use the `gitignore` prefix `temp` to avoid future conflicts. +# +# This is one notebook in a multi-part series on clusterless decoding in Spyglass +# +# - To set up your Spyglass environment and database, see +# [the Setup notebook](./00_Setup.ipynb) +# - For additional info on DataJoint syntax, including table definitions and +# inserts, see +# [the Insert Data notebook](./01_Insert_Data.ipynb) +# - Prior to running, please familiarize yourself with the [spike sorting +# pipeline](./02_Spike_Sorting.ipynb) and generate input position data with +# either the [Trodes](./20_Position_Trodes.ipynb) or DLC notebooks +# ([1](./21_Position_DLC_1.ipynb), [2](./22_Position_DLC_2.ipynb), +# [3](./23_Position_DLC_3.ipynb)). +# +# The goal of this notebook is to populate the `UnitWaveformFeatures` table, which depends `SpikeSortingOutput`. This table contains the features of the waveforms of each unit. +# +# While clusterless decoding avoids actual spike sorting, we need to pass through these tables to maintain (relative) pipeline simplicity. Pass-through tables keep spike sorting and clusterless waveform extraction as similar as possible, by using shared steps. Here, "spike sorting" involves simple thresholding (sorter: clusterless_thresholder). +# + +# + +from pathlib import Path +import datajoint as dj + +dj.config.load( + Path("../dj_local_conf.json").absolute() +) # load config for database connection info +# - + +# First, if you haven't inserted the the `mediumnwb20230802.wnb` file into the database, you should do so now. This is the file that we will use for the decoding tutorials. +# +# It is a truncated version of the full NWB file, so it will run faster, but bigger than the minirec file we used in the previous tutorials so that decoding makes sense. +# + +# + +from spyglass.utils.nwb_helper_fn import get_nwb_copy_filename +import spyglass.data_import as sgi +import spyglass.position as sgp + +# Insert the nwb file +nwb_file_name = "mediumnwb20230802.nwb" +nwb_copy_file_name = get_nwb_copy_filename(nwb_file_name) +sgi.insert_sessions(nwb_file_name) + +# Position +sgp.v1.TrodesPosParams.insert_default() + +interval_list_name = "pos 0 valid times" + +trodes_s_key = { + "nwb_file_name": nwb_copy_file_name, + "interval_list_name": interval_list_name, + "trodes_pos_params_name": "default", +} +sgp.v1.TrodesPosSelection.insert1( + trodes_s_key, + skip_duplicates=True, +) +sgp.v1.TrodesPosV1.populate(trodes_s_key) +# - + +# These next steps are the same as in the [Spike Sorting notebook](./10_Spike_SortingV1.ipynb), but we'll repeat them here for clarity. These are pre-processing steps that are shared between spike sorting and clusterless decoding. +# +# We first set the `SortGroup` to define which contacts are sorted together. +# +# We then setup for spike sorting by bandpass filtering and whitening the data via the `SpikeSortingRecording` table. +# + +# + +import spyglass.spikesorting.v1 as sgs + +sgs.SortGroup.set_group_by_shank(nwb_file_name=nwb_copy_file_name) + +sort_group_ids = (sgs.SortGroup & {"nwb_file_name": nwb_copy_file_name}).fetch( + "sort_group_id" +) + +group_keys = [] +for sort_group_id in sort_group_ids: + key = { + "nwb_file_name": nwb_copy_file_name, + "sort_group_id": sort_group_id, + "interval_list_name": interval_list_name, + "preproc_param_name": "default", + "team_name": "Alison Comrie", + } + group_keys.append(key) + sgs.SpikeSortingRecordingSelection.insert_selection(key) + +sgs.SpikeSortingRecording.populate(group_keys) +# - + +# Next we do artifact detection. Here we skip it by setting the `artifact_param_name` to `None`, but in practice you should detect artifacts as it will affect the decoding. +# + +# + +recording_ids = ( + sgs.SpikeSortingRecordingSelection & {"nwb_file_name": nwb_copy_file_name} +).fetch("recording_id") + +group_keys = [] +for recording_id in recording_ids: + key = { + "recording_id": recording_id, + "artifact_param_name": "none", + } + group_keys.append(key) + sgs.ArtifactDetectionSelection.insert_selection(key) + +sgs.ArtifactDetection.populate(group_keys) +# - + +# Now we run the "spike sorting", which in our case is simply thresholding the signal to find spikes. We use the `SpikeSorting` table to store the results. Note that `sorter_param_name` defines the parameters for thresholding the signal. +# + +(sgs.SpikeSorterParameters() & {"sorter": "clusterless_thresholder"}).fetch1() + +group_keys = [] +for recording_id in recording_ids: + key = { + "recording_id": recording_id, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", + "nwb_file_name": nwb_copy_file_name, + "interval_list_name": str( + ( + sgs.ArtifactDetectionSelection + & {"recording_id": recording_id, "artifact_param_name": "none"} + ).fetch1("artifact_id") + ), + } + group_keys.append(key) + sgs.SpikeSortingSelection.insert_selection(key) +sort_keys = (sgs.SpikeSortingSelection & group_keys).fetch("KEY") +sgs.SpikeSorting.populate(sort_keys) + +# For clusterless decoding we do not need any manual curation, but for the sake of the pipeline, we need to store the output of the thresholding in the `CurationV1` table and insert this into the `SpikeSortingOutput` table. +# + +sgs.SpikeSorting().populate( + sgs.SpikeSortingSelection + & { + "nwb_file_name": nwb_copy_file_name, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", + } +) + +# + +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput + +sorting_ids = ( + sgs.SpikeSortingSelection + & { + "nwb_file_name": nwb_copy_file_name, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", + } +).fetch("sorting_id") + +for sorting_id in sorting_ids: + try: + sgs.CurationV1.insert_curation(sorting_id=sorting_id) + except KeyError as e: + pass + +SpikeSortingOutput.insert( + sgs.CurationV1().fetch("KEY"), + part_name="CurationV1", + skip_duplicates=True, +) +# - + +# Finally, we extract the waveform features of each SortGroup. This is done by the `UnitWaveformFeatures` table. +# +# To set this up, we use the `WaveformFeaturesParams` to define the time around the spike that we want to use for feature extraction, and which features to extract. Here is an example of the parameters used for extraction the amplitude of the negative peak of the waveform: +# +# ```python +# +# waveform_extraction_params = { +# "ms_before": 0.5, +# "ms_after": 0.5, +# "max_spikes_per_unit": None, +# "n_jobs": 5, +# "total_memory": "5G", +# } +# waveform_feature_params = { +# "amplitude": { +# "peak_sign": "neg", +# "estimate_peak_time": False, +# } +# } +# ``` +# +# We see that we want 0.5 ms of time before and after the peak of the negative spike. We also see that we want to extract the amplitude of the negative peak, and that we do not want to estimate the peak time (since we know it is at 0 ms). +# +# You can define other features to extract such as spatial location of the spike: +# +# ```python +# waveform_extraction_params = { +# "ms_before": 0.5, +# "ms_after": 0.5, +# "max_spikes_per_unit": None, +# "n_jobs": 5, +# "total_memory": "5G", +# } +# waveform_feature_params = { +# "amplitude": { +# "peak_sign": "neg", +# "estimate_peak_time": False, +# }, +# "spike location": {} +# } +# +# ``` +# +# _Note_: Members of the Frank Lab can use "ampl_10_jobs_v2" instead of "amplitude" +# for significant speed improvements. +# + +# + +from spyglass.decoding.v1.waveform_features import WaveformFeaturesParams + +waveform_extraction_params = { + "ms_before": 0.5, + "ms_after": 0.5, + "max_spikes_per_unit": None, + "n_jobs": 5, + "total_memory": "5G", +} +waveform_feature_params = { + "amplitude": { + "peak_sign": "neg", + "estimate_peak_time": False, + } +} + +WaveformFeaturesParams.insert1( + { + "features_param_name": "amplitude", + "params": { + "waveform_extraction_params": waveform_extraction_params, + "waveform_feature_params": waveform_feature_params, + }, + }, + skip_duplicates=True, +) + +WaveformFeaturesParams() +# - + +# Now that we've inserted the waveform features parameters, we need to define which parameters to use for each SortGroup. This is done by the `UnitWaveformFeaturesSelection` table. We need to link the primary key `merge_id` from the `SpikeSortingOutput` table to a features parameter set. +# + +# + +from spyglass.decoding.v1.waveform_features import UnitWaveformFeaturesSelection + +UnitWaveformFeaturesSelection() +# - + +# First we find the units we need. We can use the method `SpikeSortingOutput.get_restricted_merge_ids()` to perform the needed joins to find them: +# + +nwb_copy_file_name = "mediumnwb20230802_.nwb" +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput + +key = { + "nwb_file_name": nwb_copy_file_name, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", +} +merge_ids = SpikeSortingOutput().get_restricted_merge_ids( + { + "nwb_file_name": nwb_copy_file_name, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", + }, + sources=["v1"], +) + +# Then we link them with the features parameters: +# + +# + +selection_keys = [ + { + "spikesorting_merge_id": merge_id, + "features_param_name": "amplitude", + } + for merge_id in merge_ids +] +UnitWaveformFeaturesSelection.insert(selection_keys, skip_duplicates=True) + +UnitWaveformFeaturesSelection & selection_keys +# - + +# Finally, we extract the waveform features, by populating the `UnitWaveformFeatures` table: +# + +# + +from spyglass.decoding.v1.waveform_features import UnitWaveformFeatures + +UnitWaveformFeatures.populate(selection_keys) +# - + +UnitWaveformFeatures & selection_keys + +# Now that we've extracted the data, we can inspect the results. Let's fetch the data: +# + +spike_times, spike_waveform_features = ( + UnitWaveformFeatures & selection_keys +).fetch_data() + +# Let's look at the features shape. This is a list corresponding to tetrodes, with each element being a numpy array of shape (n_spikes, n_features). The features in this case are the amplitude of each tetrode wire at the negative peak of the waveform. +# + +for features in spike_waveform_features: + print(features.shape) + +# We can plot the amplitudes to see if there is anything that looks neural and to look for outliers: +# + +# + +import matplotlib.pyplot as plt + +tetrode_ind = 1 +plt.scatter( + spike_waveform_features[tetrode_ind][:, 0], + spike_waveform_features[tetrode_ind][:, 1], + s=1, +) +# - diff --git a/notebooks/py_scripts/41_Decoding_Clusterless.py b/notebooks/py_scripts/41_Decoding_Clusterless.py new file mode 100644 index 000000000..6c286bf8d --- /dev/null +++ b/notebooks/py_scripts/41_Decoding_Clusterless.py @@ -0,0 +1,450 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.15.2 +# kernelspec: +# display_name: spyglass +# language: python +# name: python3 +# --- + +# # Clusterless Decoding +# +# ## Overview +# +# _Developer Note:_ if you may make a PR in the future, be sure to copy this +# notebook, and use the `gitignore` prefix `temp` to avoid future conflicts. +# +# This is one notebook in a multi-part series on Spyglass. +# +# - To set up your Spyglass environment and database, see +# [the Setup notebook](./00_Setup.ipynb) +# - This tutorial assumes you've already +# [extracted waveforms](./41_Extracting_Clusterless_Waveform_Features.ipynb), as well as loaded +# [position data](./20_Position_Trodes.ipynb). If 1D decoding, this data should also be +# [linearized](./24_Linearization.ipynb). +# +# Clusterless decoding can be performed on either 1D or 2D data. We will start with 2D data. +# +# ## Elements of Clusterless Decoding +# - **Position Data**: This is the data that we want to decode. It can be 1D or 2D. +# - **Spike Waveform Features**: These are the features that we will use to decode the position data. +# - **Decoding Model Parameters**: This is how we define the model that we will use to decode the position data. +# +# ## Grouping Data +# An important concept will be groups. Groups are tables that allow use to specify collections of data. We will use groups in two situations here: +# 1. Because we want to decode from more than one tetrode (or probe), so we will create a group that contains all of the tetrodes that we want to decode from. +# 2. Similarly, we will create a group for the position data that we want to decode, so that we can decode from position data from multiple sessions. +# +# ### Grouping Waveform Features +# Let's start with grouping the Waveform Features. We will first inspect the waveform features that we have extracted to figure out the primary keys of the data that we want to decode from. We need to use the tables `SpikeSortingSelection` and `SpikeSortingOutput` to figure out the `merge_id` associated with `nwb_file_name` to get the waveform features associated with the NWB file of interest. +# + +# + +from pathlib import Path +import datajoint as dj + +dj.config.load( + Path("../dj_local_conf.json").absolute() +) # load config for database connection info + +# + +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput +import spyglass.spikesorting.v1 as sgs +from spyglass.decoding.v1.waveform_features import ( + UnitWaveformFeaturesSelection, + UnitWaveformFeatures, +) + + +nwb_copy_file_name = "mediumnwb20230802_.nwb" + +sorter_keys = { + "nwb_file_name": nwb_copy_file_name, + "sorter": "clusterless_thresholder", + "sorter_param_name": "default_clusterless", +} + +feature_key = {"features_param_name": "amplitude"} + +( + UnitWaveformFeaturesSelection.proj(merge_id="spikesorting_merge_id") + * SpikeSortingOutput.CurationV1 + * sgs.SpikeSortingSelection +) & SpikeSortingOutput().get_restricted_merge_ids( + sorter_keys, sources=["v1"], as_dict=True +) + +# + +from spyglass.decoding.v1.waveform_features import UnitWaveformFeaturesSelection + +# find the merge ids that correspond to the sorter key restrictions +merge_ids = SpikeSortingOutput().get_restricted_merge_ids( + sorter_keys, sources=["v1"], as_dict=True +) + +# find the previously populated waveform selection keys that correspond to these sorts +waveform_selection_keys = ( + UnitWaveformFeaturesSelection().proj(merge_id="spikesorting_merge_id") + & merge_ids + & feature_key +).fetch(as_dict=True) +for key in waveform_selection_keys: + key["spikesorting_merge_id"] = key.pop("merge_id") + +UnitWaveformFeaturesSelection & waveform_selection_keys +# - + +# We will create a group called `test_group` that contains all of the tetrodes that we want to decode from. We will use the `create_group` function to create this group. This function takes two arguments: the name of the group, and the keys of the tables that we want to include in the group. + +# + +from spyglass.decoding.v1.clusterless import UnitWaveformFeaturesGroup + +UnitWaveformFeaturesGroup().create_group( + nwb_file_name=nwb_copy_file_name, + group_name="test_group", + keys=waveform_selection_keys, +) +UnitWaveformFeaturesGroup & {"waveform_features_group_name": "test_group"} +# - + +# We can see that we successfully associated "test_group" with the tetrodes that we want to decode from by using the `get_group` function. + +UnitWaveformFeaturesGroup.UnitFeatures & { + "nwb_file_name": nwb_copy_file_name, + "waveform_features_group_name": "test_group", +} + +# ### Grouping Position Data +# +# We will now create a group called `02_r1` that contains all of the position data that we want to decode from. As before, we will use the `create_group` function to create this group. This function takes two arguments: the name of the group, and the keys of the tables that we want to include in the group. +# +# We use the the `PositionOutput` table to figure out the `merge_id` associated with `nwb_file_name` to get the position data associated with the NWB file of interest. In this case, we only have one position to insert, but we could insert multiple positions if we wanted to decode from multiple sessions. +# +# Note that the position data sampling frequency is what determines the time step of the decoding. In this case, the position data sampling frequency is 30 Hz, so the time step of the decoding will be 1/30 seconds. In practice, you will want to use a smaller time step such as 500 Hz. This will allow you to decode at a finer time scale. To do this, you will want to interpolate the position data to a higher sampling frequency as shown in the [position trodes notebook](./20_Position_Trodes.ipynb). +# +# You will also want to specify the name of the position variables if they are different from the default names. The default names are `position_x` and `position_y`. + +# + +from spyglass.position import PositionOutput +import spyglass.position as sgp + + +sgp.v1.TrodesPosParams.insert1( + { + "trodes_pos_params_name": "default_decoding", + "params": { + "max_LED_separation": 9.0, + "max_plausible_speed": 300.0, + "position_smoothing_duration": 0.125, + "speed_smoothing_std_dev": 0.100, + "orient_smoothing_std_dev": 0.001, + "led1_is_front": 1, + "is_upsampled": 1, + "upsampling_sampling_rate": 250, + "upsampling_interpolation_method": "linear", + }, + }, + skip_duplicates=True, +) + +trodes_s_key = { + "nwb_file_name": nwb_copy_file_name, + "interval_list_name": "pos 0 valid times", + "trodes_pos_params_name": "default_decoding", +} +sgp.v1.TrodesPosSelection.insert1( + trodes_s_key, + skip_duplicates=True, +) +sgp.v1.TrodesPosV1.populate(trodes_s_key) + +PositionOutput.TrodesPosV1 & trodes_s_key + +# + +from spyglass.decoding.v1.core import PositionGroup + +position_merge_ids = ( + PositionOutput.TrodesPosV1 + & { + "nwb_file_name": nwb_copy_file_name, + "interval_list_name": "pos 0 valid times", + "trodes_pos_params_name": "default_decoding", + } +).fetch("merge_id") + +PositionGroup().create_group( + nwb_file_name=nwb_copy_file_name, + group_name="test_group", + keys=[{"pos_merge_id": merge_id} for merge_id in position_merge_ids], +) + +PositionGroup & { + "nwb_file_name": nwb_copy_file_name, + "position_group_name": "test_group", +} +# - + +( + PositionGroup + & {"nwb_file_name": nwb_copy_file_name, "position_group_name": "test_group"} +).fetch1("position_variables") + +PositionGroup.Position & { + "nwb_file_name": nwb_copy_file_name, + "position_group_name": "test_group", +} + +# ## Decoding Model Parameters +# +# We will use the `non_local_detector` package to decode the data. This package is highly flexible and allows several different types of models to be used. In this case, we will use the `ContFragClusterlessClassifier` to decode the data. This has two discrete states: Continuous and Fragmented, which correspond to different types of movement models. To read more about this model, see: +# > Denovellis, E.L., Gillespie, A.K., Coulter, M.E., Sosa, M., Chung, J.E., Eden, U.T., and Frank, L.M. (2021). Hippocampal replay of experience at real-world speeds. eLife 10, e64505. [10.7554/eLife.64505](https://doi.org/10.7554/eLife.64505). +# +# Let's first look at the model and the default parameters: +# + +# + +from non_local_detector.models import ContFragClusterlessClassifier + +ContFragClusterlessClassifier() +# - + +# You can change these parameters like so: + +# + +from non_local_detector.models import ContFragClusterlessClassifier + +ContFragClusterlessClassifier( + clusterless_algorithm_params={ + "block_size": 10000, + "position_std": 12.0, + "waveform_std": 24.0, + }, +) +# - + +# This is how to insert the model parameters into the database: + +# + +from spyglass.decoding.v1.core import DecodingParameters + + +DecodingParameters.insert1( + { + "decoding_param_name": "contfrag_clusterless", + "decoding_params": ContFragClusterlessClassifier(), + "decoding_kwargs": dict(), + }, + skip_duplicates=True, +) + +DecodingParameters & {"decoding_param_name": "contfrag_clusterless"} +# - + +# We can retrieve these parameters and rebuild the model like so: + +# + +model_params = ( + DecodingParameters & {"decoding_param_name": "contfrag_clusterless"} +).fetch1() + +ContFragClusterlessClassifier(**model_params["decoding_params"]) +# - + +# ### 1D Decoding +# +# If you want to do 1D decoding, you will need to specify the `track_graph`, `edge_order`, and `edge_spacing` in the `environments` parameter. You can read more about these parameters in the [linearization notebook](./24_Linearization.ipynb). You can retrieve these parameters from the `TrackGraph` table if you have stored them there. These will then go into the `environments` parameter of the `ContFragClusterlessClassifier` model. + +# + +from non_local_detector.environment import Environment + +# ?Environment +# - + +# ## Decoding +# +# Now that we have grouped the data and defined the model parameters, we have finally set up the elements in tables that we need to decode the data. We now need to use the `ClusterlessDecodingSelection` to fully specify all the parameters and data that we want. +# +# This has: +# - `waveform_features_group_name`: the name of the group that contains the waveform features that we want to decode from +# - `position_group_name`: the name of the group that contains the position data that we want to decode from +# - `decoding_param_name`: the name of the decoding parameters that we want to use +# - `nwb_file_name`: the name of the NWB file that we want to decode from +# - `encoding_interval`: the interval of time that we want to train the initial model on +# - `decoding_interval`: the interval of time that we want to decode from +# - `estimate_decoding_params`: whether or not we want to estimate the decoding parameters +# +# +# The first three parameters should be familiar to you. +# +# +# ### Decoding and Encoding Intervals +# The `encoding_interval` is the interval of time that we want to train the initial model on. The `decoding_interval` is the interval of time that we want to decode from. These two intervals can be the same, but they do not have to be. For example, we may want to train the model on a long interval of time, but only decode from a short interval of time. This is useful if we want to decode from a short interval of time that is not representative of the entire session. In this case, we will train the model on a longer interval of time that is representative of the entire session. +# +# These keys come from the `IntervalList` table. We can see that the `IntervalList` table contains the `nwb_file_name` and `interval_name` that we need to specify the `encoding_interval` and `decoding_interval`. We will specify a short decoding interval called `test decoding interval` and use that to decode from. +# +# +# ### Estimating Decoding Parameters +# The last parameter is `estimate_decoding_params`. This is a boolean that specifies whether or not we want to estimate the decoding parameters. If this is `True`, then we will estimate the initial conditions and discrete transition matrix from the data. +# +# NOTE: If estimating parameters, then we need to treat times outside decoding interval as missing. this means that times outside the decoding interval will not use the spiking data and only the state transition matrix and previous time step will be used. This may or may not be desired depending on the length of this missing interval. +# + +# + +from spyglass.decoding.v1.clusterless import ClusterlessDecodingSelection + +ClusterlessDecodingSelection() + +# + +from spyglass.common import IntervalList + +IntervalList & {"nwb_file_name": nwb_copy_file_name} + +# + +decoding_interval_valid_times = [ + [1625935714.6359036, 1625935714.6359036 + 15.0] +] + +IntervalList.insert1( + { + "nwb_file_name": "mediumnwb20230802_.nwb", + "interval_list_name": "test decoding interval", + "valid_times": decoding_interval_valid_times, + }, + skip_duplicates=True, +) +# - + +# Once we have figured out the keys that we need, we can insert the `ClusterlessDecodingSelection` into the database. + +# + +selection_key = { + "waveform_features_group_name": "test_group", + "position_group_name": "test_group", + "decoding_param_name": "contfrag_clusterless", + "nwb_file_name": nwb_copy_file_name, + "encoding_interval": "pos 0 valid times", + "decoding_interval": "test decoding interval", + "estimate_decoding_params": False, +} + +ClusterlessDecodingSelection.insert1( + selection_key, + skip_duplicates=True, +) + +ClusterlessDecodingSelection & selection_key +# - + +ClusterlessDecodingSelection() + +# To run decoding, we simply populate the `ClusterlessDecodingOutput` table. This will run the decoding and insert the results into the database. We can then retrieve the results from the database. + +# + +from spyglass.decoding.v1.clusterless import ClusterlessDecodingV1 + +ClusterlessDecodingV1.populate(selection_key) +# - + +# We can now see it as an entry in the `DecodingOutput` table. + +# + +from spyglass.decoding.decoding_merge import DecodingOutput + +DecodingOutput.ClusterlessDecodingV1 & selection_key +# - + +# We can load the results of the decoding: + +decoding_results = (ClusterlessDecodingV1 & selection_key).fetch_results() +decoding_results + +# Finally, if we deleted the results, we can use the `cleanup` function to delete the results from the file system: + +DecodingOutput().cleanup() + +# ## Visualization of decoding output. +# +# The output of decoding can be challenging to visualize with static graphs, especially if the decoding is performed on 2D data. +# +# We can interactively visualize the output of decoding using the [figurl](https://github.com/flatironinstitute/figurl) package. This package allows to create a visualization of the decoding output that can be viewed in a web browser. This is useful for exploring the decoding output over time and sharing the results with others. +# +# **NOTE**: You will need a kachery cloud instance to use this feature. If you are a member of the Frank lab, you should have access to the Frank lab kachery cloud instance. If you are not a member of the Frank lab, you can create your own kachery cloud instance by following the instructions [here](https://github.com/flatironinstitute/kachery-cloud/blob/main/doc/create_kachery_zone.md). +# +# For each user, you will need to run `kachery-cloud-init` in the terminal and follow the instructions to associate your computer with your GitHub user on the kachery-cloud network. +# + +# + +# from non_local_detector.visualization import ( +# create_interactive_2D_decoding_figurl, +# ) + +# ( +# position_info, +# position_variable_names, +# ) = ClusterlessDecodingV1.fetch_position_info(selection_key) +# results_time = decoding_results.acausal_posterior.isel(intervals=0).time.values +# position_info = position_info.loc[results_time[0] : results_time[-1]] + +# env = ClusterlessDecodingV1.fetch_environments(selection_key)[0] +# spike_times, _ = ClusterlessDecodingV1.fetch_spike_data(selection_key) + + +# create_interactive_2D_decoding_figurl( +# position_time=position_info.index.to_numpy(), +# position=position_info[position_variable_names], +# env=env, +# results=decoding_results, +# posterior=decoding_results.acausal_posterior.isel(intervals=0) +# .unstack("state_bins") +# .sum("state"), +# spike_times=spike_times, +# head_dir=position_info["orientation"], +# speed=position_info["speed"], +# ) +# - + +# ## GPUs +# We can use GPUs for decoding which will result in a significant speedup. This is achieved using the [jax](https://jax.readthedocs.io/en/latest/) package. +# +# ### Ensuring jax can find a GPU +# Assuming you've set up a GPU, we can use `jax.devices()` to make sure the decoding code can see the GPU. If a GPU is available, it will be listed. +# +# In the following instance, we do not have a GPU: + +# + +import jax + +jax.devices() +# - + +# ### Selecting a GPU +# If you do have multiple GPUs, you can use the `jax` package to set the device (GPU) that you want to use. For example, if you want to use the second GPU, you can use the following code (uncomment first): + +# + +# device_id = 2 +# device = jax.devices()[device_id] +# jax.config.update("jax_default_device", device) +# device +# - + +# ### Monitoring GPU Usage +# +# You can see which GPUs are occupied (if you have multiple GPUs) by running the command `nvidia-smi` in +# a terminal (or `!nvidia-smi` in a notebook). Pick a GPU with low memory usage. +# +# We can monitor GPU use with the terminal command `watch -n 0.1 nvidia-smi`, will +# update `nvidia-smi` every 100 ms. This won't work in a notebook, as it won't +# display the updates. +# +# Other ways to monitor GPU usage are: +# +# - A +# [jupyter widget by nvidia](https://github.com/rapidsai/jupyterlab-nvdashboard) +# to monitor GPU usage in the notebook +# - A [terminal program](https://github.com/peci1/nvidia-htop) like nvidia-smi +# with more information about which GPUs are being utilized and by whom. diff --git a/notebooks/py_scripts/42_Decoding_SortedSpikes.py b/notebooks/py_scripts/42_Decoding_SortedSpikes.py new file mode 100644 index 000000000..74576b553 --- /dev/null +++ b/notebooks/py_scripts/42_Decoding_SortedSpikes.py @@ -0,0 +1,182 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# jupytext_version: 1.15.2 +# kernelspec: +# display_name: spyglass +# language: python +# name: python3 +# --- + +# # Sorted Spikes Decoding +# +# The mechanics of decoding with sorted spikes are largely similar to those of decoding with unsorted spikes. You should familiarize yourself with the [clusterless decoding tutorial](./42_Decoding_Clusterless.ipynb) before proceeding with this one. +# +# The elements we will need to decode with sorted spikes are: +# - `PositionGroup` +# - `SortedSpikesGroup` +# - `DecodingParameters` +# - `encoding_interval` +# - `decoding_interval` +# +# This time, instead of extracting waveform features, we can proceed directly from the SpikeSortingOutput table to specify which units we want to decode. The rest of the decoding process is the same as before. +# +# + +# + +from pathlib import Path +import datajoint as dj + +dj.config.load( + Path("../dj_local_conf.json").absolute() +) # load config for database connection info +# - + +# ## SortedSpikesGroup +# +# `SortedSpikesGroup` is a child table of `SpikeSortingOutput` in the spikesorting pipeline. It allows us to group the spikesorting results from multiple +# sources (e.g. multiple tetrode groups or intervals) into a single entry. Here we will group together the spiking of multiple tetrode groups to use for decoding. +# +# +# This table allows us filter units by their annotation labels from curation (e.g only include units labeled "good", exclude units labeled "noise") by defining parameters from `UnitSelectionParams`. When accessing data through `SortedSpikesGroup` the table will include only units with at least one label in `include_labels` and no labels in `exclude_labels`. We can look at those here: +# + +# + +from spyglass.spikesorting.analysis.v1.group import UnitSelectionParams + +UnitSelectionParams().insert_default() + +# look at the filter set we'll use here +unit_filter_params_name = "default_exclusion" +print( + ( + UnitSelectionParams() + & {"unit_filter_params_name": unit_filter_params_name} + ).fetch1() +) +# look at full table +UnitSelectionParams() +# - + +# Now we can make our sorted spikes group with this unit selection parameter + +# + +from spyglass.spikesorting.spikesorting_merge import SpikeSortingOutput +import spyglass.spikesorting.v1 as sgs + +nwb_copy_file_name = "mediumnwb20230802_.nwb" + +sorter_keys = { + "nwb_file_name": nwb_copy_file_name, + "sorter": "mountainsort4", + "curation_id": 1, +} +# check the set of sorting we'll use +( + sgs.SpikeSortingSelection & sorter_keys +) * SpikeSortingOutput.CurationV1 & sorter_keys +# - + +# Finding the merge id's corresponding to an interpretable restriction such as `merge_id` or `interval_list` can require several join steps with upstream tables. To simplify this process we can use the included helper function `SpikeSortingOutput().get_restricted_merge_ids()` to perform the necessary joins and return the matching merge id's + +# + +# get the merge_ids for the selected sorting +spikesorting_merge_ids = SpikeSortingOutput().get_restricted_merge_ids( + sorter_keys, restrict_by_artifact=False +) + +# create a new sorted spikes group +unit_filter_params_name = "default_exclusion" +SortedSpikesGroup().create_group( + group_name="test_group", + nwb_file_name=nwb_copy_file_name, + keys=[ + {"spikesorting_merge_id": merge_id} + for merge_id in spikesorting_merge_ids + ], + unit_filter_params_name=unit_filter_params_name, +) +# check the new group +SortedSpikesGroup & { + "nwb_file_name": nwb_copy_file_name, + "sorted_spikes_group_name": "test_group", +} +# - + +# look at the sorting within the group we just made +SortedSpikesGroup.Units & { + "nwb_file_name": nwb_copy_file_name, + "sorted_spikes_group_name": "test_group", + "unit_filter_params_name": unit_filter_params_name, +} + +# ## Model parameters +# +# As before we can specify the model parameters. The only difference is that we will use the `ContFragSortedSpikesClassifier` instead of the `ContFragClusterlessClassifier`. + +# + +from spyglass.decoding.v1.core import DecodingParameters +from non_local_detector.models import ContFragSortedSpikesClassifier + + +DecodingParameters.insert1( + { + "decoding_param_name": "contfrag_sorted", + "decoding_params": ContFragSortedSpikesClassifier(), + "decoding_kwargs": dict(), + }, + skip_duplicates=True, +) + +DecodingParameters() +# - + +# ### 1D Decoding +# +# As in the clusterless notebook, we can decode 1D position if we specify the `track_graph`, `edge_order`, and `edge_spacing` parameters in the `Environment` class constructor. See the [clusterless decoding tutorial](./42_Decoding_Clusterless.ipynb) for more details. + +# ## Decoding +# +# Now we can decode the position using the sorted spikes using the `SortedSpikesDecodingSelection` table. Here we assume that `PositionGroup` has been specified as in the clusterless decoding tutorial. + +# + +selection_key = { + "sorted_spikes_group_name": "test_group", + "unit_filter_params_name": "default_exclusion", + "position_group_name": "test_group", + "decoding_param_name": "contfrag_sorted", + "nwb_file_name": "mediumnwb20230802_.nwb", + "encoding_interval": "pos 0 valid times", + "decoding_interval": "test decoding interval", + "estimate_decoding_params": False, +} + +from spyglass.decoding import SortedSpikesDecodingSelection + +SortedSpikesDecodingSelection.insert1( + selection_key, + skip_duplicates=True, +) + +# + +from spyglass.decoding.v1.sorted_spikes import SortedSpikesDecodingV1 + +SortedSpikesDecodingV1.populate(selection_key) +# - + +# We verify that the results have been inserted into the `DecodingOutput` merge table. + +# + +from spyglass.decoding.decoding_merge import DecodingOutput + +DecodingOutput.SortedSpikesDecodingV1 & selection_key +# - + +# We can load the results as before: + +results = (SortedSpikesDecodingV1 & selection_key).fetch_results() +results diff --git a/src/spyglass/spikesorting/spikesorting_merge.py b/src/spyglass/spikesorting/spikesorting_merge.py index ee41e1e04..fda83cdd7 100644 --- a/src/spyglass/spikesorting/spikesorting_merge.py +++ b/src/spyglass/spikesorting/spikesorting_merge.py @@ -7,9 +7,16 @@ from spyglass.spikesorting.v0.spikesorting_curation import ( # noqa: F401 CuratedSpikeSorting, ) -from spyglass.spikesorting.v1.curation import CurationV1 # noqa: F401 +from spyglass.spikesorting.v1 import ( # noqa: F401 + ArtifactDetectionSelection, + CurationV1, + MetricCurationSelection, + SpikeSortingRecordingSelection, + SpikeSortingSelection, +) from spyglass.utils.dj_merge_tables import _Merge from spyglass.utils.dj_mixin import SpyglassMixin +from spyglass.utils.logging import logger schema = dj.schema("spikesorting_merge") @@ -50,6 +57,93 @@ class CuratedSpikeSorting(SpyglassMixin, dj.Part): # noqa: F811 -> CuratedSpikeSorting """ + def get_restricted_merge_ids( + self, + key: dict, + sources: list = ["v0", "v1"], + restrict_by_artifact: bool = True, + as_dict: bool = False, + ): + """Helper function to get merge ids for a given interpretable key + + Parameters + ---------- + key : dict + restriction for any stage of the spikesorting pipeline + sources : list, optional + list of sources to restrict to + restrict_by_artifact : bool, optional + whether to restrict by artifact rather than original interval name. Relevant to v1 pipeline, by default True + as_dict : bool, optional + whether to return merge_ids as a list of dictionaries, by default False + + Returns + ------- + merge_ids : list + list of merge ids from the restricted sources + """ + merge_ids = [] + + if "v1" in sources: + key_v1 = key.copy() + # Recording restriction + table = SpikeSortingRecordingSelection() & key_v1 + if restrict_by_artifact: + # Artifact restriction + table_artifact = ArtifactDetectionSelection * table & key_v1 + artifact_restrict = table_artifact.proj( + interval_list_name="artifact_id" + ).fetch(as_dict=True) + # convert interval_list_name from artifact uuid to string + for key_i in artifact_restrict: + key_i["interval_list_name"] = str( + key_i["interval_list_name"] + ) + if "interval_list_name" in key_v1: + key_v1.pop( + "interval_list_name" + ) # pop the interval list since artifact intervals are now the restriction + # Spike sorting restriction + table = ( + (SpikeSortingSelection() * table.proj()) + & artifact_restrict + & key_v1 + ) + else: + # use the supplied interval to restrict + table = (SpikeSortingSelection() * table.proj()) & key_v1 + # Metric Curation restriction + headings = MetricCurationSelection.heading.names + headings.pop( + headings.index("curation_id") + ) # this is the parent curation id of the final entry. dont restrict by this name here + # metric curation is an optional process. only do this join if the headings are present in the key + if any([heading in key_v1 for heading in headings]): + table = ( + MetricCurationSelection().proj(*headings) * table + ) & key_v1 + # get curations + table = (CurationV1() * table) & key_v1 + table = SpikeSortingOutput().CurationV1() & table + merge_ids.extend(table.fetch("merge_id", as_dict=as_dict)) + + if "v0" in sources: + if restrict_by_artifact: + logger.warning( + 'V0 requires artifact restrict. Ignoring "restrict_by_artifact" flag.' + ) + key_v0 = key.copy() + if "sort_interval" not in key_v0 and "interval_list_name" in key_v0: + key_v0["sort_interval"] = key_v0["interval_list_name"] + _ = key_v0.pop("interval_list_name") + merge_ids.extend( + (SpikeSortingOutput.CuratedSpikeSorting() & key_v0).fetch( + "merge_id", as_dict=as_dict + ) + ) + + return merge_ids + @classmethod def get_recording(cls, key): """get the recording associated with a spike sorting output""" @@ -68,6 +162,27 @@ def get_sorting(cls, key): query = source_table & cls.merge_get_part(key) return query.get_sorting(query.fetch("KEY")) + @classmethod + def get_sort_group_info(cls, key): + """get the sort group info associated with a spike sorting output + (e.g. electrode location, brain region, etc.) + Parameters: + ----------- + key : dict + dictionary specifying the restriction (note: multi-source not currently supported) + Returns: + ------- + sort_group_info : Table + Table linking a merge id to information about the electrode group. + """ + source_table = source_class_dict[ + to_camel_case(cls.merge_get_parent(key).table_name) + ] + part_table = cls.merge_get_part(key) + query = source_table & part_table + sort_group_info = source_table.get_sort_group_info(query.fetch("KEY")) + return part_table * sort_group_info # join the info with merge id's + def get_spike_times(self, key): spike_times = [] for nwb_file in self.fetch_nwb(key): diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index d960bb796..dab6ed740 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -6,11 +6,11 @@ import warnings from pathlib import Path from typing import List -from packaging import version import datajoint as dj import numpy as np import spikeinterface as si +from packaging import version if version.parse(si.__version__) < version.parse("0.99.1"): raise ImportError( @@ -20,6 +20,7 @@ import spikeinterface.preprocessing as sip import spikeinterface.qualitymetrics as sq +from spyglass.common import BrainRegion, Electrode from spyglass.common.common_interval import IntervalList from spyglass.common.common_nwbfile import AnalysisNwbfile from spyglass.settings import waveforms_dir @@ -32,6 +33,7 @@ ) from spyglass.utils import SpyglassMixin, logger +from .spikesorting_recording import SortGroup from .spikesorting_sorting import SpikeSorting schema = dj.schema("spikesorting_curation") @@ -1034,6 +1036,37 @@ def get_sorting(cls, key): sorting_key = (cls & key).fetch1("KEY") return Curation.get_curated_sorting(sorting_key) + @classmethod + def get_sort_group_info(cls, key): + """Returns the sort group information for the curation + (e.g. brain region, electrode placement, etc.) + + Parameters + ---------- + key : dict + restriction on CuratedSpikeSorting table + + Returns + ------- + sort_group_info : Table + Table with information about the sort groups + """ + electrode_restrict_list = [] + for entry in cls & key: + # Just take one electrode entry per sort group + electrode_restrict_list.extend( + ((SortGroup.SortGroupElectrode() & entry) * Electrode).fetch( + limit=1 + ) + ) + # Run joins with the tables with info and return + sort_group_info = ( + (Electrode & electrode_restrict_list) + * (cls & key) + * SortGroup.SortGroupElectrode() + ) * BrainRegion() + return sort_group_info + @schema class UnitInclusionParameters(SpyglassMixin, dj.Manual): diff --git a/src/spyglass/spikesorting/v0/spikesorting_sorting.py b/src/spyglass/spikesorting/v0/spikesorting_sorting.py index 7e4743b93..9e4d84a61 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_sorting.py +++ b/src/spyglass/spikesorting/v0/spikesorting_sorting.py @@ -204,6 +204,10 @@ def make(self, key: dict): sorter_params.pop("tempdir", None) sorter_params.pop("whiten", None) sorter_params.pop("outputs", None) + if "local_radius_um" in sorter_params: + sorter_params["radius_um"] = sorter_params.pop( + "local_radius_um" + ) # correct existing parameter sets for spikeinterface>=0.99.1 # Detect peaks for clusterless decoding detected_spikes = detect_peaks(recording, **sorter_params) diff --git a/src/spyglass/spikesorting/v1/curation.py b/src/spyglass/spikesorting/v1/curation.py index 5e862dd90..ac9c78f2b 100644 --- a/src/spyglass/spikesorting/v1/curation.py +++ b/src/spyglass/spikesorting/v1/curation.py @@ -7,9 +7,14 @@ import spikeinterface.curation as sc import spikeinterface.extractors as se +from spyglass.common import BrainRegion, Electrode from spyglass.common.common_ephys import Raw from spyglass.common.common_nwbfile import AnalysisNwbfile -from spyglass.spikesorting.v1.recording import SpikeSortingRecording +from spyglass.spikesorting.v1.recording import ( + SortGroup, + SpikeSortingRecording, + SpikeSortingRecordingSelection, +) from spyglass.spikesorting.v1.sorting import SpikeSorting, SpikeSortingSelection from spyglass.utils.dj_mixin import SpyglassMixin @@ -260,6 +265,42 @@ def get_merged_sorting(cls, key: dict) -> si.BaseSorting: else: return si_sorting + @classmethod + def get_sort_group_info(cls, key: dict) -> dj.Table: + """Returns the sort group information for the curation + (e.g. brain region, electrode placement, etc.) + + Parameters + ---------- + key : dict + restriction on CuratedSpikeSorting table + + Returns + ------- + sort_group_info : Table + Table with information about the sort groups + """ + table = ( + (cls & key) * SpikeSortingSelection() + ) * SpikeSortingRecordingSelection().proj( + "recording_id", "sort_group_id" + ) + electrode_restrict_list = [] + for entry in table: + # pull just one electrode from each sort group for info + electrode_restrict_list.extend( + ((SortGroup.SortGroupElectrode() & entry) * Electrode).fetch( + limit=1 + ) + ) + + sort_group_info = ( + (Electrode & electrode_restrict_list) + * table + * SortGroup.SortGroupElectrode() + ) * BrainRegion() + return (cls & key).proj() * sort_group_info + def _write_sorting_to_nwb_with_curation( sorting_id: str, diff --git a/src/spyglass/spikesorting/v1/sorting.py b/src/spyglass/spikesorting/v1/sorting.py index b650c1c5f..9e3d19d6a 100644 --- a/src/spyglass/spikesorting/v1/sorting.py +++ b/src/spyglass/spikesorting/v1/sorting.py @@ -232,6 +232,10 @@ def make(self, key: dict): sorter_params.pop("tempdir", None) sorter_params.pop("whiten", None) sorter_params.pop("outputs", None) + if "local_radius_um" in sorter_params: + sorter_params["radius_um"] = sorter_params.pop( + "local_radius_um" + ) # correct existing parameter sets for spikeinterface>=0.99.1 # Detect peaks for clusterless decoding detected_spikes = detect_peaks(recording, **sorter_params)