Merge branch 'master' into moseq

LorenFrankLab · Dec 20, 2024 · ccf370f · ccf370f
2 parents fe3c6d4 + 75ad067
commit ccf370f
Show file tree

Hide file tree

Showing 228 changed files with 14,366 additions and 8,793 deletions.
diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml
@@ -2,7 +2,7 @@ name: Publish docs
 on:
   push:
     tags: # See PEP 440 for valid version format
-      - "*.*.*" # For docs bump, use X.X.XaX
+      - "*.*.*" # For docs bump, use workflow_dispatch
     branches:
       - test_branch
   workflow_dispatch: # Manually trigger with 'Run workflow' button
@@ -23,13 +23,17 @@ jobs:
       issues: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           fetch-depth: 0
 
+      - name: Git describe # Get tags
+        id: ghd # see Deploy below. Will fail if no tags on branch
+        uses: proudust/gh-describe@v2
+
       - name: Set up Python runtime
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -40,8 +44,8 @@ jobs:
           git config user.name 'github-actions[bot]' && git config user.email 'github-actions[bot]@users.noreply.github.com'
 
       - name: Deploy
-        run: |
-          FULL_VERSION=${{ github.ref_name }}
+        run: | # github.ref_name is branch name if dispatch
+          FULL_VERSION=${{ steps.ghd.outputs.tag }}
           export MAJOR_VERSION=${FULL_VERSION:0:3}
           echo "OWNER: ${REPO_OWNER}. BUILD: ${MAJOR_VERSION}"
           bash ./docs/build-docs.sh push $REPO_OWNER

diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml
@@ -2,11 +2,13 @@ name: Tests
 
 on:
   push:
-    branches: 
+    branches:
       - '!test_branch'
       - '!documentation'
   schedule:  # once a day at midnight UTC
     - cron: '0 0 * * *'
+  pull_request: # requires approval for first-time contributors
+    types: [synchronize, opened, reopened, labeled]
   workflow_dispatch: # Manually trigger with 'Run workflow' button
 
 concurrency: # Replace Cancel Workflow Action
@@ -22,8 +24,6 @@ jobs:
     env:
       OS: ubuntu-latest
       PYTHON: '3.9'
-      UCSF_BOX_TOKEN: ${{ secrets.UCSF_BOX_TOKEN }} # for download and testing
-      UCSF_BOX_USER: ${{ secrets.UCSF_BOX_USER }}
     services:
       mysql:
         image: datajoint/mysql:8.0
@@ -45,7 +45,6 @@ jobs:
       with:
         activate-environment: spyglass
         environment-file: environment.yml
-        miniforge-variant: Mambaforge
         miniforge-version: latest
         use-mamba: true
     - name: Install apt dependencies
@@ -54,27 +53,27 @@ jobs:
         sudo apt-get install mysql-client libmysqlclient-dev libgirepository1.0-dev -y
         sudo apt-get install ffmpeg libsm6 libxext6 -y # non-dlc position deps
     - name: Run pip install for test deps
-      run: | 
+      run: |
         pip install --quiet .[test]
     - name: Download data
       env:
-        BASEURL: ftps://ftp.box.com/trodes_to_nwb_test_data/
-        NWBFILE: minirec20230622.nwb # Relative to Base URL
-        VID_ONE: 20230622_sample_01_a1/20230622_sample_01_a1.1.h264
-        VID_TWO: 20230622_sample_02_a1/20230622_sample_02_a1.1.h264
-        RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ 
-        VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ 
+        BASEURL: https://ucsf.box.com/shared/static/
+        NWB_URL: k3sgql6z475oia848q1rgms4zdh4rkjn.nwb
+        VID1URL: ykep8ek4ogad20wz4p0vuyuqfo60cv3w.h264
+        VID2URL: d2jjk0y565ru75xqojio3hymmehzr5he.h264
+        NWBFILE: minirec20230622.nwb
+        VID_ONE: 20230622_minirec_01_s1.1.h264
+        VID_TWO: 20230622_minirec_02_s2.1.h264
+        RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/
+        VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/
       run: |
         mkdir -p $RAW_DIR $VID_DIR
-        wget_opts() { # Declare func with download options
-          wget \
-            --recursive --no-verbose --no-host-directories --no-directories \
-            --user "$UCSF_BOX_USER" --password "$UCSF_BOX_TOKEN" \
-            -P "$1" "$BASEURL""$2"
+        curl_opts() { # Declare func with download options
+          curl -L --output "$1""$2" "$BASEURL""$3"
         }
-        wget_opts $RAW_DIR $NWBFILE
-        wget_opts $VID_DIR $VID_ONE
-        wget_opts $VID_DIR $VID_TWO
+        curl_opts $RAW_DIR $NWBFILE $NWB_URL
+        curl_opts $VID_DIR $VID_ONE $VID1URL
+        curl_opts $VID_DIR $VID_TWO $VID2URL
     - name: Run tests
       run: |
-        pytest --no-docker --no-dlc 
+        pytest --no-docker --no-dlc tests/
diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml
@@ -27,7 +27,8 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: 3.9
-      - run: pip install --upgrade build twine
+      - run: |
+          pip install --upgrade build twine
       - name: Build sdist and wheel
         run: python -m build
       - run: twine check dist/*
@@ -48,6 +49,7 @@ jobs:
     needs: [build]
     strategy:
       matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         package: ['wheel', 'sdist', 'archive']
     steps:
       - name: Download sdist and wheel artifacts
@@ -62,13 +64,15 @@ jobs:
         with:
           name: archive
           path: archive/
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
-      - name: Update pip
-        run: pip install --upgrade pip
+      - name: Install build dependencies
+        run: |
+          pip install --upgrade setuptools wheel
+          pip install --upgrade pip
       - name: Install wheel
         if: matrix.package == 'wheel'
         run: pip install dist/*.whl

diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ spyglass.code-workspace
 mountainsort4_output/
 .idea/
 mysql_config
+memray*
 
 # Notebooks
 *.ipynb
@@ -60,6 +61,7 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 tests/_data/*
+wget-log*
 
 # Translations
 *.mo
@@ -128,6 +130,7 @@ dmypy.json
 .pyre/
 
 # Test Data Files
+tests/_data/*
 *.dat
 *.mda
 *.rec
@@ -137,6 +140,7 @@ dmypy.json
 *.videoTimeStamps
 *.cameraHWSync
 *.stateScriptLog
+tests/_data/*
 
 *.nwb
 *.DS_Store

diff --git a/.markdownlint.yaml b/.markdownlint.yaml
@@ -1,8 +1,10 @@
 # https://github.com/DavidAnson/markdownlint
 # https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md
-MD007: false # permit indenting 4 spaces instead of 2
+MD007: # permit indenting 4 spaces instead of 2
+  indent: 4
+  start_indent: 4
 MD013:
-    line_length: "80" # Line length limits
+    line_length: 80 # Line length limits
     tables: false # disable for tables
     code_blocks: false # disable for code blocks
 MD025: false # permit adjacent headings

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,87 @@
 # Change Log
 
-## [0.5.3] (Unreleased)
+## [0.5.4] (December 20, 2024)
 
-### Release Notes
+### Infrastructure
+
+- Disable populate transaction protection for long-populating tables #1066,
+    #1108, #1172, #1187
+- Add docstrings to all public methods #1076
+- Update DataJoint to 0.14.2 #1081
+- Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126
+- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116,
+    #1137, #1162
+- Fix bool settings imported from dj config file #1117
+- Allow definition of tasks and new probe entries from config #1074, #1120,
+    #1179
+- Enforce match between ingested nwb probe geometry and existing table entry
+    #1074
+- Update DataJoint install and password instructions #1131
+- Fix dandi upload process for nwb's with video or linked objects #1095, #1151
+- Minor docs fixes #1145
+- Test fixes
+    - Remove stored hashes from pytests #1152
+    - Remove mambaforge from tests #1153
+    - Remove debug statement #1164
+    - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169
+    - Initialize tables in pytests #1181
+    - Download test data without credentials, trigger on approved PRs #1180
+    - Add coverage of decoding pipeline to pytests #1155
+- Allow python \< 3.13 #1169
+- Remove numpy version restriction #1169
+- Merge table delete removes orphaned master entries #1164
+- Edit `merge_fetch` to expect positional before keyword arguments #1181
+- Allow part restriction `SpyglassMixinPart.delete` #1192
+- Move cleanup of `IntervalList` orphan entries to cron job cleanup process
+    #1195
+- Add mixin method `get_fully_defined_key` #1198
+
+### Pipelines
+
+- Common
+
+    - Drop `SessionGroup` table #1106
+    - Improve electrodes import efficiency #1125
+    - Fix logger method call in `common_task` #1132
+    - Export fixes #1164
+        - Allow `get_abs_path` to add selection entry. #1164
+        - Log restrictions and joins. #1164
+        - Check if querying table inherits mixin in `fetch_nwb`. #1192, #1201
+        - Ensure externals entries before adding to export. #1192
+    - Error specificity in `LabMemberInfo` #1192
+
+- Decoding
+
+    - Fix edge case errors in spike time loading #1083
+    - Allow fetch of partial key from `DecodingParameters` #1198
+    - Allow data fetching with partial but unique key #1198
+
+- Linearization
+
+    - Add edge_map parameter to LinearizedPositionV1 #1091
+
+- Position
+
+    - Fix video directory bug in `DLCPoseEstimationSelection` #1103
+    - Restore #973, allow DLC without position tracking #1100
+    - Minor fix to `DLCCentroid` make function order #1112, #1148
+    - Video creator tools:
+        - Pass output path as string to `cv2.VideoWriter` #1150
+        - Set `DLCPosVideo` default processor to `matplotlib`, remove support for
+            `open-cv` #1168
+        - `VideoMaker` class to process frames in multithreaded batches #1168, #1174
+        - `TrodesPosVideo` updates for `matplotlib` processor #1174
+    - User prompt if ambiguous insert in `DLCModelSource` #1192
 
-<!-- Running draft to be removed immediately prior to release. -->
+- Spike Sorting
+
+    - Fix bug in `get_group_by_shank` #1096
+    - Fix bug in `_compute_metric` #1099
+    - Fix bug in `insert_curation` returned key #1114
+    - Fix handling of waveform extraction sparse parameter #1132
+    - Limit Artifact detection intervals to valid times #1196
+
+## [0.5.3] (August 27, 2024)
 
 ### Infrastructure
 
@@ -19,23 +96,79 @@
 - Add pytests for position pipeline, various `test_mode` exceptions #966
 - Migrate `pip` dependencies from `environment.yml`s to `pyproject.toml` #966
 - Add documentation for common error messages #997
+- Expand `delete_downstream_merge` -> `delete_downstream_parts`. #1002
+- `cautious_delete` now ...
+    - Checks `IntervalList` and externals tables. #1002
+    - Ends early if called on empty table. #1055
+- Allow mixin tables with parallelization in `make` to run populate with
+    `processes > 1` #1001, #1052, #1068
+- Speed up fetch_nwb calls through merge tables #1017
+- Allow `ModuleNotFoundError` or `ImportError` for optional dependencies #1023
+- Ensure integrity of group tables #1026
+- Convert list of LFP artifact removed interval list to array #1046
+- Merge duplicate functions in decoding and spikesorting #1050, #1053, #1062,
+    #1066, #1069
+- Reivise docs organization.
+    - Misc -> Features/ForDevelopers. #1029
+    - Installation instructions -> Setup notebook. #1029
+- Migrate SQL export tools to `utils` to support exporting `DandiPath` #1048
+- Add tool for checking threads for metadata locks on a table #1063
+- Use peripheral tables as fallback in `TableChains` #1035
+- Ignore non-Spyglass tables during descendant check for `part_masters` #1035
 
 ### Pipelines
 
 - Common
+
     - `PositionVideo` table now inserts into self after `make` #966
+    - Don't insert lab member when creating lab team #983
     - Files created by `AnalysisNwbfile.create()` receive new object_id #999
-- Decoding: Default values for classes on `ImportError` #966
-- DLC
+    - Remove unused `ElectrodeBrainRegion` table #1003
+    - Files created by `AnalysisNwbfile.create()` receive new object_id #999,
+        #1004
+    - Remove redundant calls to tables in `populate_all_common` #870
+    - Improve logging clarity in `populate_all_common` #870
+    - `PositionIntervalMap` now inserts null entries for missing intervals #870
+    - `AnalysisFileLog` now truncates table names that exceed field length #1021
+    - Disable logging with `AnalysisFileLog` #1024
+    - Remove `common_ripple` schema #1061
+
+- Decoding:
+
+    - Default values for classes on `ImportError` #966
+    - Add option to upsample data rate in `PositionGroup` #1008
+    - Avoid interpolating over large `nan` intervals in position #1033
+    - Minor code calling corrections #1073
+
+- Position
+
     - Allow dlc without pre-existing tracking data #973, #975
     - Raise `KeyError` for missing input parameters across helper funcs #966
     - `DLCPosVideo` table now inserts into self after `make` #966
-- Common
-    - Don't insert lab member when creating lab team #983
+    - Remove unused `PositionVideoSelection` and `PositionVideo` tables #1003
+    - Fix SQL query error in `DLCPosV1.fetch_nwb` #1011
+    - Add keyword args to all calls of `convert_to_pixels` #870
+    - Unify `make_video` logic across `DLCPosVideo` and `TrodesVideo` #870
+    - Replace `OutputLogger` context manager with decorator #870
+    - Rename `check_videofile` -> `find_mp4` and `get_video_path` ->
+        `get_video_info` to reflect actual use #870
+    - Fix `red_led_bisector` `np.nan` handling issue from #870. Fixed in #1034
+    - Fix `one_pt_centoid` `np.nan` handling issue from #870. Fixed in #1034
+
 - Spikesorting
+
     - Allow user to set smoothing timescale in `SortedSpikesGroup.get_firing_rate`
         #994
     - Update docstrings #996
+    - Remove unused `UnitInclusionParameters` table from `spikesorting.v0` #1003
+    - Fix bug in identification of artifact samples to be zeroed out in
+        `spikesorting.v1.SpikeSorting` #1009
+    - Remove deprecated dependencies on kachery_client #1014
+    - Add `UnitAnnotation` table and naming convention for units #1027, #1052
+    - Set `sparse` parameter to waveform extraction step in `spikesorting.v1`
+        #1039
+    - Efficiency improvement to `v0.Curation.insert_curation` #1072
+    - Add pytests for `spikesorting.v1` #1078
 
 ## [0.5.2] (April 22, 2024)
 
@@ -79,11 +212,15 @@
 
 ### Pipelines
 
+- Common:
+    - Add ActivityLog to `common_usage` to track unreferenced utilities. #870
 - Position:
     - Fixes to `environment-dlc.yml` restricting tensortflow #834
     - Video restriction for multicamera epochs #834
     - Fixes to `_convert_mp4` #834
     - Replace deprecated calls to `yaml.safe_load()` #834
+    - Refactoring to reduce redundancy #870
+    - Migrate `OutputLogger` behavior to decorator #870
 - Spikesorting:
     - Increase`spikeinterface` version to >=0.99.1, \<0.100 #852
     - Bug fix in single artifact interval edge case #859
@@ -265,3 +402,4 @@
 [0.5.1]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.1
 [0.5.2]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.2
 [0.5.3]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.3
+[0.5.4]: https://github.com/LorenFrankLab/spyglass/releases/tag/0.5.4