Tested DLC multicam pipeline (#841)

* Fault-permit insert and remove mutual exclusivity protections on Merge (#824)

* #719, #804, #212

* #768

* Add merge delete and populate

* Changes following PR review @edeno

* Replace delayed import of ImportedSpikeSorting

* Update CITATION.cff (#826)

* Update CITATION.cff

* Update change log

* Update ref

* Add MUA notebook and fix numbering.

* Only apply include labels filter if include_labels not empty (#827)

* dont skip unit if include_labels list is empty

* update check for np array size

* gh-actions docs fixes (#828)

* Update 'latest' in docs deploy

* Docs bugfix. Rename Link action, incorporate cspell.

* MUA as own heading

* Update README.md

* Fix citation

* Fix citation

* include all relevant restrictions on video file

* Proposed structure for user roles. (#832)

* Add roles

* Remove use of unix user group. Add note for retroactive role assign

* Add docs on roles and external tables. Reduce key length

* Fix test for update of position tools (#835)

Related to single LED halving the data bug

* fix build error in mamba and restriction for dlc

* flush stdout before converting mp4

* Fix notebook name (#840)

* remove deprecated yaml.safe_load function

* Fix test

* replace deprecated yaml.safe_load function

* only call no_transaction_make if video key not present

---------

Co-authored-by: Chris Brozdowski <Chris.Broz@ucsf.edu>
Co-authored-by: Eric Denovellis <edeno@users.noreply.github.com>
Co-authored-by: Eric Denovellis <edeno@bu.edu>

environment_dlc.yml

@@ -22,7 +22,6 @@ dependencies:
   - libgcc   # dlc-only
   - matplotlib
   - non_local_detector
-  - numpy<1.24
   - pip>=20.2.*
   - position_tools
   - pybind11  # req by mountainsort4 -> isosplit5
@@ -47,4 +46,5 @@ dependencies:
       - pynwb>=2.2.0,<3
       - sortingview>=0.11
       - spikeinterface>=0.98.2,<0.99
+      - tensorflow<=2.12  # dlc-only
       - .[dlc]

src/spyglass/position/v1/dlc_reader.py

@@ -114,7 +114,8 @@ def pkl(self):
     def yml(self):
         if self._yml is None:
             with open(self.yml_path, "rb") as f:
-                self._yml = yaml.safe_load(f)
+                safe_yaml = yaml.YAML(typ="safe", pure=True)
+                self._yml = safe_yaml.load(f)
         return self._yml
 
     @property

src/spyglass/position/v1/dlc_utils.py

@@ -6,6 +6,7 @@
 import pathlib
 import pwd
 import subprocess
+import sys
 from collections import abc
 from contextlib import redirect_stdout
 from itertools import groupby
@@ -419,8 +420,10 @@ def get_video_path(key):
     """
     import pynwb
 
-    vf_key = {"nwb_file_name": key["nwb_file_name"], "epoch": key["epoch"]}
-    VideoFile()._no_transaction_make(vf_key, verbose=False)
+    valid_fields = VideoFile.fetch().dtype.fields.keys()
+    vf_key = {k: val for k, val in key.items() if k in valid_fields}
+    if not VideoFile & vf_key:
+        VideoFile()._no_transaction_make(vf_key, verbose=False)
     video_query = VideoFile & vf_key
 
     if len(video_query) != 1:
@@ -537,17 +540,23 @@ def _convert_mp4(
         "copy",
         f"{dest_path.as_posix()}",
     ]
-    try:
-        convert_process = subprocess.Popen(
-            convert_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-        )
-    except subprocess.CalledProcessError as err:
-        raise RuntimeError(
-            f"command {err.cmd} return with error (code {err.returncode}): {err.output}"
-        ) from err
-    out, _ = convert_process.communicate()
-    print(out.decode("utf-8"))
-    print(f"finished converting {filename}")
+    if dest_path.exists():
+        print(f"{dest_path} already exists, skipping conversion")
+    else:
+        try:
+            sys.stdout.flush()
+            convert_process = subprocess.Popen(
+                convert_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+        except subprocess.CalledProcessError as err:
+            raise RuntimeError(
+                f"command {err.cmd} return with error (code {err.returncode}): {err.output}"
+            ) from err
+        out, _ = convert_process.communicate()
+        print(out.decode("utf-8"))
+        print(f"finished converting {filename}")
     print(
         f"Checking that number of packets match between {orig_filename} and {dest_filename}"
     )

src/spyglass/position/v1/position_dlc_model.py

@@ -196,7 +196,8 @@ def make(self, key):
             raise OSError(f"config_path {config_path} does not exist.")
         if config_path.suffix in (".yml", ".yaml"):
             with open(config_path, "rb") as f:
-                dlc_config = yaml.safe_load(f)
+                safe_yaml = yaml.YAML(typ="safe", pure=True)
+                dlc_config = safe_yaml.load(f)
             if isinstance(params["params"], dict):
                 dlc_config.update(params["params"])
                 del params["params"]