Tested DLC multicam pipeline

environment_dlc.yml

@@ -22,7 +22,6 @@ dependencies:
   - libgcc   # dlc-only
   - matplotlib
   - non_local_detector
-  - numpy<1.24
   - pip>=20.2.*
   - position_tools
   - pybind11  # req by mountainsort4 -> isosplit5
@@ -47,4 +46,5 @@ dependencies:
       - pynwb>=2.2.0,<3
       - sortingview>=0.11
       - spikeinterface>=0.98.2,<0.99
+      - tensorflow<=2.12  # dlc-only
       - .[dlc]

src/spyglass/position/v1/dlc_reader.py

@@ -114,7 +114,8 @@ def pkl(self):
     def yml(self):
         if self._yml is None:
             with open(self.yml_path, "rb") as f:
-                self._yml = yaml.safe_load(f)
+                safe_yaml = yaml.YAML(typ="safe", pure=True)
+                self._yml = safe_yaml.load(f)
         return self._yml
 
     @property

src/spyglass/position/v1/dlc_utils.py

@@ -6,6 +6,7 @@
 import pathlib
 import pwd
 import subprocess
+import sys
 from collections import abc
 from contextlib import redirect_stdout
 from itertools import groupby
@@ -419,8 +420,10 @@ def get_video_path(key):
     """
     import pynwb
 
-    vf_key = {"nwb_file_name": key["nwb_file_name"], "epoch": key["epoch"]}
-    VideoFile()._no_transaction_make(vf_key, verbose=False)
+    valid_fields = VideoFile.fetch().dtype.fields.keys()
+    vf_key = {k: val for k, val in key.items() if k in valid_fields}
+    if not VideoFile & vf_key:
+        VideoFile()._no_transaction_make(vf_key, verbose=False)
     video_query = VideoFile & vf_key
 
     if len(video_query) != 1:
@@ -537,17 +540,23 @@ def _convert_mp4(
         "copy",
         f"{dest_path.as_posix()}",
     ]
-    try:
-        convert_process = subprocess.Popen(
-            convert_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-        )
-    except subprocess.CalledProcessError as err:
-        raise RuntimeError(
-            f"command {err.cmd} return with error (code {err.returncode}): {err.output}"
-        ) from err
-    out, _ = convert_process.communicate()
-    print(out.decode("utf-8"))
-    print(f"finished converting {filename}")
+    if dest_path.exists():
+        print(f"{dest_path} already exists, skipping conversion")
+    else:
+        try:
+            sys.stdout.flush()
+            convert_process = subprocess.Popen(
+                convert_command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+            )
+        except subprocess.CalledProcessError as err:
+            raise RuntimeError(
+                f"command {err.cmd} return with error (code {err.returncode}): {err.output}"
+            ) from err
+        out, _ = convert_process.communicate()
+        print(out.decode("utf-8"))
+        print(f"finished converting {filename}")
     print(
         f"Checking that number of packets match between {orig_filename} and {dest_filename}"
     )

src/spyglass/position/v1/position_dlc_model.py

@@ -196,7 +196,8 @@ def make(self, key):
             raise OSError(f"config_path {config_path} does not exist.")
         if config_path.suffix in (".yml", ".yaml"):
             with open(config_path, "rb") as f:
-                dlc_config = yaml.safe_load(f)
+                safe_yaml = yaml.YAML(typ="safe", pure=True)
+                dlc_config = safe_yaml.load(f)
             if isinstance(params["params"], dict):
                 dlc_config.update(params["params"])
                 del params["params"]