From ca5383b0880c07a605ff48395b524fb0fbaa54d9 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Sun, 16 Jan 2022 15:24:57 -0500 Subject: [PATCH 01/34] Changes on ephys and imaging to accept sessionless acquisitions --- u19_pipeline/ephys.py | 22 ++++++++++- u19_pipeline/imaging_acq.py | 78 +++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 u19_pipeline/imaging_acq.py diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index c50b3f34..ee3867eb 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -2,7 +2,7 @@ import pathlib import numpy as np -from u19_pipeline import ephys, behavior +from u19_pipeline import ephys, behavior, acquisition, subject from element_array_ephys import probe as probe_element from element_array_ephys import ephys as ephys_element @@ -57,6 +57,26 @@ class EphysSession(dj.Manual): """ +@schema +class EphysAcquisitions(dj.Manual): + definition = """ + acquisition_id: INT(11) AUTO_INCREMENT # Unique number assigned to each acquisition + ----- + -> [nullable] acquisition.Session # acquisition Session key + -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # subject inherited from subjects table (in case there is no related session) + ephys_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + """ + + +@schema +class EphysSortings(dj.Manual): + definition = """ + sorting_id: INT(11) AUTO_INCREMENT # Unique number assigned to each sorting + ----- + ->EphysAcquisitions # acquisition id (id to raw path of acquisition) + sorting_directory=null: VARCHAR(255) # relative path to specific sorting + """ + # ephys element requires table with name Session Session = EphysSession diff --git a/u19_pipeline/imaging_acq.py b/u19_pipeline/imaging_acq.py new file mode 100644 index 00000000..17cbd3f8 --- /dev/null +++ b/u19_pipeline/imaging_acq.py @@ -0,0 +1,78 @@ +import datajoint as dj +from u19_pipeline import acquisition + +from u19_pipeline import acquisition, subject + +schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_acq') + + +@schema +class Scan(dj.Imported): + definition = """ + scan_id : INT(11) AUTO_INCREMENT # Unique number assigned to each scan + --- + -> [nullable] acquisition.Session # acquisition Session key + -> [nullable] subject.Subject.proj(scan_subject='subject_fullname') # subject inherited from subjects table (in case there is no related session) + scan_directory : varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + """ + +@schema +class ScanInfo(dj.Imported): + definition = """ + # metainfo about imaging session + -> Scan + --- + file_name_base : varchar(255) # base name of the file + scan_width : int # width of scanning in pixels + scan_height : int # height of scanning in pixels + acq_time : datetime # acquisition time + n_depths : tinyint # number of depths + scan_depths : blob # depth values in this scan + frame_rate : float # imaging frame rate + inter_fov_lag_sec : float # time lag in secs between fovs + frame_ts_sec : longblob # frame timestamps in secs 1xnFrames + power_percent : float # percentage of power used in this scan + channels : blob # is this the channer number or total number of channels + cfg_filename : varchar(255) # cfg file path + usr_filename : varchar(255) # usr file path + fast_z_lag : float # fast z lag + fast_z_flyback_time : float # time it takes to fly back to fov + line_period : float # scan time per line + scan_frame_period : float + scan_volume_rate : float + flyback_time_per_frame : float + flyto_time_per_scan_field : float + fov_corner_points : blob # coordinates of the corners of the full 5mm FOV, in microns + nfovs : int # number of field of view + nframes : int # number of frames in the scan + nframes_good : int # number of frames in the scan before acceptable sample bleaching threshold is crossed + last_good_file : int # number of the file containing the last good frame because of bleaching + """ + + +@schema +class FieldOfView(dj.Imported): + definition = """ + # meta-info about specific FOV within mesoscope imagining session + -> Scan + fov : tinyint # number of the field of view in this scan + --- + fov_directory : varchar(255) # the absolute directory created for this fov + fov_name=null : varchar(32) # name of the field of view + fov_depth : float # depth of the field of view should be a number or a vector? + fov_center_xy : blob # X-Y coordinate for the center of the FOV in microns. One for each FOV in scan + fov_size_xy : blob # X-Y size of the FOV in microns. One for each FOV in scan (sizeXY) + fov_rotation_degrees : float # rotation of the FOV with respect to cardinal axes in degrees. One for each FOV in scan + fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan + fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? + """ + + class File(dj.Part): + definition = """ + # list of files per FOV + -> master + file_number : int + --- + fov_filename : varchar(255) # file name of the new fov tiff file + file_frame_range : blob # [first last] frame indices in this file, with respect to the whole imaging session + """ From 6bb413bb30e90a3f16aa23be249b3f06d5224635 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Wed, 19 Jan 2022 16:50:39 -0500 Subject: [PATCH 02/34] First draft recording tables on acquisition DB --- u19_pipeline/acquisition.py | 118 ++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) diff --git a/u19_pipeline/acquisition.py b/u19_pipeline/acquisition.py index 78dc78ce..0bf986b1 100644 --- a/u19_pipeline/acquisition.py +++ b/u19_pipeline/acquisition.py @@ -52,3 +52,121 @@ class DataDirectory(dj.Computed): file_name : varchar(255) # file name combined_file_name : varchar(255) # combined filename """ + +#Status pipeline dictionary +status_pipeline_dict = { + 'ERROR': {'Value': -1, + 'Label': 'Error in process', + 'Task_Field': None}, + 'NEW_SESSION': {'Value': 0, + 'Label': 'New session', + 'Task_Field': None}, + 'RAW_FILE_REQUEST': {'Value': 1, + 'Label': 'Raw file transfer requested', + 'Task_Field': 'task_copy_id_pre_path'}, + 'RAW_FILE_CLUSTER': {'Value': 2, + 'Label': 'Raw file transferred to cluster', + 'Task_Field': None}, + 'JOB_QUEUE': {'Value': 3, + 'Label': 'Processing job in queue', + 'Task_Field': 'slurm_id_sorting'}, + 'JOB_FINISHED': {'Value': 4, + 'Label': 'Processing job finished', + 'Task_Field': None}, + 'PROC_FILE_REQUEST': {'Value': 5, + 'Label': 'Processed file transfer requested', + 'Task_Field': 'task_copy_id_pos_path'}, + 'PROC_FILE_HOME': {'Value': 6, + 'Label': 'Processed file transferred to PNI', + 'Task_Field': None}, + 'CANONICAL_PIPELINE': {'Value': 7, + 'Label': 'Processed with Canonical pipeline', + 'Task_Field': None}, +} + +def get_content_list_from_status_dict(): + contents = list() + for i in status_pipeline_dict.keys(): + contents.append([status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']]) + return contents + + +@schema +class StatusDefinition(dj.Lookup): + definition = """ + status_pipeline: TINYINT(1) # status in the ephys/imaging pipeline + --- + status_definition: VARCHAR(256) # Status definition + """ + contents = get_content_list_from_status_dict() + +@schema +class RecordingModality(dj.Manual): + definition = """ + recording_modality: INT(11) AUTO_INCREMENT # Unique number assigned to each recording + ----- + -> [nullable] acquisition.Session # acquisition Session key + -> [nullable] subject.Subject.proj(recording_subject='subject_fullname') # Recording subject when no behavior Session present + -> [nullable] recording_datetime: datetime # Recording datetime when no bheavior Session present + modality: + recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + """ + + +(dj.Lookup): + definition = """ + recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) + --- + modality_description: varchar(255) # description for the modality + root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) + """ + contents = [ + ['ephys', '', '/braininit/Data/eletrophysiology'], + ['imaging', '', '/braininit/Data/eletrophysiology'], + ['ephys', '', '/braininit/Data/eletrophysiology'] + ] + + + +@schema +class Recordings(dj.Manual): + definition = """ + recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to each recording + ----- + -> [nullable] acquisition.Session # acquisition Session key + -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present + -> [nullable] recording_datetime: datetime # Recording datetime when no bheavior Session present + modality: + recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + """ + + + +@schema +class AcquisitionSessionsTestAutoPipeline(dj.Manual): + definition = """ + ->Sessions + ----- + ->StatusDefinition # current status in the ephys pipeline + session_rat: VARCHAR(8) # ratname inherited from rats table + session_userid: VARCHAR(32) # rat owner inherited from contacts table + session_rigid: INT(3) # rig id number inherited from riginfo table + acquisition_type: VARCHAR(32) # ephys or imaging + acquisition_raw_rel_path=null: VARCHAR(200) # relative path of raw files + acquisition_post_rel_path=null: VARCHAR(200) # relative path for sorted files + task_copy_id_pre_path=null: UUID # id for globus transfer task raw file cup->tiger + task_copy_id_pos_path=null: UUID # id for globus transfer task sorted file tiger->cup + slurm_id_sorting=null: VARCHAR(16) # id for slurm process in tiger + """ + +@schema +class AcquisitionSessionsStatus(dj.Manual): + definition = """ + ->AcquisitionSessions + ----- + -> StatusDefinition.proj(status_pipeline_old='status_pipeline') # old status in the ephys pipeline + -> StatusDefinition.proj(status_pipeline_new='status_pipeline') # current status in the ephys pipeline + status_timestamp: DATETIME # timestamp when status change ocurred + error_message=null: VARCHAR(4096) # Error message if status now is failed + error_exception=null: BLOB # Error exception if status is failed + """ From 526957ea6272d35680cd3ff82d3a8778b31a9b92 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Thu, 20 Jan 2022 16:04:54 -0500 Subject: [PATCH 03/34] First Draft design recording schema for aitomatic pipeline of multiple modalities --- notebooks/TestAutomaticPipeline.ipynb | 807 ++++++++++++++++++ u19_pipeline/acquisition.py | 118 --- u19_pipeline/ephys.py | 40 +- .../{imaging_acq.py => imaging_rec.py} | 28 +- u19_pipeline/recording.py | 181 ++++ 5 files changed, 1027 insertions(+), 147 deletions(-) create mode 100644 notebooks/TestAutomaticPipeline.ipynb rename u19_pipeline/{imaging_acq.py => imaging_rec.py} (86%) create mode 100644 u19_pipeline/recording.py diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb new file mode 100644 index 00000000..f4d188f9 --- /dev/null +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -0,0 +1,807 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check if configuration has been run already and change directory to repository root:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Local configuration file found !!, no need to run the configuration (unless configuration has changed)\n" + ] + } + ], + "source": [ + "from scripts.conf_file_finding import try_find_conf_file\n", + "try_find_conf_file()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Connecting alvaros@datajoint00.pni.princeton.edu:3306\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "'0.13.2'" + ] + }, + "metadata": {}, + "execution_count": 2 + } + ], + "source": [ + "import datajoint as dj\n", + "import os\n", + "import getpass\n", + "import pandas as pd\n", + "import subprocess\n", + "import json\n", + "\n", + "from u19_pipeline import recording, ephys, imaging_rec\n", + "\n", + "#import utility.dj_shortcuts as dj_short\n", + "#import utility.acquisition_pipeline_handler as aph\n", + "dj.__version__" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Declare AutoPipeline and Status tables if they don't exist" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "image/svg+xml": "\n\n\n\n\n16\n\n16\n\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n\n16->recording.RecordingProcessStatus\n\n\n\n\n15\n\n15\n\n\n\n15->recording.RecordingProcessStatus\n\n\n\n\n14\n\n14\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\n\n14->recording.Recording\n\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\n\nimaging_rec.ImagingSegmentation\n\n\nimaging_rec.ImagingSegmentation\n\n\n\n\n\nimaging_rec.Scan->imaging_rec.ImagingSegmentation\n\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\n\nrecording.StatusProcessDefinition->16\n\n\n\n\nrecording.StatusProcessDefinition->15\n\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\n\nephys.EphysSorting\n\n\nephys.EphysSorting\n\n\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\n\nrecording.PreprocessingParamSet\n\n\nrecording.PreprocessingParamSet\n\n\n\n\n\nrecording.PreprocessingParamSet->recording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->ephys.EphysSorting\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingSegmentation\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n\nrecording.acquisition.Session->recording.Recording\n\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\n\nrecording.RecordingModality->recording.PreprocessingParamSet\n\n\n\n\nrecording.ProcessingParamSet\n\n\nrecording.ProcessingParamSet\n\n\n\n\n\nrecording.RecordingModality->recording.ProcessingParamSet\n\n\n\n\nephys.EphysRecording->ephys.EphysSorting\n\n\n\n\nrecording.ProcessingParamSet->recording.RecordingProcess\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\n\nrecording.subject.Subject->14\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\n" + }, + "metadata": {}, + "execution_count": 3 + } + ], + "source": [ + "dj.ERD(recording) -1 + dj.ERD(ephys.EphysRecording) + dj.ERD(ephys.EphysSorting) + dj.ERD(imaging_rec.Scan) + dj.ERD(imaging_rec.ImagingSegmentation) \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Just for now, folders for ephys things in Globus\n", + "#FOR PNI endpoint\n", + "pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b'\n", + "pni_ephys_root_data_dir = '/jukebox/archive/brody/RATTER/PhysData/Raw/'\n", + "pni_ephys_sorted_data_dir = '/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/'\n", + "\n", + "#For tiger endpoint\n", + "tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1'\n", + "tiger_ephys_root_data_dir = '/tigress/alvaros/ephys_raw/'\n", + "tiger_ephys_sorted_data_dir = '/tigress/alvaros/ephys_sorted/'\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete and insert on the AcquisitionSessionsTestAutoPipeline and Status tables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#acquisition.AcquisitionSessionsTestAutoPipeline.delete()\n", + "#acquisition.AcquisitionSessionsStatus.delete" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get 5 \"real\" sessions to test and insert them to the TestAutoPipeline" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "acq_sessions_df = pd.DataFrame(acquisition.AcquisitionSessions.fetch(limit=5, as_dict=True))\n", + "acq_sessions_df['status_pipeline'] = 0\n", + "acquisition.AcquisitionSessionsTestAutoPipeline.insert(acq_sessions_df, skip_duplicates=True)\n", + "acquisition.AcquisitionSessionsTestAutoPipeline()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check if logged to globus" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'alvaros@princeton.edu\\n'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "s = subprocess.run([\"globus\", \"whoami\"], capture_output=True)\n", + "s.stdout.decode('UTF-8')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get all \"active sessions\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sessidstatus_pipelinesession_ratsession_useridsession_rigidacquisition_typeacquisition_raw_rel_pathacquisition_post_rel_pathtask_copy_id_pre_pathtask_copy_id_pos_pathslurm_id_sortingquery_key
05905770T173zhihaol202ephysAnn/miscTestFolder2NoneNoneNone{'sessid': 590577}
\n", + "
" + ], + "text/plain": [ + " sessid status_pipeline session_rat session_userid session_rigid \\\n", + "0 590577 0 T173 zhihaol 202 \n", + "\n", + " acquisition_type acquisition_raw_rel_path acquisition_post_rel_path \\\n", + "0 ephys Ann/misc TestFolder2 \n", + "\n", + " task_copy_id_pre_path task_copy_id_pos_path slurm_id_sorting \\\n", + "0 None None None \n", + "\n", + " query_key \n", + "0 {'sessid': 590577} " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sessions_active = acquisition.AcquisitionSessionsTestAutoPipeline & 'status_pipeline < 7 and status_pipeline >= 0'\n", + "df_sessions = pd.DataFrame(sessions_active.fetch(as_dict=True))\n", + "\n", + "#Add a column to have a key dictionary as a new column\n", + "key_list = dj_short.get_primary_key_fields(acquisition.AcquisitionSessionsTestAutoPipeline)\n", + "df_sessions['query_key'] = df_sessions.loc[:, key_list].to_dict(orient='records')\n", + "df_sessions" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 0\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'transfer', '6ce834d6-ff8a-11e6-bad1-22000b9a448b:/jukebox/archive/brody/RATTER/PhysData/Raw/Ann/misc', 'a9df83d2-42f0-11e6-80cf-22000b1701d1:/tigress/alvaros/ephys_raw/Ann/misc', '--recursive', '--format', 'json']\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'task', 'show', 'd6aaa788-36b6-11ec-9e53-3df4ed83d858', '--format', 'json']\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "0 1\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'task', 'show', 'd6aaa788-36b6-11ec-9e53-3df4ed83d858', '--format', 'json']\n", + "0 2\n", + "Name: status_pipeline, dtype: int64\n", + "['ssh', 'alvaros@tigergpu.princeton.edu', 'sbatch', '/tigress/alvaros/slurm_files/slurm_sessid_590577.slurm']\n", + "aftercommand before comm\n", + "aftercommand after comm\n", + "Submitted batch job 7518327\n", + "\n", + "\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", + " State \n", + "---------- \n", + " PENDING \n", + " RUNNING \n", + " RUNNING \n", + "\n", + "PENDING\n", + "COMPLETED\n", + "b'PENDING'\n", + "b'COMPLETED'\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", + " State \n", + "---------- \n", + " RUNNING \n", + " RUNNING \n", + " RUNNING \n", + "\n", + "RUNNING\n", + "COMPLETED\n", + "b'RUNNING'\n", + "b'COMPLETED'\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", + " State \n", + "---------- \n", + " RUNNING \n", + " RUNNING \n", + " RUNNING \n", + "\n", + "RUNNING\n", + "COMPLETED\n", + "b'RUNNING'\n", + "b'COMPLETED'\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "0 3\n", + "Name: status_pipeline, dtype: int64\n", + "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", + " State \n", + "---------- \n", + " COMPLETED \n", + " COMPLETED \n", + " COMPLETED \n", + "\n", + "COMPLETED\n", + "COMPLETED\n", + "b'COMPLETED'\n", + "b'COMPLETED'\n", + "0 4\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'transfer', 'a9df83d2-42f0-11e6-80cf-22000b1701d1:/tigress/alvaros/ephys_sorted/TestFolder2', '6ce834d6-ff8a-11e6-bad1-22000b9a448b:/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/TestFolder2', '--recursive', '--format', 'json']\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'task', 'show', 'e2a6afab-36b6-11ec-95ea-853490a236f9', '--format', 'json']\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "0 5\n", + "Name: status_pipeline, dtype: int64\n", + "['globus', 'task', 'show', 'e2a6afab-36b6-11ec-95ea-853490a236f9', '--format', 'json']\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n", + "0 6\n", + "Name: status_pipeline, dtype: int64\n" + ] + } + ], + "source": [ + "import time\n", + "for i in range(30):\n", + " \n", + " aph.pipeline_handler_main()\n", + " time.sleep(3)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f" + }, + "kernelspec": { + "display_name": "Python 3.7.0 64-bit ('u19_datajoint': conda)", + "language": "python", + "name": "python37064bitu19datajointcondaf4e03803b61e40799835623898ea8018" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0-final" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/u19_pipeline/acquisition.py b/u19_pipeline/acquisition.py index 0bf986b1..78dc78ce 100644 --- a/u19_pipeline/acquisition.py +++ b/u19_pipeline/acquisition.py @@ -52,121 +52,3 @@ class DataDirectory(dj.Computed): file_name : varchar(255) # file name combined_file_name : varchar(255) # combined filename """ - -#Status pipeline dictionary -status_pipeline_dict = { - 'ERROR': {'Value': -1, - 'Label': 'Error in process', - 'Task_Field': None}, - 'NEW_SESSION': {'Value': 0, - 'Label': 'New session', - 'Task_Field': None}, - 'RAW_FILE_REQUEST': {'Value': 1, - 'Label': 'Raw file transfer requested', - 'Task_Field': 'task_copy_id_pre_path'}, - 'RAW_FILE_CLUSTER': {'Value': 2, - 'Label': 'Raw file transferred to cluster', - 'Task_Field': None}, - 'JOB_QUEUE': {'Value': 3, - 'Label': 'Processing job in queue', - 'Task_Field': 'slurm_id_sorting'}, - 'JOB_FINISHED': {'Value': 4, - 'Label': 'Processing job finished', - 'Task_Field': None}, - 'PROC_FILE_REQUEST': {'Value': 5, - 'Label': 'Processed file transfer requested', - 'Task_Field': 'task_copy_id_pos_path'}, - 'PROC_FILE_HOME': {'Value': 6, - 'Label': 'Processed file transferred to PNI', - 'Task_Field': None}, - 'CANONICAL_PIPELINE': {'Value': 7, - 'Label': 'Processed with Canonical pipeline', - 'Task_Field': None}, -} - -def get_content_list_from_status_dict(): - contents = list() - for i in status_pipeline_dict.keys(): - contents.append([status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']]) - return contents - - -@schema -class StatusDefinition(dj.Lookup): - definition = """ - status_pipeline: TINYINT(1) # status in the ephys/imaging pipeline - --- - status_definition: VARCHAR(256) # Status definition - """ - contents = get_content_list_from_status_dict() - -@schema -class RecordingModality(dj.Manual): - definition = """ - recording_modality: INT(11) AUTO_INCREMENT # Unique number assigned to each recording - ----- - -> [nullable] acquisition.Session # acquisition Session key - -> [nullable] subject.Subject.proj(recording_subject='subject_fullname') # Recording subject when no behavior Session present - -> [nullable] recording_datetime: datetime # Recording datetime when no bheavior Session present - modality: - recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket - """ - - -(dj.Lookup): - definition = """ - recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) - --- - modality_description: varchar(255) # description for the modality - root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) - """ - contents = [ - ['ephys', '', '/braininit/Data/eletrophysiology'], - ['imaging', '', '/braininit/Data/eletrophysiology'], - ['ephys', '', '/braininit/Data/eletrophysiology'] - ] - - - -@schema -class Recordings(dj.Manual): - definition = """ - recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to each recording - ----- - -> [nullable] acquisition.Session # acquisition Session key - -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present - -> [nullable] recording_datetime: datetime # Recording datetime when no bheavior Session present - modality: - recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket - """ - - - -@schema -class AcquisitionSessionsTestAutoPipeline(dj.Manual): - definition = """ - ->Sessions - ----- - ->StatusDefinition # current status in the ephys pipeline - session_rat: VARCHAR(8) # ratname inherited from rats table - session_userid: VARCHAR(32) # rat owner inherited from contacts table - session_rigid: INT(3) # rig id number inherited from riginfo table - acquisition_type: VARCHAR(32) # ephys or imaging - acquisition_raw_rel_path=null: VARCHAR(200) # relative path of raw files - acquisition_post_rel_path=null: VARCHAR(200) # relative path for sorted files - task_copy_id_pre_path=null: UUID # id for globus transfer task raw file cup->tiger - task_copy_id_pos_path=null: UUID # id for globus transfer task sorted file tiger->cup - slurm_id_sorting=null: VARCHAR(16) # id for slurm process in tiger - """ - -@schema -class AcquisitionSessionsStatus(dj.Manual): - definition = """ - ->AcquisitionSessions - ----- - -> StatusDefinition.proj(status_pipeline_old='status_pipeline') # old status in the ephys pipeline - -> StatusDefinition.proj(status_pipeline_new='status_pipeline') # current status in the ephys pipeline - status_timestamp: DATETIME # timestamp when status change ocurred - error_message=null: VARCHAR(4096) # Error message if status now is failed - error_exception=null: BLOB # Error exception if status is failed - """ diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index ee3867eb..b1bb7a4c 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -2,7 +2,7 @@ import pathlib import numpy as np -from u19_pipeline import ephys, behavior, acquisition, subject +from u19_pipeline import behavior, recording from element_array_ephys import probe as probe_element from element_array_ephys import ephys as ephys_element @@ -48,37 +48,33 @@ class SkullReference(dj.Lookup): @schema -class EphysSession(dj.Manual): +class EphysRecording(dj.Computed): definition = """ # General information of an ephys session - -> acquisition.Session + -> recording.Recording --- - ephys_directory: varchar(255) # the absolute directory where the ephys data for this session will be stored in bucket """ + key_source = recording.Recording & {'recording_modality': 'ephys'} + + def make(self, key): + self.insert(key) @schema -class EphysAcquisitions(dj.Manual): - definition = """ - acquisition_id: INT(11) AUTO_INCREMENT # Unique number assigned to each acquisition - ----- - -> [nullable] acquisition.Session # acquisition Session key - -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # subject inherited from subjects table (in case there is no related session) - ephys_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket - """ +class EphysSorting(dj.Computed): + definition = """ + -> EphysRecording + -> recording.RecordingProcess + ----- + """ + + def make(self, key): + self.insert(key['']) -@schema -class EphysSortings(dj.Manual): - definition = """ - sorting_id: INT(11) AUTO_INCREMENT # Unique number assigned to each sorting - ----- - ->EphysAcquisitions # acquisition id (id to raw path of acquisition) - sorting_directory=null: VARCHAR(255) # relative path to specific sorting - """ # ephys element requires table with name Session -Session = EphysSession +Session = EphysSorting # 3. Utility functions @@ -112,7 +108,7 @@ def get_session_directory(session_key): @schema class BehaviorSync(dj.Imported): definition = """ - -> ephys.EphysSession + -> ephys.EphysRecording --- nidq_sampling_rate : float # sampling rate of behavioral iterations niSampRate in nidq meta file iteration_index_nidq : longblob # Virmen index time series. Length of this longblob should be the number of samples in the nidaq file. diff --git a/u19_pipeline/imaging_acq.py b/u19_pipeline/imaging_rec.py similarity index 86% rename from u19_pipeline/imaging_acq.py rename to u19_pipeline/imaging_rec.py index 17cbd3f8..23da97a5 100644 --- a/u19_pipeline/imaging_acq.py +++ b/u19_pipeline/imaging_rec.py @@ -1,20 +1,34 @@ import datajoint as dj -from u19_pipeline import acquisition -from u19_pipeline import acquisition, subject +from u19_pipeline import acquisition, subject, recording schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_acq') @schema -class Scan(dj.Imported): +class Scan(dj.Computed): definition = """ - scan_id : INT(11) AUTO_INCREMENT # Unique number assigned to each scan + # General information of an ephys session + -> recording.Recording --- - -> [nullable] acquisition.Session # acquisition Session key - -> [nullable] subject.Subject.proj(scan_subject='subject_fullname') # subject inherited from subjects table (in case there is no related session) - scan_directory : varchar(255) # the relative directory where the ephys data for this session will be stored in bucket """ + key_source = recording.Recording & {'recording_modality': 'imaging'} + + def make(self, key): + self.insert(key) + + +@schema +class ImagingSegmentation(dj.Computed): + definition = """ + -> Scan + -> recording.RecordingProcess + ----- + """ + + def make(self, key): + self.insert(key['']) + @schema class ScanInfo(dj.Imported): diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py new file mode 100644 index 00000000..17f464c0 --- /dev/null +++ b/u19_pipeline/recording.py @@ -0,0 +1,181 @@ +import datajoint as dj +from u19_pipeline import lab, task, subject + +schema = dj.schema('u19_recording') + +from u19_pipeline import acquisition, subject + + +@schema +class RecordingModality(dj.Lookup): + definition = """ + recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) + --- + modality_description: varchar(255) # description for the modality + root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) + """ + contents = [ + ['ephys', '', '/braininit/Data/eletrophysiology'], + ['imaging', '', '/braininit/Data/eletrophysiology'], + ['video_acquisition', '', '/braininit/Data/video_acquisition'] + ] + + +@schema +class PreprocessingParamSet(dj.Lookup): + definition = """ + # Parameter set to be used in the preprocessing steps + preprocess_paramset_idx: smallint + --- + -> RecordingModality + preprocess_paramset_desc: varchar(128) + preprocess_param_set_hash: uuid + unique index (preprocess_param_set_hash) + preprocessing_params: longblob # dictionary of all applicable parameters + """ + + @classmethod + def insert_new_params(cls, recording_modality: str, paramset_idx: int, + paramset_desc: str, params: dict): + param_dict = {'recording_modality': recording_modality, + 'preprocess_paramset_idx': paramset_idx, + 'preprocess_paramset_desc': paramset_desc, + 'preprocessing_params': params, + 'preprocess_param_set_hash': dict_to_uuid(params)} + param_query = cls & {'preprocess_param_set_hash': param_dict['preprocess_param_set_hash']} + + if param_query: # If the specified param-set already exists + existing_paramset_idx = param_query.fetch1('preprocess_paramset_idx') + if existing_paramset_idx == paramset_idx: # If the existing set has the same paramset_idx: job done + return + else: # If not same name: human error, trying to add the same paramset with different name + raise dj.DataJointError( + 'The specified param-set' + ' already exists - paramset_idx: {}'.format(existing_paramset_idx)) + else: + cls.insert1(param_dict) + + +@schema +class ProcessingParamSet(dj.Lookup): + definition = """ + # Parameter set to be used in the preprocessing steps + processing_paramset_idx: smallint + --- + -> RecordingModality + processing_paramset_desc: varchar(128) + processing_param_set_hash: uuid + unique index (processing_param_set_hash) + processing_params: longblob # dictionary of all applicable parameters + """ + + @classmethod + def insert_new_params(cls, recording_modality: str, paramset_idx: int, + paramset_desc: str, params: dict): + param_dict = {'recording_modality': recording_modality, + 'processing_paramset_idx': paramset_idx, + 'processing_paramset_desc': paramset_desc, + 'processing_params': params, + 'processing_param_set_hash': dict_to_uuid(params)} + param_query = cls & {'processing_param_set_hash': param_dict['processing_param_set_hash']} + + if param_query: # If the specified param-set already exists + existing_paramset_idx = param_query.fetch1('processing_paramset_idx') + if existing_paramset_idx == paramset_idx: # If the existing set has the same paramset_idx: job done + return + else: # If not same name: human error, trying to add the same paramset with different name + raise dj.DataJointError( + 'The specified param-set' + ' already exists - paramset_idx: {}'.format(existing_paramset_idx)) + else: + cls.insert1(param_dict) + +@schema +class Recording(dj.Manual): + definition = """ + recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to each recording + ----- + -> [nullable] acquisition.Session # acquisition Session key + -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present + recording_datetime=null: datetime # Recording datetime when no bheavior Session present + -> RecordingModality + recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + """ + + +#Status pipeline dictionary +status_pipeline_dict = { + 'ERROR': {'Value': -1, + 'Label': 'Error in process', + 'Task_Field': None}, + 'NEW_SESSION': {'Value': 0, + 'Label': 'New session', + 'Task_Field': None}, + 'RAW_FILE_REQUEST': {'Value': 1, + 'Label': 'Raw file transfer requested', + 'Task_Field': 'task_copy_id_pre_path'}, + 'RAW_FILE_CLUSTER': {'Value': 2, + 'Label': 'Raw file transferred to cluster', + 'Task_Field': None}, + 'JOB_QUEUE': {'Value': 3, + 'Label': 'Processing job in queue', + 'Task_Field': 'slurm_id_sorting'}, + 'JOB_FINISHED': {'Value': 4, + 'Label': 'Processing job finished', + 'Task_Field': None}, + 'PROC_FILE_REQUEST': {'Value': 5, + 'Label': 'Processed file transfer requested', + 'Task_Field': 'task_copy_id_pos_path'}, + 'PROC_FILE_HOME': {'Value': 6, + 'Label': 'Processed file transferred to PNI', + 'Task_Field': None}, + 'CANONICAL_PIPELINE': {'Value': 7, + 'Label': 'Processed with Canonical pipeline', + 'Task_Field': None}, +} + +def get_content_list_from_status_dict(): + contents = list() + for i in status_pipeline_dict.keys(): + contents.append([status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']]) + return contents + + +@schema +class StatusProcessDefinition(dj.Lookup): + definition = """ + status_pipeline: TINYINT(1) # status in the automate process pipeline + --- + status_definition: VARCHAR(256) # Status definition + """ + contents = get_content_list_from_status_dict() + + +@schema +class RecordingProcess(dj.Manual): + definition = """ + recording_process_id: INT(11) AUTO_INCREMENT # Unique number assigned to each processing job for a recording + ----- + -> Recording + -> StatusProcessDefinition # current status in the pipeline + -> PreprocessingParamSet # reference to params to preprocess recording + -> ProcessingParamSet # reference to params to process recording + recording_process_path=null: VARCHAR(200) # relative path for processed recording + task_copy_id_pre=null: UUID # id for globus transfer task raw file cup->tiger + task_copy_id_pos=null: UUID # id for globus transfer task sorted file tiger->cup + + slurm_id=null: VARCHAR(16) # id for slurm process in tiger + """ + + +@schema +class RecordingProcessStatus(dj.Manual): + definition = """ + ->RecordingProcess + ----- + -> StatusProcessDefinition.proj(status_pipeline_old='status_pipeline') # old status in the pipeline + -> StatusProcessDefinition.proj(status_pipeline_new='status_pipeline') # current status in the pipeline + status_timestamp: DATETIME # timestamp when status change ocurred + error_message=null: VARCHAR(4096) # Error message if status now is failed + error_exception=null: BLOB # Error exception if status now is failed + """ From a28460329860bc55f4da9edaf99c63dfee6ed1b7 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Wed, 26 Jan 2022 11:42:05 -0500 Subject: [PATCH 04/34] Added file extensions to recording --- u19_pipeline/imaging_rec.py | 2 +- u19_pipeline/recording.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 23da97a5..6b881d67 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -2,7 +2,7 @@ from u19_pipeline import acquisition, subject, recording -schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_acq') +schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_rec') @schema diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 17f464c0..8b349405 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -9,15 +9,16 @@ @schema class RecordingModality(dj.Lookup): definition = """ - recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) + recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) --- - modality_description: varchar(255) # description for the modality - root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) + modality_description: varchar(255) # description for the modality + root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) + recording_file_extensions: blob # file extensions specific for this modality """ contents = [ - ['ephys', '', '/braininit/Data/eletrophysiology'], - ['imaging', '', '/braininit/Data/eletrophysiology'], - ['video_acquisition', '', '/braininit/Data/video_acquisition'] + ['ephys', '', '/braininit/Data/eletrophysiology', ['ap.bin', 'ap.meta']], + ['imaging', '', '/braininit/Data/eletrophysiology', ['.avi', '.tiff','.tif']], + ['video_acquisition', '', '/braininit/Data/video_acquisition', ['.avi', '.mp4']] ] From 2a7b89b0e59d7295f68865b79cd7c1d8555020ff Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 12:59:21 -0500 Subject: [PATCH 05/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 8b349405..349a6a7f 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -1,5 +1,5 @@ import datajoint as dj -from u19_pipeline import lab, task, subject +from u19_pipeline import lab, task, subject, acquisition schema = dj.schema('u19_recording') From 78b4199874de164180cdc93743caf37f714f76e1 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 12:59:33 -0500 Subject: [PATCH 06/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 1 - 1 file changed, 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 349a6a7f..568fb18e 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -3,7 +3,6 @@ schema = dj.schema('u19_recording') -from u19_pipeline import acquisition, subject @schema From 9c83baac72074c4ade2ce25c304d9e12fdd379d5 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:09:09 -0500 Subject: [PATCH 07/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 568fb18e..9cf5191f 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -99,7 +99,7 @@ class Recording(dj.Manual): -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present recording_datetime=null: datetime # Recording datetime when no bheavior Session present -> RecordingModality - recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in bucket + recording_directory: varchar(255) # relative directory where the data for this session will be stored on cup """ From a2622bc1c1eb2ef837f485329ac126bef8142e45 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:09:45 -0500 Subject: [PATCH 08/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 9cf5191f..9c30ea41 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -15,8 +15,8 @@ class RecordingModality(dj.Lookup): recording_file_extensions: blob # file extensions specific for this modality """ contents = [ - ['ephys', '', '/braininit/Data/eletrophysiology', ['ap.bin', 'ap.meta']], - ['imaging', '', '/braininit/Data/eletrophysiology', ['.avi', '.tiff','.tif']], + ['ephys', '', '/braininit/Data/electrophysiology', ['ap.bin', 'ap.meta']], + ['imaging', '', '/braininit/Data/imaging', ['.avi', '.tiff','.tif']], ['video_acquisition', '', '/braininit/Data/video_acquisition', ['.avi', '.mp4']] ] From 0ce267c41396230ff2128dd6008f8e86f0cd1040 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:09:57 -0500 Subject: [PATCH 09/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 6b881d67..650d9436 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -15,7 +15,7 @@ class Scan(dj.Computed): key_source = recording.Recording & {'recording_modality': 'imaging'} def make(self, key): - self.insert(key) + self.insert1(key) @schema From 40a7e48bb389af2e6ef73861c223316beff11501 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:10:06 -0500 Subject: [PATCH 10/34] Update u19_pipeline/ephys.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index b1bb7a4c..d4bf5128 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -57,7 +57,7 @@ class EphysRecording(dj.Computed): key_source = recording.Recording & {'recording_modality': 'ephys'} def make(self, key): - self.insert(key) + self.insert1(key) @schema From 2d6c80374d05da7f4b8b0f21936fb3cefcd5f219 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:10:24 -0500 Subject: [PATCH 11/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 650d9436..4805e747 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -27,7 +27,7 @@ class ImagingSegmentation(dj.Computed): """ def make(self, key): - self.insert(key['']) + self.insert1(key) @schema From 19ff2cd715479891353ae3bf23e7ef81e43ff0f8 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:10:49 -0500 Subject: [PATCH 12/34] Update u19_pipeline/ephys.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/ephys.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index d4bf5128..9385089c 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -54,7 +54,9 @@ class EphysRecording(dj.Computed): -> recording.Recording --- """ - key_source = recording.Recording & {'recording_modality': 'ephys'} + @property + def key_source(self): + return recording.Recording & {'recording_modality': 'ephys'} def make(self, key): self.insert1(key) From fc9242b6625094be453902e9477d340e7eda7147 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:10:56 -0500 Subject: [PATCH 13/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 4805e747..e990a79a 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -12,7 +12,9 @@ class Scan(dj.Computed): -> recording.Recording --- """ - key_source = recording.Recording & {'recording_modality': 'imaging'} + @property + def key_source(self): + return recording.Recording & {'recording_modality': 'imaging'} def make(self, key): self.insert1(key) From 963abaee6f4b59fc2cfa3e91477e3d599e22c94c Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:11:03 -0500 Subject: [PATCH 14/34] Update u19_pipeline/ephys.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index 9385089c..a18d36d2 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -71,7 +71,7 @@ class EphysSorting(dj.Computed): """ def make(self, key): - self.insert(key['']) + self.insert1(key) From 997a5936eb6ae343b05073f55bddebafda61cfcf Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Mon, 31 Jan 2022 15:11:09 -0500 Subject: [PATCH 15/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index e990a79a..a7c3b091 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -8,7 +8,7 @@ @schema class Scan(dj.Computed): definition = """ - # General information of an ephys session + # General information of an imaging session -> recording.Recording --- """ From d19687b402dc602d132c9f193571b24e26a2202f Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Wed, 2 Feb 2022 09:37:32 -0500 Subject: [PATCH 16/34] merge pull request and local changes --- u19_pipeline/recording.py | 68 +++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 9c30ea41..75b9d005 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -15,8 +15,13 @@ class RecordingModality(dj.Lookup): recording_file_extensions: blob # file extensions specific for this modality """ contents = [ +<<<<<<< Updated upstream ['ephys', '', '/braininit/Data/electrophysiology', ['ap.bin', 'ap.meta']], ['imaging', '', '/braininit/Data/imaging', ['.avi', '.tiff','.tif']], +======= + ['electrophysiology', '', '/braininit/Data/eletrophysiology', ['ap.bin', 'ap.meta']], + ['imaging', '', '/braininit/Data/eletrophysiology', ['.avi', '.tiff','.tif']], +>>>>>>> Stashed changes ['video_acquisition', '', '/braininit/Data/video_acquisition', ['.avi', '.mp4']] ] @@ -25,12 +30,12 @@ class RecordingModality(dj.Lookup): class PreprocessingParamSet(dj.Lookup): definition = """ # Parameter set to be used in the preprocessing steps - preprocess_paramset_idx: smallint + preprocessing_paramset_idx: int(11) AUTO_INCREMENT --- -> RecordingModality - preprocess_paramset_desc: varchar(128) - preprocess_param_set_hash: uuid - unique index (preprocess_param_set_hash) + preprocessing_paramset_desc: varchar(128) + preprocessing_paramset_hash: uuid + unique index (preprocessing_paramset_hash) preprocessing_params: longblob # dictionary of all applicable parameters """ @@ -38,14 +43,14 @@ class PreprocessingParamSet(dj.Lookup): def insert_new_params(cls, recording_modality: str, paramset_idx: int, paramset_desc: str, params: dict): param_dict = {'recording_modality': recording_modality, - 'preprocess_paramset_idx': paramset_idx, - 'preprocess_paramset_desc': paramset_desc, + 'preprocessing_paramset_idx': paramset_idx, + 'preprocessing_paramset_desc': paramset_desc, 'preprocessing_params': params, - 'preprocess_param_set_hash': dict_to_uuid(params)} - param_query = cls & {'preprocess_param_set_hash': param_dict['preprocess_param_set_hash']} + 'preprocessing_paramset_hash': dict_to_uuid(params)} + param_query = cls & {'preprocessing_paramset_hash': param_dict['preprocessing_paramset_hash']} if param_query: # If the specified param-set already exists - existing_paramset_idx = param_query.fetch1('preprocess_paramset_idx') + existing_paramset_idx = param_query.fetch1('preprocessing_paramset_idx') if existing_paramset_idx == paramset_idx: # If the existing set has the same paramset_idx: job done return else: # If not same name: human error, trying to add the same paramset with different name @@ -60,12 +65,12 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, class ProcessingParamSet(dj.Lookup): definition = """ # Parameter set to be used in the preprocessing steps - processing_paramset_idx: smallint + processing_paramset_idx: int(11) AUTO_INCREMENT --- -> RecordingModality processing_paramset_desc: varchar(128) - processing_param_set_hash: uuid - unique index (processing_param_set_hash) + processing_paramset_hash: uuid + unique index (processing_paramset_hash) processing_params: longblob # dictionary of all applicable parameters """ @@ -76,8 +81,8 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, 'processing_paramset_idx': paramset_idx, 'processing_paramset_desc': paramset_desc, 'processing_params': params, - 'processing_param_set_hash': dict_to_uuid(params)} - param_query = cls & {'processing_param_set_hash': param_dict['processing_param_set_hash']} + 'processing_paramset_hash': dict_to_uuid(params)} + param_query = cls & {'processing_paramset_hash': param_dict['processing_paramset_hash']} if param_query: # If the specified param-set already exists existing_paramset_idx = param_query.fetch1('processing_paramset_idx') @@ -96,10 +101,16 @@ class Recording(dj.Manual): recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to each recording ----- -> [nullable] acquisition.Session # acquisition Session key - -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present - recording_datetime=null: datetime # Recording datetime when no bheavior Session present -> RecordingModality +<<<<<<< Updated upstream recording_directory: varchar(255) # relative directory where the data for this session will be stored on cup +======= + -> lab.Location + recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in braininit drive + local_directory: varchar(255) # local directory where this file is stored on the recording system + -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present + recording_datetime=null: datetime # Recording datetime when no behavior Session present +>>>>>>> Stashed changes """ @@ -111,25 +122,34 @@ class Recording(dj.Manual): 'NEW_SESSION': {'Value': 0, 'Label': 'New session', 'Task_Field': None}, - 'RAW_FILE_REQUEST': {'Value': 1, + 'PNI_DRIVE_TRANSFER_REQUEST': {'Value': 1, + 'Label': 'Recording directory transfer to PNI requested', + ' Task_Field': 'task_copy_id_pni'}, + 'PNI_DRIVE_TRANSFER_END': {'Value': 2, + 'Label': 'Recording directory transferred to PNI', + ' Task_Field': None}, + 'MODALITY_PREINGESTION': {'Value': 3, + 'Label': 'Preprocessing & Syncing jobs', + ' Task_Field': None}, + 'RAW_FILE_REQUEST': {'Value': 4, 'Label': 'Raw file transfer requested', 'Task_Field': 'task_copy_id_pre_path'}, - 'RAW_FILE_CLUSTER': {'Value': 2, + 'RAW_FILE_CLUSTER': {'Value': 5, 'Label': 'Raw file transferred to cluster', 'Task_Field': None}, - 'JOB_QUEUE': {'Value': 3, + 'JOB_QUEUE': {'Value': 6, 'Label': 'Processing job in queue', 'Task_Field': 'slurm_id_sorting'}, - 'JOB_FINISHED': {'Value': 4, + 'JOB_FINISHED': {'Value': 7, 'Label': 'Processing job finished', 'Task_Field': None}, - 'PROC_FILE_REQUEST': {'Value': 5, + 'PROC_FILE_REQUEST': {'Value': 8, 'Label': 'Processed file transfer requested', 'Task_Field': 'task_copy_id_pos_path'}, - 'PROC_FILE_HOME': {'Value': 6, + 'PROC_FILE_HOME': {'Value': 9, 'Label': 'Processed file transferred to PNI', 'Task_Field': None}, - 'CANONICAL_PIPELINE': {'Value': 7, + 'CANONICAL_PIPELINE': {'Value': 10, 'Label': 'Processed with Canonical pipeline', 'Task_Field': None}, } @@ -161,9 +181,9 @@ class RecordingProcess(dj.Manual): -> PreprocessingParamSet # reference to params to preprocess recording -> ProcessingParamSet # reference to params to process recording recording_process_path=null: VARCHAR(200) # relative path for processed recording + task_copy_id_pni=null: UUID # id for globus transfer task raw file local->cup task_copy_id_pre=null: UUID # id for globus transfer task raw file cup->tiger task_copy_id_pos=null: UUID # id for globus transfer task sorted file tiger->cup - slurm_id=null: VARCHAR(16) # id for slurm process in tiger """ From 43eccc0e9af1782b0a5be274d4e992dab621e098 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:41:42 -0500 Subject: [PATCH 17/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 75b9d005..6ea0afc9 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -1,7 +1,7 @@ import datajoint as dj from u19_pipeline import lab, task, subject, acquisition -schema = dj.schema('u19_recording') +schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording') From f5112cfcff7f862cdb763a81966017a3d42bd6eb Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:41:54 -0500 Subject: [PATCH 18/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 6ea0afc9..fb412716 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -11,7 +11,7 @@ class RecordingModality(dj.Lookup): recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) --- modality_description: varchar(255) # description for the modality - root_direcory: varchar(255) # root directory where that modality is stored (e.g. ephys = /braininit/Data/eletrophysiology) + root_directory: varchar(255) # root directory where that modality is stored recording_file_extensions: blob # file extensions specific for this modality """ contents = [ From 2409029acec8399c8e0f3cfa5d037822bb60ef5f Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:43:29 -0500 Subject: [PATCH 19/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index fb412716..89859c90 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -56,7 +56,7 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, else: # If not same name: human error, trying to add the same paramset with different name raise dj.DataJointError( 'The specified param-set' - ' already exists - paramset_idx: {}'.format(existing_paramset_idx)) + ' already exists - preprocess_paramset_idx: {}'.format(existing_paramset_idx)) else: cls.insert1(param_dict) From e582fe3df4d394f99feaa1b8007685a44bb2ea96 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:48:09 -0500 Subject: [PATCH 20/34] Update u19_pipeline/ephys.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/ephys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index a18d36d2..0db205ca 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -63,7 +63,7 @@ def make(self, key): @schema -class EphysSorting(dj.Computed): +class EphysProcessing(dj.Computed): definition = """ -> EphysRecording -> recording.RecordingProcess From 450f1b02545b4bccfa6a8b20914e9f0eacf4a734 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:48:39 -0500 Subject: [PATCH 21/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index a7c3b091..867b6721 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -21,7 +21,7 @@ def make(self, key): @schema -class ImagingSegmentation(dj.Computed): +class ImagingProcessing(dj.Computed): definition = """ -> Scan -> recording.RecordingProcess From 0c677a5155a9ffae357f03cf405a6ec6db35eee5 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:48:58 -0500 Subject: [PATCH 22/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 867b6721..8fd762ba 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -36,6 +36,7 @@ def make(self, key): class ScanInfo(dj.Imported): definition = """ # metainfo about imaging session + # `make` function is declared in the `U19-pipeline-matlab` -> Scan --- file_name_base : varchar(255) # base name of the file From c48db07bbe6cd5c0591b67e1b2a33265b559eae0 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:49:13 -0500 Subject: [PATCH 23/34] Update u19_pipeline/imaging_rec.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/imaging_rec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 8fd762ba..926c2f43 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -70,7 +70,8 @@ class ScanInfo(dj.Imported): @schema class FieldOfView(dj.Imported): definition = """ - # meta-info about specific FOV within mesoscope imagining session + # meta-info about specific FOV within mesoscope imaging session + # `make` function is declared in the `U19-pipeline-matlab` repository -> Scan fov : tinyint # number of the field of view in this scan --- From 8a30d7e2516c2590f1021b01ae2781250660d95e Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 08:50:54 -0500 Subject: [PATCH 24/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 89859c90..0e5bb647 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -191,7 +191,7 @@ class RecordingProcess(dj.Manual): @schema class RecordingProcessStatus(dj.Manual): definition = """ - ->RecordingProcess + -> RecordingProcess ----- -> StatusProcessDefinition.proj(status_pipeline_old='status_pipeline') # old status in the pipeline -> StatusProcessDefinition.proj(status_pipeline_new='status_pipeline') # current status in the pipeline From 9a35232cadd5c75eada563ffa89ffe3666492f35 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 09:04:23 -0500 Subject: [PATCH 25/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 1 - 1 file changed, 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 0e5bb647..a75aed64 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -114,7 +114,6 @@ class Recording(dj.Manual): """ -#Status pipeline dictionary status_pipeline_dict = { 'ERROR': {'Value': -1, 'Label': 'Error in process', From f256967d3df9c27d040cae2d91e2e697db4c95a1 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 09:04:43 -0500 Subject: [PATCH 26/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index a75aed64..3d018a16 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -163,7 +163,7 @@ def get_content_list_from_status_dict(): @schema class StatusProcessDefinition(dj.Lookup): definition = """ - status_pipeline: TINYINT(1) # status in the automate process pipeline + status_pipeline_idx: TINYINT(1) # status in the automate process pipeline --- status_definition: VARCHAR(256) # Status definition """ From f7ade5bd30f3105f1b525c43c4374c133ec80d08 Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 09:04:56 -0500 Subject: [PATCH 27/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 3d018a16..058aca02 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -154,10 +154,7 @@ class Recording(dj.Manual): } def get_content_list_from_status_dict(): - contents = list() - for i in status_pipeline_dict.keys(): - contents.append([status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']]) - return contents + return [[status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']] for i in status_pipeline_dict.keys()] @schema From 6d685f182b6a08ef4244172d04efb31b9f13769f Mon Sep 17 00:00:00 2001 From: Alvaro Luna <56653225+Alvalunasan@users.noreply.github.com> Date: Tue, 8 Feb 2022 09:05:42 -0500 Subject: [PATCH 28/34] Update u19_pipeline/recording.py Co-authored-by: Kabilar Gunalan --- u19_pipeline/recording.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 058aca02..0e40296c 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -58,7 +58,7 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, 'The specified param-set' ' already exists - preprocess_paramset_idx: {}'.format(existing_paramset_idx)) else: - cls.insert1(param_dict) + cls.insert1(paramset_dict) @schema From 729b2da2350c4f5db57c15231b90692a0e93efb9 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Mon, 14 Feb 2022 13:42:36 -0500 Subject: [PATCH 29/34] functional recording handler pipeline --- requirements.txt | 1 + u19_pipeline/automatic_job/__init__.py | 0 u19_pipeline/automatic_job/file_transfers.py | 69 +++++ .../automatic_job/ingest_scaninfo_shell.sh | 8 + u19_pipeline/automatic_job/params_config.py | 176 ++++++++++++ .../automatic_job/recording_handler.py | 259 ++++++++++++++++++ .../recording_process_handler.py | 258 +++++++++++++++++ u19_pipeline/automatic_job/slurm_creator.py | 121 ++++++++ u19_pipeline/ephys.py | 50 +++- u19_pipeline/imaging.py | 10 +- u19_pipeline/imaging_rec.py | 32 ++- u19_pipeline/recording.py | 255 +++++++++-------- u19_pipeline/utility.py | 9 +- u19_pipeline/utils/dj_shortcuts.py | 51 ++++ u19_pipeline/utils/path_utils.py | 8 + 15 files changed, 1165 insertions(+), 142 deletions(-) create mode 100644 u19_pipeline/automatic_job/__init__.py create mode 100644 u19_pipeline/automatic_job/file_transfers.py create mode 100755 u19_pipeline/automatic_job/ingest_scaninfo_shell.sh create mode 100644 u19_pipeline/automatic_job/params_config.py create mode 100644 u19_pipeline/automatic_job/recording_handler.py create mode 100644 u19_pipeline/automatic_job/recording_process_handler.py create mode 100644 u19_pipeline/automatic_job/slurm_creator.py create mode 100644 u19_pipeline/utils/dj_shortcuts.py diff --git a/requirements.txt b/requirements.txt index e4b63a55..75535af1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +globus-cli scipy python-dotenv element-calcium-imaging diff --git a/u19_pipeline/automatic_job/__init__.py b/u19_pipeline/automatic_job/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/u19_pipeline/automatic_job/file_transfers.py b/u19_pipeline/automatic_job/file_transfers.py new file mode 100644 index 00000000..0cc50901 --- /dev/null +++ b/u19_pipeline/automatic_job/file_transfers.py @@ -0,0 +1,69 @@ + +import pathlib +import subprocess +import json +import re + + +#Functions to transfer files (globus, scp, smbclient) + + +#FOR PNI endpoint +pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' +#pni_ephys_sorted_data_dir = '/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/' + +#PNI directories +pni_root_data_dir = '/braininit/Data/' + +#For tiger endpoint +default_user = 'alvaros' # This will change to our automatic client for globus transfers +tiger_gpu_host = 'tigergpu.princeton.edu' +tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' + +# Tiger directories +tiger_home_dir = '/tigress/alvaros' # This will be changed to /scratch/gpfs/BRAINCOGS when permissions are granted +tiger_raw_root_data_dir = tiger_home_dir + '/DataRaw' +tiger_sorted_root_data_dir = tiger_home_dir + '/DataProcessed' +tiger_slurm_files_dir = tiger_home_dir + '/slurm_files/' +tiger_log_files_dir = tiger_home_dir + '/job_log/' + + +def scp_file_transfer(source, dest): + + p = subprocess.Popen(["scp", source, dest]) + transfer_status = p.wait() + return transfer_status + + +def request_globus_transfer(source, destination): + + globus_command = ["globus", "transfer", source, destination, '--recursive', '--format', 'json'] + print(globus_command) + s = subprocess.run(globus_command, capture_output=True) + transfer_request = json.loads(s.stdout.decode('UTF-8')) + return transfer_request + + +def request_globus_transfer_status(id_task): + + globus_command = ["globus", "task", "show", id_task, '--format', 'json'] + print(globus_command) + s = subprocess.run(globus_command, capture_output=True) + transfer_request = json.loads(s.stdout.decode('UTF-8')) + return transfer_request + + +def globus_transfer_to_tiger(raw_rel_path): + + source_ep = pni_ep_id + ':' + pni_root_data_dir + raw_rel_path + dest_ep = tiger_ep_dir + ':' + tiger_raw_root_data_dir + raw_rel_path + transfer_request = request_globus_transfer(source_ep, dest_ep) + return transfer_request + + +def globus_transfer_to_pni(sorted_rel_path): + + source_ep = tiger_ep_dir + ':' + tiger_sorted_root_data_dir + sorted_rel_path + dest_ep = pni_ep_id + ':' + pni_root_data_dir + sorted_rel_path + transfer_request = request_globus_transfer(source_ep, dest_ep) + return transfer_request \ No newline at end of file diff --git a/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh b/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh new file mode 100755 index 00000000..78f99e55 --- /dev/null +++ b/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh @@ -0,0 +1,8 @@ + + +cd $1 + +module load matlab/R2020b + +key = $2 +matlab -singleCompThread -nodisplay -nosplash -r populate_ScanInfo_spock(key); \ No newline at end of file diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py new file mode 100644 index 00000000..0d4dccc4 --- /dev/null +++ b/u19_pipeline/automatic_job/params_config.py @@ -0,0 +1,176 @@ +import pandas as pd +import numpy as np + +recording_modality_dict = [ + { + 'RecordingModality': 'electrophysiology', + 'Description': '', + 'RootDirectorty': '/braininit/Data/electrophysiology', + 'FileExtensions': np.asarray(['ap.bin', 'ap.meta']), + 'RecordingFilePattern': np.asarray(['/*g[0-9]/*imec[0-9]']), + 'ProcessUnitFilePattern': np.asarray(['/*imec[0-9]/']), + 'ProcessUnitDirectoryField': 'probe_directory', + }, + { + 'RecordingModality': 'imaging', + 'Description': '', + 'RootDirectorty': '/braininit/Data/imaging', + 'FileExtensions': np.asarray(['.avi', '.tiff','.tif']), + 'RecordingFilePattern': np.asarray(['']), + 'ProcessUnitFilePattern': np.asarray(['']), + 'ProcessUnitDirectoryField': 'fov_directory', + }, + { + 'RecordingModality': 'video_acquisition', + 'Description': '', + 'RootDirectorty': '/braininit/Data/imaging', + 'FileExtensions': np.asarray(['.avi', '.mp4']), + 'RecordingFilePattern': np.asarray(['']), + 'ProcessUnitFilePattern': np.asarray(['']), + 'ProcessUnitDirectoryField': 'video_directory' + }, +] + +recording_modality_list = [list(i.values()) for i in recording_modality_dict] +recording_modality_df = pd.DataFrame(recording_modality_dict) + +recording_status_dict = [ + { + 'Value': -1, + 'Key': 'ERROR', + 'Label': 'Error in recording handling', + 'UpdateField': None, + 'ProcessFunction': None, + 'FunctionField': None, + }, + { + 'Value': 0, + 'Key': 'NEW_RECORDING', + 'Label': 'New recording', + 'UpdateField': None, + 'ProcessFunction': None, + 'FunctionField': None, + }, + { + 'Value': 1, + 'Key': 'PNI_DRIVE_TRANSFER_REQUEST', + 'Label': 'Recording directory transfer to PNI requested', + 'UpdateField': 'task_copy_id_pni', + 'ProcessFunction': 'local_transfer_request', + 'FunctionField': 'recording_process_pre_path', + }, + { + 'Value': 2, + 'Key': 'PNI_DRIVE_TRANSFER_END', + 'Label': 'Recording directory transferred to PNI', + 'UpdateField': None, + 'ProcessFunction': 'local_transfer_check', + 'FunctionField': 'task_copy_id_pni', + }, + { + 'Value': 3, + 'Key': 'MODALITY_PREINGESTION', + 'Label': 'modality ingestion & Syncing jobs done', + 'UpdateField': None, + 'ProcessFunction': 'modality_preingestion', + 'FunctionField': None, + }, + +] + +recording_status_list = [[i['Value'], i['Label']] for i in recording_status_dict] +recording_status_df = pd.DataFrame(recording_status_dict) + + +recording_process_status_dict = [ + { + 'Value': -1, + 'Key': 'ERROR', + 'Label': 'Error in recording process', + 'UpdateField': None, + 'ProcessFunction': None, + 'FunctionField': None, + }, + { + 'Value': 0, + 'Key': 'NEW_RECORDING_PROCESS', + 'Label': 'New recording process', + 'UpdateField': None, + 'ProcessFunction': None, + 'FunctionField': None, + }, + { + 'Value': 1, + 'Key': 'RAW_FILE_TRANSFER_REQUEST', + 'Label': 'Raw file transfer requested', + 'UpdateField': 'task_copy_id_pre_path', + 'ProcessFunction': 'transfer_request', + 'FunctionField': 'recording_process_pre_path', + }, + { + 'Value': 2, + 'Key': 'RAW_FILE_TRANSFER_END', + 'Label': 'Raw file transferred to cluster', + 'UpdateField': None, + 'ProcessFunction': 'transfer_check', + 'FunctionField': 'task_copy_id_pre_path', + }, + { + 'Value': 3, + 'Key': 'JOB_QUEUE', + 'Label': 'Processing job in queue', + 'UpdateField': 'slurm_id', + 'ProcessFunction': 'slurm_job_queue', + 'FunctionField': None, + }, + { + 'Value': 4, + 'Key': 'JOB_FINISHED', + 'Label': 'Processing job finished', + 'UpdateField': None, + 'ProcessFunction': 'slurm_job_check', + 'FunctionField': 'slurm_id', + }, + { + 'Value': 5, + 'Key': 'PROC_FILE_TRANSFER_REQUEST', + 'Label': 'Processed file transfer requested', + 'UpdateField': 'task_copy_id_post', + 'ProcessFunction': 'transfer_request', + 'FunctionField': 'recording_process_post_path', + }, + { + 'Value': 6, + 'Key': 'PROC_FILE_TRANSFER_END', + 'Label': 'Processed file transferred to PNI', + 'UpdateField': None, + 'ProcessFunction': 'transfer_check', + 'FunctionField': 'task_copy_id_post', + }, + { + 'Value': 7, + 'Key': 'JOB_QUEUE_ELEMENT_WORKFLOW', + 'Label': 'Job Queue Element Workflow ingestion', + 'UpdateField': None, + 'ProcessFunction': None, + #'ProcessFunction': RecProcessHandler.slurm_job_element, + 'FunctionField': None, + }, + { + 'Value': 8, + 'Key': 'JOB_FINSISHED_ELEMENT_WORKFLOW', + 'Label': 'Process finished', + 'UpdateField': None, + 'ProcessFunction': 'slurm_job_check', + 'FunctionField': None, + } +] + +recording_process_status_list = [[i['Value'], i['Label']] for i in recording_process_status_dict] + +system_process = { + 'SUCCESS': 0 +} + + +startup_pipeline_matlab_dir = '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts' \ No newline at end of file diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py new file mode 100644 index 00000000..4d244784 --- /dev/null +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -0,0 +1,259 @@ + +import os +import pathlib +import subprocess +import json +import time +import re +import pandas as pd +import datajoint as dj +from u19_pipeline import recording, ephys, imaging_rec +import u19_pipeline.utils.dj_shortcuts as dj_short +import u19_pipeline.automatic_job.file_transfers as ft +import u19_pipeline.automatic_job.slurm_creator as slurmlib +import u19_pipeline.automatic_job.params_config as config + +recording_status_df = pd.DataFrame(config.recording_status_dict) + +class RecordingHandler(): + + default_update_value_dict ={ + 'value_update': None, + 'error_info': None + } + + @staticmethod + def pipeline_handler_main(): + ''' + Call all processing functions according to the current status of each recording + Update status of each process job accordingly + ''' + + #Get info from all the possible status for a processjob + df_all_recordings = RecordingHandler.get_active_recordings() + + #For all active process jobs + for i in range(df_all_recordings.shape[0]): + + #Filter current process job + recording_series = df_all_recordings.loc[i, :] + + #Filter current status info + current_status = recording_series['status_recording_idx'] + current_status_series = recording_status_df.loc[recording_status_df['Value'] == current_status, :].squeeze() + next_status_series = recording_status_df.loc[recording_status_df['Value'] == current_status+1, :].squeeze() + + print('function to apply:', next_status_series['ProcessFunction']) + + # Get processing function + function_status_process = getattr(RecordingHandler, next_status_series['ProcessFunction']) + + #Trigger process, if success update recording process record + try: + success_process, update_dict = function_status_process(recording_series, next_status_series) + + if success_process: + #Get dictionary of record process + key = recording_series['query_key'] + #Get values to update + next_status = next_status_series['Value'] + value_update = update_dict['value_update'] + field_update = next_status_series['UpdateField'] + + print('key to update', key) + + RecordingHandler.update_status_pipeline(key, next_status, field_update, value_update) + except Exception as err: + raise(err) + ## Send notification error, update recording to error + + time.sleep(2) + + + @staticmethod + def local_transfer_request(rec_series, status_series): + """ + Request a transfer from PNI to Tiger Cluster + Input: + rec_series (pd.Series) = Series with information about the recording + status_series (pd.Series) = Series with information about the next status of the recording (if neeeded) + Returns: + status_update (int) = 1 if recording status has to be updated to next step in recording.Recording + = 0 if recording status not to be changed + = -1 if recording status has to be updated to ERROR in recording.Recording + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + update_value_dict = RecordingHandler.default_update_value_dict + directory_path = status_series['FunctionField'] + + status_update = True + return (status_update, update_value_dict) + + #smbclient + ''' + if transfer_request['code'] == 'Accepted': + status_update = True + update_value_dict['value_update'] = transfer_request['task_id'] + + return (status_update, update_value_dict) + ''' + + @staticmethod + def local_transfer_check(rec_series, status_series): + """ + Check status of transfer from local to PNI + Input: + rec_series (pd.Series) = Series with information about the recording + status_series (pd.Series) = Series with information about the next status of the recording (if neeeded) + Returns: + status_update (int) = 1 if recording status has to be updated to next step in recording.Recording + = 0 if recording status not to be changed + = -1 if recording status has to be updated to ERROR in recording.Recording + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + update_value_dict = RecordingHandler.default_update_value_dict + id_task = status_series['FunctionField'] + + status_update = True + return (status_update, update_value_dict) + #smbclient copy check ??? + + @staticmethod + def modality_preingestion(rec_series, status_series): + """ + Ingest "first" tables of modality specific recordings + Input: + rec_series (pd.Series) = Series with information about the recording + status_series (pd.Series) = Series with information about the next status of the recording (if neeeded) + Returns: + status_update (int) = 1 if recording status has to be updated to next step in recording.Recording + = 0 if recording status not to be changed + = -1 if recording status has to be updated to ERROR in recording.Recording + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + update_value_dict = RecordingHandler.default_update_value_dict + + # Get fieldname for processing unit for current recording modality + process_unit_dir_fieldname = \ + config.recording_modality_df.loc[config.recording_modality_df['RecordingModality'] == rec_series['recording_modality'], + 'ProcessUnitDirectoryField'].squeeze() + + if rec_series['recording_modality'] == 'electrophysiology': + + this_modality_recording_table = ephys.EphysRecording + this_modality_recording_unit_table = ephys.EphysRecordingProbes + this_modality_processing_unit_table = ephys.EphysProcessing + + print('this_modality_recording_table', this_modality_recording_table) + + # Insert this modality recording and recording "unit" + this_modality_recording_table.populate(rec_series['query_key']) + this_modality_recording_unit_table.populate(rec_series['query_key']) + + elif rec_series['recording_modality'] == 'imaging': + + this_modality_recording_table = imaging_rec.Scan + this_modality_recording_unit_table = imaging_rec.FieldOfView + this_modality_processing_unit_table = imaging_rec.ImagingProcessing + + print('imaging scan insert', rec_series) + this_modality_recording_table.populate(rec_series['query_key']) + + # Insert this modality recording and recording "unit" + # ------------- Call matlab insertion --------- + + # Get all recording probes ("units") from this recording + recording_units = (this_modality_recording_unit_table & rec_series['query_key']).fetch(as_dict=True) + + print('recording_units', recording_units) + + if len(recording_units) > 0: + # Select only primary keys for recording unit + rec_units_primary_key_fields = dj_short.get_primary_key_fields(this_modality_recording_unit_table) + + #Insert recording Process for all ("units") (one by one to get matching recording process id) + connection = recording.RecordingProcess.connection + with connection.transaction: + for rec_unit in recording_units: + + #Insert recording process and associated ephysProcessing records for this probe + rec_process_table = recording.RecordingProcess() + rec_process_table.insert_recording_process(rec_series, rec_unit, process_unit_dir_fieldname) + recording_process = recording.RecordingProcess.fetch('recording_process_id', order_by='recording_process_id DESC', limit=1) + + # Get recording unit key fields + recording_unit_key = {k: v for k, v in rec_unit.items() if k in rec_units_primary_key_fields} + + print('recording_process', recording_process) + + #Insert modality processing unit + this_mod_processing = recording_unit_key.copy() + this_mod_processing['recording_process_id'] = recording_process[0] + + print('this_mod_processing', this_mod_processing) + + this_modality_processing_unit_table.insert1(this_mod_processing) + status_update = True + + return (status_update, update_value_dict) + + + @staticmethod + def get_active_recordings(): + ''' + get all recordings that have to go through some action in the pipeline + Return: + df_recordings (pd.DataFrame): all recordings that are going to be processed in the pipeline + ''' + + status_query = 'status_recording_idx > ' + str(recording_status_df['Value'].min()) + status_query += ' and status_recording_idx < ' + str(recording_status_df['Value'].max()) + + recordings_active = recording.Recording & status_query + df_recordings = pd.DataFrame(recordings_active.fetch(as_dict=True)) + + if df_recordings.shape[0] > 0: + key_list = dj_short.get_primary_key_fields(recording.Recording) + df_recordings['query_key'] = df_recordings.loc[:, key_list].to_dict(orient='records') + + return df_recordings + + + @staticmethod + def update_status_pipeline(recording_key_dict, status, update_field=None, update_value=None): + """ + Update recording.Recording table status and optional task field + Args: + recording_key_dict (dict): key to find recording record + status (int): value of the status to be updated + update_field (str): name of the field to be updated as extra (only applicable to some status) + update_value (str|int): field value to be inserted on in task_field + """ + + print('recording_key_dict', recording_key_dict) + print('status', status) + print('update_field', update_field) + print('update_value', update_value) + + if update_field is not None: + update_task_id_dict = recording_key_dict.copy() + update_task_id_dict[update_field] = update_value + print('update_task_id_dict', update_task_id_dict) + recording.Recording.update1(update_task_id_dict) + + update_status_dict = recording_key_dict.copy() + update_status_dict['status_recording_idx'] = status + print('update_status_dict', update_status_dict) + recording.Recording.update1(update_status_dict) diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py new file mode 100644 index 00000000..82052c66 --- /dev/null +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -0,0 +1,258 @@ +import os +import pathlib +import subprocess +import json +import time +import re +import pandas as pd +import datajoint as dj +import u19_pipeline.recording as recording +import u19_pipeline.utils.dj_shortcuts as dj_short +import u19_pipeline.automatic_job.file_transfers as ft +import u19_pipeline.automatic_job.slurm_creator as slurmlib +import u19_pipeline.automatic_job.params_config as config + +recording_process_status_df = pd.DataFrame(config.recording_process_status_dict) + +class RecProcessHandler(): + + default_update_value_dict ={ + 'value_update': None, + 'error_info': None + } + + @staticmethod + def pipeline_handler_main(): + ''' + Call all processing functions according to the current status of each process job + Update status of each process job accordingly + ''' + + #Get info from all the possible status for a processjob + df_all_process_job = RecProcessHandler.get_active_process_jobs() + + #For all active process jobs + for i in range(df_all_process_job.shape[0]): + + #Filter current process job + rec_process_series = df_all_process_job.loc[i, :] + + preprocess_paramset = recording.PreprocessParamSet.get_preprocess_params(rec_process_series['preprocess_paramset_idx']) + process_paramset = recording.ProcessParamSet.get_process_params(rec_process_series['process_paramset_idx']) + + #Get params inside the recording process series + rec_process_series['preprocess_paramset'] = preprocess_paramset + rec_process_series['process_paramset'] = process_paramset + + #Filter current status info + curent_status = rec_process_series['status_pipeline_idx'] + current_status_series = df_status_pipeline.loc[df_status_pipeline['Value'] == current_status, :].squeeze() + next_status_series = df_status_pipeline.loc[df_status_pipeline['Value'] == current_status+1, :].squeeze() + + # Get processing function + function_status_process = next_status_series['ProcessFunction'] + + #Trigger process, if success update recording process record + success_process, update_dict = function_status_process(rec_process_series, next_status_series, preprocess_paramset, process_paramset) + + if success_process: + #Get dictionary of record process + key = rec_process_series['query_key'] + #Get values to update + next_status = next_status_series['Value'] + value_update = update_dict['value_update'] + field_update = next_status_series['UpdateField'] + + RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) + + time.sleep(2) + + @staticmethod + def transfer_request(rec_series, status_series, preprocess_paramset, process_paramset) + """ + Request a transfer from PNI to Tiger Cluster + Input: + rec_series (pd.Series) = Series with information about the recording process + status_series (pd.Series) = Series with information about the next status of the process (if neeeded) + Returns: + status_update (int) = 1 if recording process status has to be updated to next step in recording.RecordingProcess + = 0 if recording process status not to be changed + = -1 if recording process status has to be updated to ERROR in recording.RecordingProcess + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + update_value_dict = RecProcessHandler.default_update_value_dict + directory_path = status_series['FunctionField'] + + if status_series['Key'] is 'RAW_FILE_TRANSFER_REQUEST': + transfer_request = globus_transfer_raw_ephys_pni_tiger(directory_path) + elif status_series['Key'] is 'PROC_FILE_TRANSFER_REQUEST': + #ALS, which recording directory for processed file + transfer_request = globus_transfer_sorted_ephys_tiger_pni(directory_path) + + if transfer_request['code'] == 'Accepted': + status_update = True + update_value_dict['value_update'] = transfer_request['task_id'] + + return (status_update, update_value_dict) + + @staticmethod + def transfer_check(rec_series, status_series): + """ + Check status of globus transfer from local to PNI + Input: + rec_series (pd.Series) = Series with information about the recording process + status_series (pd.Series) = Series with information about the next status of the process (if neeeded) + Returns: + status_update (int) = 1 if recording process status has to be updated to next step in recording.RecordingProcess + = 0 if recording process status not to be changed + = -1 if recording process status has to be updated to ERROR in recording.RecordingProcess + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + value_update = RecProcessHandler.default_update_value_dict + id_task = status_series['FunctionField'] + + transfer_request = request_globus_transfer_status(str(id_task)) + if transfer_request['status'] == 'SUCCEEDED': + status_update = True + + return (status_update, update_value_dict) + + @staticmethod + def slurm_job_queue(rec_series, status_series): + """ + Request a transfer from local machine to PNI drive + Input: + rec_series (pd.Series) = Series with information about the recording process + status_series (pd.Series) = Series with information about the next status of the process (if neeeded) + Returns: + status_update (int) = 1 if recording process status has to be updated to next step in recording.RecordingProcess + = 0 if recording process status not to be changed + = -1 if recording process status has to be updated to ERROR in recording.RecordingProcess + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + update_value_dict = RecProcessHandler.default_update_value_dict + + slurm_text = slurmlib.generate_slurm_file(rec_series) + slurm_file_name = 'slurm_' + create_str_from_dict(key) + '.slurm' + slurm_file_path = str(pathlib.Path("slurm_files",slurm_file_name)) + + write_slurm_file(slurm_file_path, slurm_text) + + tiger_slurm_user = default_user+'@'+tiger_gpu_host + tiger_slurm_location = tiger_slurm_user+':'+tiger_slurm_files_dir+slurm_file_name + transfer_request = scp_file_transfer(slurm_file_path, tiger_slurm_location) + + if transfer_request == config.system_process['SUCCESS']: + slurm_queue_status, slurm_jobid = queue_slurm_file(tiger_slurm_user, tiger_slurm_files_dir+slurm_file_name) + + if transfer_request == config.system_process['SUCCESS']: + status_update = True + update_status_pipeline(key, status_dict['JOB_QUEUE']['Task_Field'], slurm_jobid, status_dict['JOB_QUEUE']['Value']) + + return (status_update, update_value_dict) + + @staticmethod + def slurm_job_check(rec_series, status_series): + """ + Check slurm job in cluster machine + Input: + rec_series (pd.Series) = Series with information about the recording process + status_series (pd.Series) = Series with information about the next status of the process (if neeeded) + Returns: + status_update (int) = 1 if recording process status has to be updated to next step in recording.RecordingProcess + = 0 if recording process status not to be changed + = -1 if recording process status has to be updated to ERROR in recording.RecordingProcess + update_value_dict (dict) = Dictionary with next keys: + {'value_update': value to be updated in this stage (if applicable) + 'error_info': error info to be inserted if error occured } + """ + + status_update = False + value_update = RecProcessHandler.default_update_value_dict + slurm_jobid = status_series['FunctionField'] + + ssh_user = tiger_slurm_user = default_user+'@'+tiger_gpu_host + job_status = check_slurm_job(ssh_user, slurm_jobid) + + print('job status', job_status) + print(slurm_states['SUCCESS']) + + print('job status encode uft8 ', job_status.encode('UTF-8')) + print(slurm_states['SUCCESS'].encode('UTF-8')) + + if job_status == slurm_states['SUCCESS']: + status_update = True + + return (status_update, update_value_dict) + + @staticmethod + def get_active_process_jobs(): + ''' + get all process jobs that have to go through some action in the pipeline + Return: + df_process_jobs (pd.DataFrame): all jobs that are going to be processed in the pipeline + ''' + + status_query = 'status_pipeline_idx > ' + str(recording_process_status_df['Value'].min()) + status_query += ' and status_pipeline_idx < ' + str(recording_process_status_df['Value'].max()) + + jobs_active = recording.RecordingProcess & status_query + df_process_jobs = pd.DataFrame(jobs_active.fetch(as_dict=True)) + + if df_process_jobs.shape[0] > 0: + key_list = dj_short.get_primary_key_fields(recording.RecordingProcess) + df_process_jobs['query_key'] = df_process_jobs.loc[:, key_list].to_dict(orient='records') + + return df_process_jobs + + +RecProcessHandler.get_preprocess_params(rec_process_series['query_key']) + + @staticmethod + def update_status_pipeline(recording_process_key_dict, status, update_field=None, update_value=None): + """ + Update recording.RecordingProcess table status and optional task field + Args: + recording_process_key_dict (dict): key to find recording_process record + status (int): value of the status to be updated + update_field (str): name of the field to be updated as extra (only applicable to some status) + update_value (str|int): field value to be inserted on in task_field + """ + + if update_field is not None: + update_task_id_dict = recording_process_key_dict.copy() + update_task_id_dict[update_field] = update_value + recording.RecordingProcess.update1(update_task_id_dict) + + update_status_dict = recording_process_dict.copy() + update_status_dict['status_pipeline_idx'] = status + recording.RecordingProcess.update1(update_status_dict) + + + ''' + @staticmethod + def filter_session_status(df_rec_process, status): + """ + Filter dataframe with rec_process with a given status + Args: + df_rec_process (pd.DataFrame): recording process dataframe + status (int): value of the status to be filtered with + Returns: + df_rec_process_status (pd.DataFrame): recording process dataframe filtered with given status + """ + + df_rec_process_status = df_rec_process.loc[df_sessions['status_pipeline_idx'] == status, :] + df_rec_process_status = df_rec_process_status.reset_index(drop=True) + ''' diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py new file mode 100644 index 00000000..531772e9 --- /dev/null +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -0,0 +1,121 @@ + + +#import os +#import pathlib +import subprocess +import json +import re + +# Functions to create slurm jobs + +#Slurm default values for queue job +slurm_dict_default = { + 'job-name': 'kilosort2', + 'nodes': 1, + 'ntasks': 1, + 'time': '5:00:00', + 'mem': '200G', + 'gres': 'gpu:1', + 'mail-user': 'alvaros@princeton.edu', + 'mail-type': ['begin', 'END'], + 'output': 'job_log/kilojob.log' +} + +slurm_states = { + 'SUCCESS': 'COMPLETED' +} + + +default_preprocessing_tool = 'kilosort2' +default_matlab_ver = 'R2020b' + +def generate_slurm_file(record_process_series): + + + rel_path = record_process_series['acquisition_raw_rel_path'] + key = record_process_series['query_key'] + + slurm_dict = slurm_dict_default.copy() + slurm_dict['job-name'] = default_preprocessing_tool + "_" + create_str_from_dict(key) + slurm_dict['output'] = str(pathlib.Path(tiger_log_files_dir,default_preprocessing_tool + create_str_from_dict(key) + '.log')) + + slurm_text = generate_slurm_kilosort_text(slurm_dict, default_matlab_ver, default_user, raw_rel_path) + slurm_file_name = 'slurm_' + create_str_from_dict(key) + '.slurm' + slurm_file_path = str(pathlib.Path("slurm_files",slurm_file_name)) + + write_slurm_file(slurm_file_path, slurm_text) + + tiger_slurm_user = default_user+'@'+tiger_gpu_host + tiger_slurm_location = tiger_slurm_user+':'+tiger_slurm_files_dir+slurm_file_name + transfer_request = scp_file_transfer(slurm_file_path, tiger_slurm_location) + + +def create_slurm_params_file(slurm_dict): + + text_dict = '' + for slurm_param in slurm_dict.keys(): + + if isinstance(slurm_dict[slurm_param], list): + for list_param in slurm_dict[slurm_param]: + text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(list_param) + '\n' + else: + text_dict += '#SBATCH --' + str(slurm_param) + '=' + str(slurm_dict[slurm_param]) + '\n' + + return text_dict + + +def write_slurm_file(slurm_path, slurm_text): + + f_slurm = open(slurm_path, "w") + f_slurm.write(slurm_text) + f_slurm.close() + + +def generate_slurm_kilosort_text(slurm_dict, matlab_ver, user_run, raw_file_path): + + slurm_text = '#!/bin/bash\n' + slurm_text += create_slurm_params_file(slurm_dict) + slurm_text += 'module load matlab/' + matlab_ver + '\n' + slurm_text += 'cd /tigress/' + user_run + '\n' + slurm_text += 'matlab -singleCompThread -nodisplay -nosplash -r "pause(1); ' + "disp('aqui la chides'); exit" + '"' + #slurm_text += 'matlab -singleCompThread -nodisplay -nosplash -r "addpath(''/tigress/' + user_run + "/run_kilosort/spikesorters/'); " + #slurm_text += "run_ks2('/tigress/" + user_run + "/ephys_raw" + raw_file_path + "','/tigress/" + user_run + "/run_kilosort/tmp/'); exit" + '"' + + return slurm_text + + +def queue_slurm_file(ssh_user, slurm_file): + + id_slurm_job = -1 + command = ['ssh', ssh_user, 'sbatch', slurm_file] + print(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + print('aftercommand before comm') + stdout, stderr = p.communicate() + print('aftercommand after comm') + print(stdout.decode('UTF-8')) + print(stderr.decode('UTF-8')) + if p.returncode == system_process['SUCCESS']: + batch_job_sentence = stdout.decode('UTF-8') + id_slurm_job = batch_job_sentence.replace("Submitted batch job ","") + id_slurm_job = re.sub(r"[\n\t\s]*", "", id_slurm_job) + + return p.returncode, id_slurm_job + + +def check_slurm_job(ssh_user, jobid): + + state_job = 'FAIL' + command = ['ssh', ssh_user, 'sacct', '--job', jobid, '--format=state'] + print(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + stdout, stderr = p.communicate() + if p.returncode == system_process['SUCCESS']: + stdout = stdout.decode('UTF-8') + state_job = stdout.split("\n")[2].strip() + print(stdout) + + return state_job + diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index 0db205ca..2074bb23 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -9,6 +9,8 @@ import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX import u19_pipeline.utils.ephys_utils as ephys_utils +import u19_pipeline.utils.path_utils as pu +import u19_pipeline.automatic_job.params_config as config from u19_pipeline.utils.DemoReadSGLXData.readSGLX import readMeta @@ -36,7 +38,7 @@ # 2. Upstream tables schema_reference = dj.schema(dj.config['custom']['database.prefix'] + 'reference') -schema = dj.schema(dj.config['custom']['database.prefix'] + 'ephys') +schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'ephys') @schema_reference @@ -56,27 +58,57 @@ class EphysRecording(dj.Computed): """ @property def key_source(self): - return recording.Recording & {'recording_modality': 'ephys'} + return recording.Recording & {'recording_modality': 'electrophysiology'} def make(self, key): self.insert1(key) @schema -class EphysProcessing(dj.Computed): +class EphysRecordingProbes(dj.Computed): definition = """ + # General information of an ephys session -> EphysRecording - -> recording.RecordingProcess - ----- - """ + probe : tinyint # probe number for the recording + --- + probe_directory : varchar(255) # probe specific directory + """ def make(self, key): - self.insert1(key) + + root_dir = get_ephys_root_data_dir() + rec_diro = (recording.Recording & key).fetch1('recording_directory') + rec_dir = str(pathlib.Path(root_dir, rec_diro)) + + ephys_probe_pattern = \ + config.recording_modality_df.loc[config.recording_modality_df['RecordingModality'] == 'electrophysiology', 'ProcessUnitFilePattern'].squeeze() + + probe_dirs = pu.get_filepattern_paths(rec_dir, ephys_probe_pattern[0]) + + probe_keys = [] + for idx, probe_dir in enumerate(probe_dirs): + probe_dir = probe_dir.replace(root_dir, "") + this_key = key.copy() + this_key['probe'] = idx + this_key['probe_directory'] = probe_dir + probe_keys.append(this_key) + + if len(probe_keys) > 0: + self.insert(probe_keys) + + +@schema +class EphysProcessing(dj.Manual): + definition = """ + -> recording.RecordingProcess + ----- + -> EphysRecordingProbes + """ # ephys element requires table with name Session -Session = EphysSorting +Session = EphysProcessing # 3. Utility functions @@ -110,7 +142,7 @@ def get_session_directory(session_key): @schema class BehaviorSync(dj.Imported): definition = """ - -> ephys.EphysRecording + -> EphysRecording --- nidq_sampling_rate : float # sampling rate of behavioral iterations niSampRate in nidq meta file iteration_index_nidq : longblob # Virmen index time series. Length of this longblob should be the number of samples in the nidaq file. diff --git a/u19_pipeline/imaging.py b/u19_pipeline/imaging.py index 03571e43..df5a0f1b 100644 --- a/u19_pipeline/imaging.py +++ b/u19_pipeline/imaging.py @@ -45,6 +45,11 @@ class ScanInfo(dj.Imported): nframes : int # number of frames in the scan nframes_good : int # number of frames in the scan before acceptable sample bleaching threshold is crossed last_good_file : int # number of the file containing the last good frame because of bleaching + motion_correction_enabled=0 : tinyint # + motion_correction_mode='N/A': varchar(64) # + stacks_enabled=0 : tinyint # + stack_actuator='N/A' : varchar(64) # + stack_definition='N/A' : varchar(64) # """ @@ -61,8 +66,9 @@ class FieldOfView(dj.Imported): fov_center_xy : blob # X-Y coordinate for the center of the FOV in microns. One for each FOV in scan fov_size_xy : blob # X-Y size of the FOV in microns. One for each FOV in scan (sizeXY) fov_rotation_degrees : float # rotation of the FOV with respect to cardinal axes in degrees. One for each FOV in scan - fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan - fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? + fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan + fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? + power_percent : float # percentage of power used for this field of view """ class File(dj.Part): diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 926c2f43..7c41c377 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -2,7 +2,7 @@ from u19_pipeline import acquisition, subject, recording -schema = dj.schema(dj.config['custom']['database.prefix'] + 'imaging_rec') +schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'imaging_rec') @schema @@ -17,18 +17,9 @@ def key_source(self): return recording.Recording & {'recording_modality': 'imaging'} def make(self, key): - self.insert1(key) - -@schema -class ImagingProcessing(dj.Computed): - definition = """ - -> Scan - -> recording.RecordingProcess - ----- - """ + print('population here....', key) - def make(self, key): self.insert1(key) @@ -64,6 +55,11 @@ class ScanInfo(dj.Imported): nframes : int # number of frames in the scan nframes_good : int # number of frames in the scan before acceptable sample bleaching threshold is crossed last_good_file : int # number of the file containing the last good frame because of bleaching + motion_correction_enabled=0 : tinyint # + motion_correction_mode='N/A': varchar(64) # + stacks_enabled=0 : tinyint # + stack_actuator='N/A' : varchar(64) # + stack_definition='N/A' : varchar(64) # """ @@ -81,8 +77,9 @@ class FieldOfView(dj.Imported): fov_center_xy : blob # X-Y coordinate for the center of the FOV in microns. One for each FOV in scan fov_size_xy : blob # X-Y size of the FOV in microns. One for each FOV in scan (sizeXY) fov_rotation_degrees : float # rotation of the FOV with respect to cardinal axes in degrees. One for each FOV in scan - fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan - fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? + fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan + fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? + power_percent : float # percentage of power used for this field of view """ class File(dj.Part): @@ -94,3 +91,12 @@ class File(dj.Part): fov_filename : varchar(255) # file name of the new fov tiff file file_frame_range : blob # [first last] frame indices in this file, with respect to the whole imaging session """ + + +@schema +class ImagingProcessing(dj.Manual): + definition = """ + -> recording.RecordingProcess + ----- + -> FieldOfView + """ \ No newline at end of file diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 0e40296c..58604867 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -1,57 +1,60 @@ import datajoint as dj +import numpy as np from u19_pipeline import lab, task, subject, acquisition +import u19_pipeline.automatic_job.params_config as config -schema = dj.schema(dj.config['custom']['database.prefix'] + 'recording') +schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'recording') +@schema +class RecordingModality(dj.Lookup): + definition = """ + recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) + --- + modality_description: varchar(255) # description for the modality + root_direcory: varchar(255) # root directory where that modality is stored + recording_file_extensions: blob # file extensions specific for this modality + recording_file_pattern: blob # directory pattern to find recordings in path + process_unit_file_pattern: blob # process "unit" pattern to find in path + process_unit_dir_fieldname: varchar(64) # FieldName that stores process unit directory for specific modality + """ + contents = config.recording_modality_list @schema -class RecordingModality(dj.Lookup): +class StatusRecordingDefinition(dj.Lookup): definition = """ - recording_modality: varchar(64) # modalities for recording (ephys, imaging, video_recording, etc.) + status_recording_idx: TINYINT(1) # status in the automate process pipeline --- - modality_description: varchar(255) # description for the modality - root_directory: varchar(255) # root directory where that modality is stored - recording_file_extensions: blob # file extensions specific for this modality + status_definition: VARCHAR(256) # Status definition """ - contents = [ -<<<<<<< Updated upstream - ['ephys', '', '/braininit/Data/electrophysiology', ['ap.bin', 'ap.meta']], - ['imaging', '', '/braininit/Data/imaging', ['.avi', '.tiff','.tif']], -======= - ['electrophysiology', '', '/braininit/Data/eletrophysiology', ['ap.bin', 'ap.meta']], - ['imaging', '', '/braininit/Data/eletrophysiology', ['.avi', '.tiff','.tif']], ->>>>>>> Stashed changes - ['video_acquisition', '', '/braininit/Data/video_acquisition', ['.avi', '.mp4']] - ] + contents = config.recording_status_list @schema -class PreprocessingParamSet(dj.Lookup): +class PreprocessParamSet(dj.Lookup): definition = """ # Parameter set to be used in the preprocessing steps - preprocessing_paramset_idx: int(11) AUTO_INCREMENT + preprocess_paramset_idx: int(11) AUTO_INCREMENT --- - -> RecordingModality - preprocessing_paramset_desc: varchar(128) - preprocessing_paramset_hash: uuid - unique index (preprocessing_paramset_hash) - preprocessing_params: longblob # dictionary of all applicable parameters + -> RecordingModality + preprocess_paramset_desc: varchar(128) + preprocess_paramset_hash: uuid + unique index (preprocess_paramset_hash) + preprocess_paramset: longblob # dictionary of all applicable parameters """ @classmethod - def insert_new_params(cls, recording_modality: str, paramset_idx: int, - paramset_desc: str, params: dict): - param_dict = {'recording_modality': recording_modality, - 'preprocessing_paramset_idx': paramset_idx, - 'preprocessing_paramset_desc': paramset_desc, - 'preprocessing_params': params, - 'preprocessing_paramset_hash': dict_to_uuid(params)} - param_query = cls & {'preprocessing_paramset_hash': param_dict['preprocessing_paramset_hash']} - - if param_query: # If the specified param-set already exists - existing_paramset_idx = param_query.fetch1('preprocessing_paramset_idx') - if existing_paramset_idx == paramset_idx: # If the existing set has the same paramset_idx: job done + def insert_new_params(cls, recording_modality: str, preprocess_paramset_idx: int, + preprocess_paramset_desc: str, preprocess_paramset: dict): + paramset_dict = {'recording_modality': recording_modality, + 'preprocess_paramset_idx': preprocess_paramset_idx, + 'preprocess_paramset_desc': preprocess_paramset_desc, + 'preprocess_paramset': preprocess_paramset, + 'preprocess_paramset_hash': dict_to_uuid(preprocess_paramset)} + paramset_query = cls & {'preprocess_paramset_hash': paramset_dict['preprocess_paramset_hash']} + if paramset_query: # If the specified param-set already exists + existing_paramset_idx = paramset_query.fetch1('preprocess_paramset_idx') + if existing_paramset_idx == preprocess_paramset_idx: # If the existing set has the same preprocess_paramset_idx: job done return else: # If not same name: human error, trying to add the same paramset with different name raise dj.DataJointError( @@ -60,137 +63,155 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, else: cls.insert1(paramset_dict) + @classmethod + def get_preprocess_params(preprocess_param_idx): + ''' + Get process params for current recording process + Return: + preprocess_paramset (dict): preprocess params associated with recording process + ''' + + preprocess_paramset = (recording.ProcessParamSet & preprocess_param_idx).fetch1('preprocess_paramset') + return preprocess_paramset + @schema -class ProcessingParamSet(dj.Lookup): +class ProcessParamSet(dj.Lookup): definition = """ - # Parameter set to be used in the preprocessing steps - processing_paramset_idx: int(11) AUTO_INCREMENT + # Parameter set to be used in the processing steps + process_paramset_idx: int(11) AUTO_INCREMENT --- -> RecordingModality - processing_paramset_desc: varchar(128) - processing_paramset_hash: uuid - unique index (processing_paramset_hash) - processing_params: longblob # dictionary of all applicable parameters + process_paramset_desc: varchar(128) + process_paramset_hash: uuid + unique index (process_paramset_hash) + process_paramset: longblob # dictionary of all applicable parameters """ @classmethod def insert_new_params(cls, recording_modality: str, paramset_idx: int, paramset_desc: str, params: dict): - param_dict = {'recording_modality': recording_modality, - 'processing_paramset_idx': paramset_idx, - 'processing_paramset_desc': paramset_desc, - 'processing_params': params, - 'processing_paramset_hash': dict_to_uuid(params)} - param_query = cls & {'processing_paramset_hash': param_dict['processing_paramset_hash']} + paramset_dict = {'recording_modality': recording_modality, + 'process_paramset_idx': paramset_idx, + 'process_paramset_desc': paramset_desc, + 'process_paramset': params, + 'process_paramset_hash': dict_to_uuid(params)} + param_query = cls & {'process_paramset_hash': paramset_dict['process_paramset_hash']} if param_query: # If the specified param-set already exists - existing_paramset_idx = param_query.fetch1('processing_paramset_idx') + existing_paramset_idx = param_query.fetch1('process_paramset_idx') if existing_paramset_idx == paramset_idx: # If the existing set has the same paramset_idx: job done return else: # If not same name: human error, trying to add the same paramset with different name raise dj.DataJointError( 'The specified param-set' - ' already exists - paramset_idx: {}'.format(existing_paramset_idx)) + ' already exists - process_paramset_idx: {}'.format(existing_paramset_idx)) else: - cls.insert1(param_dict) + cls.insert1(paramset_dict) + + @staticmethod + def get_process_params(process_param_idx): + ''' + Get process params for current recording process + Return: + process_paramset (dict): process params associated with recording process + ''' + + process_paramset = (recording.ProcessParamSet & process_param_idx).fetch1('process_paramset') + return process_paramset + @schema class Recording(dj.Manual): definition = """ recording_id: INT(11) AUTO_INCREMENT # Unique number assigned to each recording ----- - -> [nullable] acquisition.Session # acquisition Session key - -> RecordingModality -<<<<<<< Updated upstream - recording_directory: varchar(255) # relative directory where the data for this session will be stored on cup -======= - -> lab.Location - recording_directory: varchar(255) # the relative directory where the ephys data for this session will be stored in braininit drive - local_directory: varchar(255) # local directory where this file is stored on the recording system - -> [nullable] subject.Subject.proj(acquisition_subject='subject_fullname') # Recording subject when no behavior Session present - recording_datetime=null: datetime # Recording datetime when no behavior Session present ->>>>>>> Stashed changes + -> RecordingModality + -> lab.Location + -> StatusRecordingDefinition # current status for recording in the pipeline + -> PreprocessParamSet # reference to params to preprocess recording (possible to inherit to recordigprocess) + -> ProcessParamSet # reference to params to process recording (possible to inherit to recordigprocess) + task_copy_id_pni=null: UUID # id for globus transfer task raw file local->cup + inherit_params_recording=1: boolean # all RecordingProcess from a recording will have same paramSets + recording_directory: varchar(255) # relative directory where the recording will be stored on cup + local_directory: varchar(255) # local directory where the recording is stored on system """ - - -status_pipeline_dict = { - 'ERROR': {'Value': -1, - 'Label': 'Error in process', - 'Task_Field': None}, - 'NEW_SESSION': {'Value': 0, - 'Label': 'New session', - 'Task_Field': None}, - 'PNI_DRIVE_TRANSFER_REQUEST': {'Value': 1, - 'Label': 'Recording directory transfer to PNI requested', - ' Task_Field': 'task_copy_id_pni'}, - 'PNI_DRIVE_TRANSFER_END': {'Value': 2, - 'Label': 'Recording directory transferred to PNI', - ' Task_Field': None}, - 'MODALITY_PREINGESTION': {'Value': 3, - 'Label': 'Preprocessing & Syncing jobs', - ' Task_Field': None}, - 'RAW_FILE_REQUEST': {'Value': 4, - 'Label': 'Raw file transfer requested', - 'Task_Field': 'task_copy_id_pre_path'}, - 'RAW_FILE_CLUSTER': {'Value': 5, - 'Label': 'Raw file transferred to cluster', - 'Task_Field': None}, - 'JOB_QUEUE': {'Value': 6, - 'Label': 'Processing job in queue', - 'Task_Field': 'slurm_id_sorting'}, - 'JOB_FINISHED': {'Value': 7, - 'Label': 'Processing job finished', - 'Task_Field': None}, - 'PROC_FILE_REQUEST': {'Value': 8, - 'Label': 'Processed file transfer requested', - 'Task_Field': 'task_copy_id_pos_path'}, - 'PROC_FILE_HOME': {'Value': 9, - 'Label': 'Processed file transferred to PNI', - 'Task_Field': None}, - 'CANONICAL_PIPELINE': {'Value': 10, - 'Label': 'Processed with Canonical pipeline', - 'Task_Field': None}, -} - -def get_content_list_from_status_dict(): - return [[status_pipeline_dict[i]['Value'], status_pipeline_dict[i]['Label']] for i in status_pipeline_dict.keys()] + + class BehaviorSession(dj.Part): + definition = """ + -> master + --- + -> acquisition.Session + """ + + class RecordingSession(dj.Part): + definition = """ + -> master + --- + -> subject.Subject + recording_datetime: datetime + """ @schema class StatusProcessDefinition(dj.Lookup): definition = """ - status_pipeline_idx: TINYINT(1) # status in the automate process pipeline + status_pipeline_idx: TINYINT(1) # status in the automate process pipeline --- status_definition: VARCHAR(256) # Status definition """ - contents = get_content_list_from_status_dict() + contents = config.recording_process_status_list @schema class RecordingProcess(dj.Manual): definition = """ - recording_process_id: INT(11) AUTO_INCREMENT # Unique number assigned to each processing job for a recording + recording_process_id: INT(11) AUTO_INCREMENT # Unique number assigned to each processing job for a recording unit ----- -> Recording -> StatusProcessDefinition # current status in the pipeline - -> PreprocessingParamSet # reference to params to preprocess recording - -> ProcessingParamSet # reference to params to process recording - recording_process_path=null: VARCHAR(200) # relative path for processed recording - task_copy_id_pni=null: UUID # id for globus transfer task raw file local->cup + -> PreprocessParamSet # reference to params to preprocess recording + -> ProcessParamSet # reference to params to process recording + recording_process_pre_path=null: VARCHAR(200) # relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview) + recording_process_post_path=null: VARCHAR(200) # relative path for processed data recording task_copy_id_pre=null: UUID # id for globus transfer task raw file cup->tiger - task_copy_id_pos=null: UUID # id for globus transfer task sorted file tiger->cup + task_copy_id_post=null: UUID # id for globus transfer task sorted file tiger->cup slurm_id=null: VARCHAR(16) # id for slurm process in tiger - """ + """ + + def insert_recording_process(self, recording_key, rec_unit, unit_directory_fieldname): + ''' + #Insert RecordingProcess(es) from recording. + # For each processing "unit" of a recording add a new recordingProcess (imaging ->field of view, electrophysiology->probe) + Input: + recording_key (dict) = Dictionary with recording record + rec_unit (dict) = Dictionary of recording "unit" to be processed + unit_directory_fieldname (str) = Unit directory fieldname to be read (ephys-> probe_directory, imaging->fov_directory) + ''' + + # Get directory fieldname for specific modality (probe_directory, fov_directory, etc.) + + #Append data for the unit to insert + this_recprocess_key = dict() + this_recprocess_key['recording_id'] = recording_key['recording_id'] + this_recprocess_key['preprocess_paramset_idx'] = recording_key['preprocess_paramset_idx'] + this_recprocess_key['process_paramset_idx'] = recording_key['process_paramset_idx'] + this_recprocess_key['recording_process_pre_path'] = rec_unit[unit_directory_fieldname] + this_recprocess_key['status_pipeline_idx'] = 0 + + print('this_recprocess_key', this_recprocess_key) + + self.insert1(this_recprocess_key) @schema class RecordingProcessStatus(dj.Manual): definition = """ - -> RecordingProcess + recording_process_status_id: INT(11) AUTO_INCREMENT # Unique number assigned to each change of status for all processing jobs ----- - -> StatusProcessDefinition.proj(status_pipeline_old='status_pipeline') # old status in the pipeline - -> StatusProcessDefinition.proj(status_pipeline_new='status_pipeline') # current status in the pipeline + -> RecordingProcess + -> StatusProcessDefinition.proj(status_pipeline_idx_old='status_pipeline_idx') # old status in the pipeline + -> StatusProcessDefinition.proj(status_pipeline_idx_new='status_pipeline_idx') # current status in the pipeline status_timestamp: DATETIME # timestamp when status change ocurred error_message=null: VARCHAR(4096) # Error message if status now is failed error_exception=null: BLOB # Error exception if status now is failed diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index d477e3fd..64f9242f 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -85,7 +85,7 @@ def get_network_path(path_name): network_path = (lab.Path & key).fetch1(*field_get) return network_path - +# Function copied in dj_shorts def smart_dj_join(t1, t2): """ Join two datajoint tables even if they have matching secondary field names @@ -274,3 +274,10 @@ def get_cols_rows_plot(num_plots, fig_size): return num_rows, num_cols + +def create_str_from_dict(key_dict): + + slurm_file_name = '' + for i in key_dict.keys(): + slurm_file_name += str(i) + '_' + str(key_dict[i]) + return slurm_file_name \ No newline at end of file diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py new file mode 100644 index 00000000..61ae5081 --- /dev/null +++ b/u19_pipeline/utils/dj_shortcuts.py @@ -0,0 +1,51 @@ +import pandas as pd + +def get_primary_key_fields(t): + """ + Get list of all fields that compose primary key + Args: + t (Dj table): Instance of a table in datajoint + Returns: + primary_field_list: (list): List of all fields that make primary key + """ + + fields_t = pd.DataFrame.from_dict(t.heading.attributes, orient='index') + primary_field_list = fields_t.loc[fields_t['in_key'] == True].index.to_list() + + return primary_field_list + + +# Function copied in dj_shorts +def smart_dj_join(t1, t2): + """ + Join two datajoint tables even if they have matching secondary field names + Matching secondary fields from t2 will be rewritten as 'field -> t2.table_name+"_"+field' + """ + + # Get all fields from tables + fields_t1 = pd.DataFrame.from_dict(t1.heading.attributes, orient='index') + fields_t2 = pd.DataFrame.from_dict(t2.heading.attributes, orient='index') + + # Get only secondary fields and check matches + fields_t1_list = set(fields_t1.loc[fields_t1['in_key'] == False].index.to_list()) + fields_t2_list = set(fields_t2.loc[fields_t2['in_key'] == False].index.to_list()) + intersected_fields = fields_t2_list.intersection(fields_t1_list) + + # If there are: + if len(intersected_fields) > 0: + # Create a dictionary to rename matching ones + suffix = t2.table_name + new_name_attr_dict = dict() + for i in intersected_fields: + new_name_attr_dict[suffix + '_' + i] = i + + # List non matching ones + non_intersected_fields = list(fields_t2_list - intersected_fields) + + # Finally merge + t = t1 * t2.proj(*non_intersected_fields, **new_name_attr_dict) + # If there are not, normal merge + else: + t = t1 * t2 + + return t diff --git a/u19_pipeline/utils/path_utils.py b/u19_pipeline/utils/path_utils.py index 492ffb7d..93fc391e 100644 --- a/u19_pipeline/utils/path_utils.py +++ b/u19_pipeline/utils/path_utils.py @@ -35,6 +35,14 @@ def check_file_pattern_dir(filepath, file_patterns): else: return 0 +def get_filepattern_paths(filepath, file_pattern): + """ + Check directory/files that correspond to file pattern in filepath + """ + found_patterns = glob.glob(filepath+file_pattern) + + return found_patterns + def get_size_directory(path): """ get directory size of a folder for linux systems From c6ec60d88ead7c970c6bf28738295ae53a3f3a99 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Mon, 14 Feb 2022 14:04:53 -0500 Subject: [PATCH 30/34] TestPipeline notebook v0.1 --- notebooks/TestAutomaticPipeline.ipynb | 333 +++++++++++++++++++++++++- 1 file changed, 327 insertions(+), 6 deletions(-) diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb index f4d188f9..7fa8943c 100644 --- a/notebooks/TestAutomaticPipeline.ipynb +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -55,14 +55,212 @@ "import pandas as pd\n", "import subprocess\n", "import json\n", + "import u19_pipeline.utils.path_utils as pu\n", + "import pathlib\n", + "import u19_pipeline.automatic_job as auto_pipeline\n", + "import u19_pipeline.automatic_job.recording_handler as rec_handler\n", "\n", - "from u19_pipeline import recording, ephys, imaging_rec\n", + "from u19_pipeline import recording, imaging_rec, ephys\n", "\n", "#import utility.dj_shortcuts as dj_short\n", "#import utility.acquisition_pipeline_handler as aph\n", "dj.__version__" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*status_record status_definit\n", + "+------------+ +------------+\n", + "-1 Error in recor\n", + "0 New recording \n", + "1 Recording dire\n", + "2 Recording dire\n", + "3 modality inges\n", + " (Total: 5)" + ], + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n
\n

status_recording_idx

\n status in the automate process pipeline\n
\n

status_definition

\n Status definition\n
-1Error in recording handling
0New recording
1Recording directory transfer to PNI requested
2Recording directory transferred to PNI
3modality ingestion & Syncing jobs done
\n \n

Total: 5

\n " + }, + "metadata": {}, + "execution_count": 3 + } + ], + "source": [ + "recording.StatusRecordingDefinition()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "fake_recording = dict()\n", + "fake_recording['recording_modality'] = 'electrophysiology'\n", + "fake_recording['location'] = 'Bezos1'\n", + "fake_recording['status_recording_idx'] = 0\n", + "fake_recording['process_paramset_idx'] = 1\n", + "fake_recording['preprocess_paramset_idx'] = 1\n", + "fake_recording['recording_directory'] = 'ms81/ms81_M004/20210507/towersTask_g0'\n", + "fake_recording['local_directory'] = 'ms81/ms81_M004/20210507/towersTask_g0'\n", + "\n", + "#recording.Recording.insert1(fake_recording, skip_duplicates=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "fake_recording = dict()\n", + "fake_recording['recording_modality'] = 'imaging'\n", + "fake_recording['location'] = 'Bezos1'\n", + "fake_recording['status_recording_idx'] = 0\n", + "fake_recording['process_paramset_idx'] = 1\n", + "fake_recording['preprocess_paramset_idx'] = 2\n", + "fake_recording['recording_directory'] = 'emdia/emdia_gps24/20220111'\n", + "fake_recording['local_directory'] = 'emdia/emdia_gps24/20220111'\n", + "\n", + "#recording.Recording.insert1(fake_recording, skip_duplicates=True)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "function to apply: modality_preingestion\n", + "imaging scan insert recording_id 4\n", + "recording_modality imaging\n", + "location Bezos1\n", + "status_recording_idx 2\n", + "preprocess_paramset_idx 2\n", + "process_paramset_idx 1\n", + "task_copy_id_pni None\n", + "inherit_params_recording 1\n", + "recording_directory emdia/emdia_gps24/20220111\n", + "local_directory emdia/emdia_gps24/20220111\n", + "query_key {'recording_id': 4}\n", + "Name: 0, dtype: object\n", + "recording_units [{'recording_id': 4, 'fov': 1, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'fov_name': 'mM2_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[-1662.46125877, 269.83897896]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 2, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'fov_name': 'RSP_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1142.3930918 , -99.24825482]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 3, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'fov_name': 'AM_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1555.00006185, 1097.92718087]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}]\n", + "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [21]\n", + "this_mod_processing {'recording_id': 4, 'fov': 1, 'recording_process_id': 21}\n", + "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [22]\n", + "this_mod_processing {'recording_id': 4, 'fov': 2, 'recording_process_id': 22}\n", + "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [23]\n", + "this_mod_processing {'recording_id': 4, 'fov': 3, 'recording_process_id': 23}\n", + "key to update {'recording_id': 4}\n", + "recording_key_dict {'recording_id': 4}\n", + "status 3\n", + "update_field None\n", + "update_value None\n", + "update_status_dict {'recording_id': 4, 'status_recording_idx': 3}\n" + ] + } + ], + "source": [ + "l = rec_handler.RecordingHandler.pipeline_handler_main()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_pro recording_id status_pipelin preprocess_par process_params recording_proc recording_proc task_copy_id_p task_copy_id_p slurm_id \n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +----------+\n", + "16 2 0 1 1 /Volumes/brain None None None None \n", + "17 2 0 1 1 /Volumes/brain None None None None \n", + "18 3 0 1 1 /Volumes/brain None None None None \n", + "19 3 0 1 1 /Volumes/brain None None None None \n", + "21 4 0 2 1 emdia/emdia_gp None None None None \n", + "22 4 0 2 1 emdia/emdia_gp None None None None \n", + "23 4 0 2 1 emdia/emdia_gp None None None None \n", + " (Total: 7)" + ], + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

status_pipeline_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

recording_process_pre_path

\n relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview)\n
\n

recording_process_post_path

\n relative path for processed data recording\n
\n

task_copy_id_pre

\n id for globus transfer task raw file cup->tiger\n
\n

task_copy_id_post

\n id for globus transfer task sorted file tiger->cup\n
\n

slurm_id

\n id for slurm process in tiger\n
162011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
172011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
183011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
193011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
214021emdia/emdia_gps24/20220111/ROI01_z1/NoneNoneNoneNone
224021emdia/emdia_gps24/20220111/ROI02_z1/NoneNoneNoneNone
234021emdia/emdia_gps24/20220111/ROI03_z1/NoneNoneNoneNone
\n \n

Total: 7

\n " + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "recording.RecordingProcess()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_id recording_moda location status_recordi preprocess_par process_params task_copy_id_p inherit_params recording_dire local_director\n", + "+------------+ +------------+ +----------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "2 electrophysiol Bezos1 3 1 1 None 1 ms81/ms81_M004 ms81/ms81_M004\n", + "3 electrophysiol Bezos1 3 1 1 None 1 ms81/ms81_M004 ms81/ms81_M004\n", + "4 imaging Bezos1 3 2 1 None 1 emdia/emdia_gp emdia/emdia_gp\n", + " (Total: 3)" + ], + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

recording_modality

\n modalities for recording (ephys, imaging, video_recording, etc.)\n
\n

location

\n \n
\n

status_recording_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

task_copy_id_pni

\n id for globus transfer task raw file local->cup\n
\n

inherit_params_recording

\n all RecordingProcess from a recording will have same paramSets\n
\n

recording_directory

\n relative directory where the recording will be stored on cup\n
\n

local_directory

\n local directory where the recording is stored on system\n
2electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
3electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
4imagingBezos1321None1emdia/emdia_gps24/20220111emdia/emdia_gps24/20220111
\n \n

Total: 3

\n " + }, + "metadata": {}, + "execution_count": 8 + } + ], + "source": [ + "recording.Recording()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_id *probe probe_director\n", + "+------------+ +-------+ +------------+\n", + "2 0 /Volumes/brain\n", + "2 1 /Volumes/brain\n", + "3 0 /Volumes/brain\n", + "3 1 /Volumes/brain\n", + " (Total: 4)" + ], + "text/html": "\n \n \n \n General information of an ephys session\n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
\n

probe_directory

\n probe specific directory\n
20/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
21/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
30/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
31/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
\n \n

Total: 4

\n " + }, + "metadata": {}, + "execution_count": 9 + } + ], + "source": [ + "ephys.EphysRecordingProbes()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -72,23 +270,146 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 10, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "" + "*recording_pro recording_id probe \n", + "+------------+ +------------+ +-------+\n", + "16 2 0 \n", + "17 2 1 \n", + "18 3 0 \n", + "19 3 1 \n", + " (Total: 4)" ], - "image/svg+xml": "\n\n\n\n\n16\n\n16\n\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n\n16->recording.RecordingProcessStatus\n\n\n\n\n15\n\n15\n\n\n\n15->recording.RecordingProcessStatus\n\n\n\n\n14\n\n14\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\n\n14->recording.Recording\n\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\n\nimaging_rec.ImagingSegmentation\n\n\nimaging_rec.ImagingSegmentation\n\n\n\n\n\nimaging_rec.Scan->imaging_rec.ImagingSegmentation\n\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\n\nrecording.StatusProcessDefinition->16\n\n\n\n\nrecording.StatusProcessDefinition->15\n\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\n\nephys.EphysSorting\n\n\nephys.EphysSorting\n\n\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\n\nrecording.PreprocessingParamSet\n\n\nrecording.PreprocessingParamSet\n\n\n\n\n\nrecording.PreprocessingParamSet->recording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->ephys.EphysSorting\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingSegmentation\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n\nrecording.acquisition.Session->recording.Recording\n\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\n\nrecording.RecordingModality->recording.PreprocessingParamSet\n\n\n\n\nrecording.ProcessingParamSet\n\n\nrecording.ProcessingParamSet\n\n\n\n\n\nrecording.RecordingModality->recording.ProcessingParamSet\n\n\n\n\nephys.EphysRecording->ephys.EphysSorting\n\n\n\n\nrecording.ProcessingParamSet->recording.RecordingProcess\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\n\nrecording.subject.Subject->14\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\n" + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
1620
1721
1830
1931
\n \n

Total: 4

\n " }, "metadata": {}, - "execution_count": 3 + "execution_count": 10 + } + ], + "source": [ + "ephys.EphysProcessing()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_id \n", + "+------------+\n", + "4 \n", + " (Total: 1)" + ], + "text/html": "\n \n \n \n General information of an imaging session\n
\n \n \n \n
\n

recording_id

\n Unique number assigned to each recording\n
4
\n \n

Total: 1

\n " + }, + "metadata": {}, + "execution_count": 11 + } + ], + "source": [ + "imaging_rec.Scan()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_id file_name_base scan_width scan_height acq_time n_depths scan_depth frame_rate inter_fov_lag_ frame_ts_s channels cfg_filename usr_filename fast_z_lag fast_z_flyback line_period scan_frame_per scan_volume_ra flyback_time_p flyto_time_per fov_corner nfovs nframes nframes_good last_good_file motion_correct motion_correct stacks_enabled stack_actuator stack_definiti\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +----------+ +--------+ +------------+ +------------+ +--------+ +--------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +--------+ +-------+ +---------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "4 emdia/emdia_gp 512 1588 2022-01-11 00: 1 =BLOB= 14.3005 0.001 =BLOB= =BLOB= C:\\Users\\admin C:\\Users\\admin 0.0008 0.004 4.14754e-05 0.0699276 14.3005 0.001 0.001 =BLOB= 3 51468 36000 18 1 automated 1 fastZ arbitrary \n", + " (Total: 1)" + ], + "text/html": "\n \n \n \n metainfo about imaging session\n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

file_name_base

\n base name of the file\n
\n

scan_width

\n width of scanning in pixels\n
\n

scan_height

\n height of scanning in pixels\n
\n

acq_time

\n acquisition time\n
\n

n_depths

\n number of depths\n
\n

scan_depths

\n depth values in this scan\n
\n

frame_rate

\n imaging frame rate\n
\n

inter_fov_lag_sec

\n time lag in secs between fovs\n
\n

frame_ts_sec

\n frame timestamps in secs 1xnFrames\n
\n

channels

\n is this the channer number or total number of channels\n
\n

cfg_filename

\n cfg file path\n
\n

usr_filename

\n usr file path\n
\n

fast_z_lag

\n fast z lag\n
\n

fast_z_flyback_time

\n time it takes to fly back to fov\n
\n

line_period

\n scan time per line\n
\n

scan_frame_period

\n \n
\n

scan_volume_rate

\n \n
\n

flyback_time_per_frame

\n \n
\n

flyto_time_per_scan_field

\n \n
\n

fov_corner_points

\n coordinates of the corners of the full 5mm FOV, in microns\n
\n

nfovs

\n number of field of view\n
\n

nframes

\n number of frames in the scan\n
\n

nframes_good

\n number of frames in the scan before acceptable sample bleaching threshold is crossed\n
\n

last_good_file

\n number of the file containing the last good frame because of bleaching\n
\n

motion_correction_enabled

\n \n
\n

motion_correction_mode

\n \n
\n

stacks_enabled

\n \n
\n

stack_actuator

\n \n
\n

stack_definition

\n \n
4emdia/emdia_gps24/20220111/gps24_20220111_mainrecording_00001_00001.tif51215882022-01-11 00:00:001=BLOB=14.30050.001=BLOB==BLOB=C:\\Users\\admin\\Documents\\MATLAB\\MikaToo.cfgC:\\Users\\admin\\Documents\\MATLAB\\Mika.usr0.00080.0044.14754e-050.069927614.30050.0010.001=BLOB=35146836000181automated1fastZarbitrary
\n \n

Total: 1

\n " + }, + "metadata": {}, + "execution_count": 13 + } + ], + "source": [ + "imaging_rec.ScanInfo()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "*recording_pro recording_id fov \n", + "+------------+ +------------+ +-----+\n", + "21 4 1 \n", + "22 4 2 \n", + "23 4 3 \n", + " (Total: 3)" + ], + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

fov

\n number of the field of view in this scan\n
2141
2242
2343
\n \n

Total: 3

\n " + }, + "metadata": {}, + "execution_count": 14 + } + ], + "source": [ + "imaging_rec.ImagingProcessing()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "11" + ] + }, + "metadata": {}, + "execution_count": 11 + } + ], + "source": [ + "a = pd.DataFrame(auto_pipeline.status_pipeline_dict)\n", + "r = a['Value'].max()\n", + "r" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "image/svg+xml": "\n\n\n\n\n20\n\n20\n\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n\n20->recording.acquisition.Session\n\n\n\n\n19\n\n19\n\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n\n19->recording.RecordingProcessStatus\n\n\n\n\n18\n\n18\n\n\n\n18->recording.RecordingProcessStatus\n\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\n\nrecording.StatusProcessDefinition->19\n\n\n\n\nrecording.StatusProcessDefinition->18\n\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\n\nimaging_rec.FieldOfView\n\n\nimaging_rec.FieldOfView\n\n\n\n\n\nimaging_rec.ImagingProcessing\n\n\nimaging_rec.ImagingProcessing\n\n\n\n\n\nimaging_rec.FieldOfView->imaging_rec.ImagingProcessing\n\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\n\nephys.EphysRecordingProbes\n\n\nephys.EphysRecordingProbes\n\n\n\n\n\nephys.EphysProcessing\n\n\nephys.EphysProcessing\n\n\n\n\n\nephys.EphysRecordingProbes->ephys.EphysProcessing\n\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\n\nrecording.lab.Location->20\n\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\n\nimaging_rec.Scan->imaging_rec.FieldOfView\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n\nephys.EphysRecording->ephys.EphysRecordingProbes\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->ephys.EphysProcessing\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingProcessing\n\n\n\n" + }, + "metadata": {}, + "execution_count": 10 } ], "source": [ - "dj.ERD(recording) -1 + dj.ERD(ephys.EphysRecording) + dj.ERD(ephys.EphysSorting) + dj.ERD(imaging_rec.Scan) + dj.ERD(imaging_rec.ImagingSegmentation) \n" + "dj.ERD(recording) -1 + dj.ERD(ephys.EphysRecording) + dj.ERD(ephys.EphysProcessing) + dj.ERD(imaging_rec.Scan) + dj.ERD(imaging_rec.ImagingProcessing) \n" ] }, { From 4e6ceea78ca7719b4f8544af790089b5fbfb83ec Mon Sep 17 00:00:00 2001 From: Alvalunasan Date: Wed, 16 Feb 2022 09:19:05 -0500 Subject: [PATCH 31/34] Process handler working version --- .gitignore | 2 + notebooks/TestAutomaticPipeline.ipynb | 904 +++++------------- .../02-process-imaging-workflow.ipynb | 6 +- requirements.txt | 1 + slurm_files/slurm_test copy.slurm | 0 slurm_files/slurm_test.slurm | 0 .../clusters_paths_and_transfers.py | 101 ++ .../automatic_job/ingest_scaninfo_shell.sh | 14 +- u19_pipeline/automatic_job/params_config.py | 31 +- .../preprocess_params_example.json | 2 + .../preprocess_params_example2.json | 4 + .../preprocess_params_example3.json | 4 + .../process_params_example.json | 22 + .../process_params_example2.json | 22 + .../automatic_job/recording_handler.py | 61 +- .../recording_process_handler.py | 124 ++- u19_pipeline/automatic_job/slurm_creator.py | 104 +- u19_pipeline/imaging_element.py | 18 +- u19_pipeline/imaging_rec.py | 18 +- u19_pipeline/ingest/imaging_element_ingest.py | 1 + u19_pipeline/recording.py | 37 +- u19_pipeline/utility.py | 30 +- u19_pipeline/utils/dj_shortcuts.py | 19 + u19_pipeline/utils/ephys_utils.py | 6 + 24 files changed, 750 insertions(+), 781 deletions(-) create mode 100644 slurm_files/slurm_test copy.slurm create mode 100644 slurm_files/slurm_test.slurm create mode 100644 u19_pipeline/automatic_job/clusters_paths_and_transfers.py create mode 100644 u19_pipeline/automatic_job/process_params_examples/preprocess_params_example.json create mode 100644 u19_pipeline/automatic_job/process_params_examples/preprocess_params_example2.json create mode 100644 u19_pipeline/automatic_job/process_params_examples/preprocess_params_example3.json create mode 100644 u19_pipeline/automatic_job/process_params_examples/process_params_example.json create mode 100644 u19_pipeline/automatic_job/process_params_examples/process_params_example2.json diff --git a/.gitignore b/.gitignore index 7084b61c..afd0997a 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ data/* # svg file *.svg + +slurm_files/slurm_recording_process_id* diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb index 7fa8943c..c532d4ea 100644 --- a/notebooks/TestAutomaticPipeline.ipynb +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -27,16 +27,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "metadata": {}, "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Connecting alvaros@datajoint00.pni.princeton.edu:3306\n" - ] - }, { "output_type": "execute_result", "data": { @@ -45,7 +38,7 @@ ] }, "metadata": {}, - "execution_count": 2 + "execution_count": 6 } ], "source": [ @@ -134,42 +127,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ - "function to apply: modality_preingestion\n", - "imaging scan insert recording_id 4\n", - "recording_modality imaging\n", - "location Bezos1\n", - "status_recording_idx 2\n", - "preprocess_paramset_idx 2\n", - "process_paramset_idx 1\n", - "task_copy_id_pni None\n", - "inherit_params_recording 1\n", - "recording_directory emdia/emdia_gps24/20220111\n", - "local_directory emdia/emdia_gps24/20220111\n", - "query_key {'recording_id': 4}\n", - "Name: 0, dtype: object\n", - "recording_units [{'recording_id': 4, 'fov': 1, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'fov_name': 'mM2_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[-1662.46125877, 269.83897896]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 2, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'fov_name': 'RSP_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1142.3930918 , -99.24825482]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 3, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'fov_name': 'AM_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1555.00006185, 1097.92718087]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}]\n", - "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'status_pipeline_idx': 0}\n", - "recording_process [21]\n", - "this_mod_processing {'recording_id': 4, 'fov': 1, 'recording_process_id': 21}\n", - "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'status_pipeline_idx': 0}\n", - "recording_process [22]\n", - "this_mod_processing {'recording_id': 4, 'fov': 2, 'recording_process_id': 22}\n", - "this_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'status_pipeline_idx': 0}\n", - "recording_process [23]\n", - "this_mod_processing {'recording_id': 4, 'fov': 3, 'recording_process_id': 23}\n", - "key to update {'recording_id': 4}\n", - "recording_key_dict {'recording_id': 4}\n", - "status 3\n", - "update_field None\n", - "update_value None\n", - "update_status_dict {'recording_id': 4, 'status_recording_idx': 3}\n" + "function to apply: modality_preingestion\nimaging scan insert recording_id 4\nrecording_modality imaging\nlocation Bezos1\nstatus_recording_idx 2\npreprocess_paramset_idx 2\nprocess_paramset_idx 1\ntask_copy_id_pni None\ninherit_params_recording 1\nrecording_directory emdia/emdia_gps24/20220111\nlocal_directory emdia/emdia_gps24/20220111\nquery_key {'recording_id': 4}\nName: 0, dtype: object\nrecording_units [{'recording_id': 4, 'fov': 1, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'fov_name': 'mM2_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[-1662.46125877, 269.83897896]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 2, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'fov_name': 'RSP_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1142.3930918 , -99.24825482]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 3, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'fov_name': 'AM_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1555.00006185, 1097.92718087]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}]\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'status_pipeline_idx': 0}\nrecording_process [24]\nthis_mod_processing {'recording_id': 4, 'fov': 1, 'recording_process_id': 24}\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'status_pipeline_idx': 0}\nrecording_process [25]\nthis_mod_processing {'recording_id': 4, 'fov': 2, 'recording_process_id': 25}\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'status_pipeline_idx': 0}\nrecording_process [26]\nthis_mod_processing {'recording_id': 4, 'fov': 3, 'recording_process_id': 26}\nkey to update {'recording_id': 4}\nrecording_key_dict {'recording_id': 4}\nstatus 3\nupdate_field None\nupdate_value None\nupdate_status_dict {'recording_id': 4, 'status_recording_idx': 3}\n" ] } ], @@ -179,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -192,15 +157,15 @@ "17 2 0 1 1 /Volumes/brain None None None None \n", "18 3 0 1 1 /Volumes/brain None None None None \n", "19 3 0 1 1 /Volumes/brain None None None None \n", - "21 4 0 2 1 emdia/emdia_gp None None None None \n", - "22 4 0 2 1 emdia/emdia_gp None None None None \n", - "23 4 0 2 1 emdia/emdia_gp None None None None \n", + "24 4 0 2 1 emdia/emdia_gp None None None None \n", + "25 4 0 2 1 emdia/emdia_gp None None None None \n", + "26 4 0 2 1 emdia/emdia_gp None None None None \n", " (Total: 7)" ], - "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

status_pipeline_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

recording_process_pre_path

\n relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview)\n
\n

recording_process_post_path

\n relative path for processed data recording\n
\n

task_copy_id_pre

\n id for globus transfer task raw file cup->tiger\n
\n

task_copy_id_post

\n id for globus transfer task sorted file tiger->cup\n
\n

slurm_id

\n id for slurm process in tiger\n
162011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
172011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
183011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
193011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
214021emdia/emdia_gps24/20220111/ROI01_z1/NoneNoneNoneNone
224021emdia/emdia_gps24/20220111/ROI02_z1/NoneNoneNoneNone
234021emdia/emdia_gps24/20220111/ROI03_z1/NoneNoneNoneNone
\n \n

Total: 7

\n " + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

status_pipeline_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

recording_process_pre_path

\n relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview)\n
\n

recording_process_post_path

\n relative path for processed data recording\n
\n

task_copy_id_pre

\n id for globus transfer task raw file cup->tiger\n
\n

task_copy_id_post

\n id for globus transfer task sorted file tiger->cup\n
\n

slurm_id

\n id for slurm process in tiger\n
162011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
172011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
183011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
193011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
244021emdia/emdia_gps24/20220111/ROI01_z1/NoneNoneNoneNone
254021emdia/emdia_gps24/20220111/ROI02_z1/NoneNoneNoneNone
264021emdia/emdia_gps24/20220111/ROI03_z1/NoneNoneNoneNone
\n \n

Total: 7

\n " }, "metadata": {}, - "execution_count": 7 + "execution_count": 8 } ], "source": [ @@ -210,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -227,7 +192,7 @@ "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

recording_modality

\n modalities for recording (ephys, imaging, video_recording, etc.)\n
\n

location

\n \n
\n

status_recording_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

task_copy_id_pni

\n id for globus transfer task raw file local->cup\n
\n

inherit_params_recording

\n all RecordingProcess from a recording will have same paramSets\n
\n

recording_directory

\n relative directory where the recording will be stored on cup\n
\n

local_directory

\n local directory where the recording is stored on system\n
2electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
3electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
4imagingBezos1321None1emdia/emdia_gps24/20220111emdia/emdia_gps24/20220111
\n \n

Total: 3

\n " }, "metadata": {}, - "execution_count": 8 + "execution_count": 9 } ], "source": [ @@ -236,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -254,7 +219,7 @@ "text/html": "\n \n \n \n General information of an ephys session\n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
\n

probe_directory

\n probe specific directory\n
20/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
21/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
30/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
31/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
\n \n

Total: 4

\n " }, "metadata": {}, - "execution_count": 9 + "execution_count": 10 } ], "source": [ @@ -270,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -288,7 +253,7 @@ "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
1620
1721
1830
1931
\n \n

Total: 4

\n " }, "metadata": {}, - "execution_count": 10 + "execution_count": 11 } ], "source": [ @@ -297,7 +262,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -312,7 +277,7 @@ "text/html": "\n \n \n \n General information of an imaging session\n
\n \n \n \n
\n

recording_id

\n Unique number assigned to each recording\n
4
\n \n

Total: 1

\n " }, "metadata": {}, - "execution_count": 11 + "execution_count": 12 } ], "source": [ @@ -354,12 +319,12 @@ "text/plain": [ "*recording_pro recording_id fov \n", "+------------+ +------------+ +-----+\n", - "21 4 1 \n", - "22 4 2 \n", - "23 4 3 \n", + "24 4 1 \n", + "25 4 2 \n", + "26 4 3 \n", " (Total: 3)" ], - "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

fov

\n number of the field of view in this scan\n
2141
2242
2343
\n \n

Total: 3

\n " + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

fov

\n number of the field of view in this scan\n
2441
2542
2643
\n \n

Total: 3

\n " }, "metadata": {}, "execution_count": 14 @@ -371,726 +336,337 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "11" - ] + "" + ], + "image/svg+xml": "\n\n%3\n\n\n56\n\n56\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n56->recording.acquisition.Session\n\n\n\n55\n\n55\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n55->recording.RecordingProcessStatus\n\n\n\n54\n\n54\n\n\n54->recording.RecordingProcessStatus\n\n\n\nephys.EphysProcessing\n\n\nephys.EphysProcessing\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nrecording.RecordingProcess->ephys.EphysProcessing\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nephys.EphysRecordingProbes\n\n\nephys.EphysRecordingProbes\n\n\n\n\nephys.EphysRecordingProbes->ephys.EphysProcessing\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->54\n\n\n\nrecording.StatusProcessDefinition->55\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->56\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.EphysRecording->ephys.EphysRecordingProbes\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n" }, "metadata": {}, - "execution_count": 11 + "execution_count": 16 } ], "source": [ - "a = pd.DataFrame(auto_pipeline.status_pipeline_dict)\n", - "r = a['Value'].max()\n", - "r" + "dj.ERD(recording) -1 + dj.ERD(ephys.EphysRecording) + dj.ERD(ephys.EphysProcessing)\n" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "" + "" ], - "image/svg+xml": "\n\n\n\n\n20\n\n20\n\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n\n20->recording.acquisition.Session\n\n\n\n\n19\n\n19\n\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n\n19->recording.RecordingProcessStatus\n\n\n\n\n18\n\n18\n\n\n\n18->recording.RecordingProcessStatus\n\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\n\nrecording.StatusProcessDefinition->19\n\n\n\n\nrecording.StatusProcessDefinition->18\n\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\n\nimaging_rec.FieldOfView\n\n\nimaging_rec.FieldOfView\n\n\n\n\n\nimaging_rec.ImagingProcessing\n\n\nimaging_rec.ImagingProcessing\n\n\n\n\n\nimaging_rec.FieldOfView->imaging_rec.ImagingProcessing\n\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\n\nephys.EphysRecordingProbes\n\n\nephys.EphysRecordingProbes\n\n\n\n\n\nephys.EphysProcessing\n\n\nephys.EphysProcessing\n\n\n\n\n\nephys.EphysRecordingProbes->ephys.EphysProcessing\n\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\n\nrecording.lab.Location->20\n\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\n\nimaging_rec.Scan->imaging_rec.FieldOfView\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n\nephys.EphysRecording->ephys.EphysRecordingProbes\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->ephys.EphysProcessing\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingProcessing\n\n\n\n" + "image/svg+xml": "\n\n%3\n\n\n110\n\n110\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n110->recording.acquisition.Session\n\n\n\n108\n\n108\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n108->recording.RecordingProcessStatus\n\n\n\n109\n\n109\n\n\n109->recording.RecordingProcessStatus\n\n\n\nimaging_rec.ImagingProcessing\n\n\nimaging_rec.ImagingProcessing\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingProcessing\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->108\n\n\n\nrecording.StatusProcessDefinition->109\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nimaging_rec.FieldOfView\n\n\nimaging_rec.FieldOfView\n\n\n\n\nimaging_rec.FieldOfView->imaging_rec.ImagingProcessing\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\nimaging_rec.Scan->imaging_rec.FieldOfView\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->110\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n" }, "metadata": {}, - "execution_count": 10 + "execution_count": 17 } ], "source": [ - "dj.ERD(recording) -1 + dj.ERD(ephys.EphysRecording) + dj.ERD(ephys.EphysProcessing) + dj.ERD(imaging_rec.Scan) + dj.ERD(imaging_rec.ImagingProcessing) \n" + "dj.ERD(recording) -1 + dj.ERD(imaging_rec.Scan) + dj.ERD(imaging_rec.ImagingProcessing) \n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataJoint connection (connected) alvaros@datajoint00.pni.princeton.edu:3306" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], "source": [ - "## Just for now, folders for ephys things in Globus\n", - "#FOR PNI endpoint\n", - "pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b'\n", - "pni_ephys_root_data_dir = '/jukebox/archive/brody/RATTER/PhysData/Raw/'\n", - "pni_ephys_sorted_data_dir = '/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/'\n", - "\n", - "#For tiger endpoint\n", - "tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1'\n", - "tiger_ephys_root_data_dir = '/tigress/alvaros/ephys_raw/'\n", - "tiger_ephys_sorted_data_dir = '/tigress/alvaros/ephys_sorted/'\n" + "s =dj.conn()\n", + "s" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 6, "metadata": {}, + "outputs": [], "source": [ - "### Delete and insert on the AcquisitionSessionsTestAutoPipeline and Status tables" + "s.close()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], - "source": [ - "#acquisition.AcquisitionSessionsTestAutoPipeline.delete()\n", - "#acquisition.AcquisitionSessionsStatus.delete" - ] + "source": [] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 89, "metadata": {}, + "outputs": [], "source": [ - "### Get 5 \"real\" sessions to test and insert them to the TestAutoPipeline" + "dicto2 = [] \n", + "dicto = dict()\n", + "dicto['wala'] = 1\n", + "dicto['walad'] = 2\n", + "\n", + "dictos = dict()\n", + "dictos['wala'] = 3\n", + "dictos['walad'] = 4\n", + "\n", + "\n", + "dicto2.append(dicto)\n", + "dicto2.append(dictos)\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 90, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "\"[{'wala': 1, 'walad': 2}, {'wala': 3, 'walad': 4}]\"" + ] + }, + "metadata": {}, + "execution_count": 90 + } + ], "source": [ - "acq_sessions_df = pd.DataFrame(acquisition.AcquisitionSessions.fetch(limit=5, as_dict=True))\n", - "acq_sessions_df['status_pipeline'] = 0\n", - "acquisition.AcquisitionSessionsTestAutoPipeline.insert(acq_sessions_df, skip_duplicates=True)\n", - "acquisition.AcquisitionSessionsTestAutoPipeline()" + "str(dicto2)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 81, "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['(wala=1 and walad=2)']" + ] + }, + "metadata": {}, + "execution_count": 81 + } + ], "source": [ - "### Check if logged to globus" + "l = [[k + '=' + str(v) for k,v in x.items()] for x in dicto2]\n", + "flat_list = ['('+' and '.join(sublist)+')' for sublist in l]\n", + "flat_list" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 78, "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { "text/plain": [ - "'alvaros@princeton.edu\\n'" + "'(wala=1 and walad=2)'" ] }, - "execution_count": 4, "metadata": {}, - "output_type": "execute_result" + "execution_count": 78 } ], "source": [ - "s = subprocess.run([\"globus\", \"whoami\"], capture_output=True)\n", - "s.stdout.decode('UTF-8')" + "soo = ' or '.join(flat_list)\n", + "soo" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[['wala', 'walad'], ['wala', 'walad']]" + ] + }, + "metadata": {}, + "execution_count": 60 + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 91, "metadata": {}, + "outputs": [], "source": [ - "### Get all \"active sessions\"" + "def get_string_key(key):\n", + "\n", + " str_key = ''\n", + " if isinstance(key, list):\n", + " key_l = [[k + '=' + str(v) for k,v in x.items()] for x in key]\n", + " flat_list = ['('+' and '.join(sublist)+')' for sublist in key_l]\n", + " str_key = ' or '.join(flat_list)\n", + " elif isinstance(key, dict):\n", + " str_key = [k + '=' + str(v) for k,v in key.items()] \n", + " elif isinstance(key,str):\n", + " str_key = key\n", + " \n", + " return str_key" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 92, "metadata": {}, "outputs": [ { + "output_type": "execute_result", "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
sessidstatus_pipelinesession_ratsession_useridsession_rigidacquisition_typeacquisition_raw_rel_pathacquisition_post_rel_pathtask_copy_id_pre_pathtask_copy_id_pos_pathslurm_id_sortingquery_key
05905770T173zhihaol202ephysAnn/miscTestFolder2NoneNoneNone{'sessid': 590577}
\n", - "
" - ], "text/plain": [ - " sessid status_pipeline session_rat session_userid session_rigid \\\n", - "0 590577 0 T173 zhihaol 202 \n", - "\n", - " acquisition_type acquisition_raw_rel_path acquisition_post_rel_path \\\n", - "0 ephys Ann/misc TestFolder2 \n", - "\n", - " task_copy_id_pre_path task_copy_id_pos_path slurm_id_sorting \\\n", - "0 None None None \n", - "\n", - " query_key \n", - "0 {'sessid': 590577} " + "'(wala=1 and walad=2) or (wala=3 and walad=4)'" ] }, - "execution_count": 5, "metadata": {}, - "output_type": "execute_result" + "execution_count": 92 } ], "source": [ - "sessions_active = acquisition.AcquisitionSessionsTestAutoPipeline & 'status_pipeline < 7 and status_pipeline >= 0'\n", - "df_sessions = pd.DataFrame(sessions_active.fetch(as_dict=True))\n", - "\n", - "#Add a column to have a key dictionary as a new column\n", - "key_list = dj_short.get_primary_key_fields(acquisition.AcquisitionSessionsTestAutoPipeline)\n", - "df_sessions['query_key'] = df_sessions.loc[:, key_list].to_dict(orient='records')\n", - "df_sessions" + "get_string_key(dicto2)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "so = imaging_rec.ScanInfo()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "metadata": {}, "outputs": [ { - "name": "stdout", "output_type": "stream", + "name": "stdout", "text": [ - "0 0\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'transfer', '6ce834d6-ff8a-11e6-bad1-22000b9a448b:/jukebox/archive/brody/RATTER/PhysData/Raw/Ann/misc', 'a9df83d2-42f0-11e6-80cf-22000b1701d1:/tigress/alvaros/ephys_raw/Ann/misc', '--recursive', '--format', 'json']\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'task', 'show', 'd6aaa788-36b6-11ec-9e53-3df4ed83d858', '--format', 'json']\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "0 1\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'task', 'show', 'd6aaa788-36b6-11ec-9e53-3df4ed83d858', '--format', 'json']\n", - "0 2\n", - "Name: status_pipeline, dtype: int64\n", - "['ssh', 'alvaros@tigergpu.princeton.edu', 'sbatch', '/tigress/alvaros/slurm_files/slurm_sessid_590577.slurm']\n", + "['/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh', '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts', 'recording_id=4']\n", "aftercommand before comm\n", "aftercommand after comm\n", - "Submitted batch job 7518327\n", "\n", + " < M A T L A B (R) >\n", + " Copyright 1984-2020 The MathWorks, Inc.\n", + " R2020b Update 1 (9.9.0.1495850) 64-bit (glnxa64)\n", + " September 30, 2020\n", "\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", - " State \n", - "---------- \n", - " PENDING \n", - " RUNNING \n", - " RUNNING \n", + " \n", + "To get started, type doc.\n", + "For product information, visit www.mathworks.com.\n", + " \n", "\n", - "PENDING\n", - "COMPLETED\n", - "b'PENDING'\n", - "b'COMPLETED'\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", - " State \n", - "---------- \n", - " RUNNING \n", - " RUNNING \n", - " RUNNING \n", + "s =\n", "\n", - "RUNNING\n", - "COMPLETED\n", - "b'RUNNING'\n", - "b'COMPLETED'\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", - " State \n", - "---------- \n", - " RUNNING \n", - " RUNNING \n", - " RUNNING \n", + " 0\n", "\n", - "RUNNING\n", - "COMPLETED\n", - "b'RUNNING'\n", - "b'COMPLETED'\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "0 3\n", - "Name: status_pipeline, dtype: int64\n", - "['ssh', 'alvaros@tigergpu.princeton.edu', 'sacct', '--job', '7518327', '--format=state']\n", - " State \n", - "---------- \n", - " COMPLETED \n", - " COMPLETED \n", - " COMPLETED \n", + " 0: datajoint00.pni.princeton.edu via TCP/IP Server version 5.5.5-10.2.33-MariaDB (encrypted)\n", + "database connection id: 64946\n", + "imaging_rec.ScanInfo: Nothing to populate\n", "\n", - "COMPLETED\n", - "COMPLETED\n", - "b'COMPLETED'\n", - "b'COMPLETED'\n", - "0 4\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'transfer', 'a9df83d2-42f0-11e6-80cf-22000b1701d1:/tigress/alvaros/ephys_sorted/TestFolder2', '6ce834d6-ff8a-11e6-bad1-22000b9a448b:/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/TestFolder2', '--recursive', '--format', 'json']\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'task', 'show', 'e2a6afab-36b6-11ec-95ea-853490a236f9', '--format', 'json']\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "0 5\n", - "Name: status_pipeline, dtype: int64\n", - "['globus', 'task', 'show', 'e2a6afab-36b6-11ec-95ea-853490a236f9', '--format', 'json']\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n" + "\n" ] - }, + } + ], + "source": [ + "key = dict()\n", + "key['recording_id'] = 4\n", + "\n", + "so.ingest(key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "keyo = dict()\n", + "keyo['subject_fullname'] = 'emdia_teto6s_12'\n", + "keyo['session_date'] = '2021-11-12'\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n", - "0 6\n", - "Name: status_pipeline, dtype: int64\n" - ] + "output_type": "execute_result", + "data": { + "text/plain": [ + "*subject_fulln *session_date *session_numbe scan_directory relative_scan_ acquisition_ty\n", + "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", + "emdia_teto6s_1 2021-11-12 0 /mnt/cup/brain /teto6s_12/202 mesoscope \n", + " (Total: 1)" + ], + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n
\n

subject_fullname

\n username_mouse_nickname\n
\n

session_date

\n date of experiment\n
\n

session_number

\n number\n
\n

scan_directory

\n \n
\n

relative_scan_directory

\n \n
\n

acquisition_type

\n # type of acquisition for this scan\n
emdia_teto6s_122021-11-120/mnt/cup/braininit/RigData/mesoscope/imaging/teto6s_12/20211112/teto6s_12/20211112mesoscope
\n \n

Total: 1

\n " + }, + "metadata": {}, + "execution_count": 10 } ], "source": [ - "import time\n", - "for i in range(30):\n", - " \n", - " aph.pipeline_handler_main()\n", - " time.sleep(3)\n", - " " + "(imaging.Scan & keyo)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ], + "image/svg+xml": "\n\n%3\n\n\n`u19_imaging`.`segmentation_roi_morphology_manual`\n\n`u19_imaging`.`segmentation_roi_morphology_manual`\n\n\n`u19_imaging`.`__sync_imaging_behavior`\n\n`u19_imaging`.`__sync_imaging_behavior`\n\n\nimaging.MotionCorrection.AcrossFiles\n\n\nimaging.MotionCorrection.AcrossFiles\n\n\n\n\nimaging.McParameterSet.Parameter\n\n\nimaging.McParameterSet.Parameter\n\n\n\n\nimaging.SegParameter\n\n\nimaging.SegParameter\n\n\n\n\nimaging.SegParameterSet.Parameter\n\n\nimaging.SegParameterSet.Parameter\n\n\n\n\nimaging.SegParameter->imaging.SegParameterSet.Parameter\n\n\n\nimaging.Segmentation.Roi\n\n\nimaging.Segmentation.Roi\n\n\n\n\nimaging.Segmentation.Roi->`u19_imaging`.`segmentation_roi_morphology_manual`\n\n\n\nimaging.Trace\n\n\nimaging.Trace\n\n\n\n\nimaging.Segmentation.Roi->imaging.Trace\n\n\n\nimaging.Segmentation.RoiMorphologyAuto\n\n\nimaging.Segmentation.RoiMorphologyAuto\n\n\n\n\nimaging.Segmentation.Roi->imaging.Segmentation.RoiMorphologyAuto\n\n\n\nimaging.McMethod\n\n\nimaging.McMethod\n\n\n\n\nimaging.McParameterSet\n\n\nimaging.McParameterSet\n\n\n\n\nimaging.McMethod->imaging.McParameterSet\n\n\n\nimaging.McParameter\n\n\nimaging.McParameter\n\n\n\n\nimaging.McMethod->imaging.McParameter\n\n\n\nimaging.SegParameterSet\n\n\nimaging.SegParameterSet\n\n\n\n\nimaging.SegParameterSet->imaging.SegParameterSet.Parameter\n\n\n\nimaging.Segmentation\n\n\nimaging.Segmentation\n\n\n\n\nimaging.SegParameterSet->imaging.Segmentation\n\n\n\nimaging.FieldOfView\n\n\nimaging.FieldOfView\n\n\n\n\nimaging.FieldOfView->`u19_imaging`.`__sync_imaging_behavior`\n\n\n\nimaging.MotionCorrection\n\n\nimaging.MotionCorrection\n\n\n\n\nimaging.FieldOfView->imaging.MotionCorrection\n\n\n\nimaging.FieldOfView.File\n\n\nimaging.FieldOfView.File\n\n\n\n\nimaging.FieldOfView->imaging.FieldOfView.File\n\n\n\nimaging.McParameterSet->imaging.McParameterSet.Parameter\n\n\n\nimaging.McParameterSet->imaging.MotionCorrection\n\n\n\nimaging.McParameter->imaging.McParameterSet.Parameter\n\n\n\nimaging.Segmentation->imaging.Segmentation.Roi\n\n\n\nimaging.Segmentation.Chunks\n\n\nimaging.Segmentation.Chunks\n\n\n\n\nimaging.Segmentation->imaging.Segmentation.Chunks\n\n\n\nimaging.Segmentation.Background\n\n\nimaging.Segmentation.Background\n\n\n\n\nimaging.Segmentation.Chunks->imaging.Segmentation.Background\n\n\n\nimaging.Scan\n\n\nimaging.Scan\n\n\n\n\nimaging.Scan->imaging.FieldOfView\n\n\n\nimaging.ScanInfo\n\n\nimaging.ScanInfo\n\n\n\n\nimaging.Scan->imaging.ScanInfo\n\n\n\nimaging.MotionCorrection->imaging.MotionCorrection.AcrossFiles\n\n\n\nimaging.MotionCorrection->imaging.Segmentation\n\n\n\nimaging.MotionCorrection.WithinFile\n\n\nimaging.MotionCorrection.WithinFile\n\n\n\n\nimaging.MotionCorrection->imaging.MotionCorrection.WithinFile\n\n\n\nimaging.FieldOfView.File->imaging.MotionCorrection.WithinFile\n\n\n\nimaging.SegmentationMethod\n\n\nimaging.SegmentationMethod\n\n\n\n\nimaging.SegmentationMethod->imaging.SegParameter\n\n\n\nimaging.SegmentationMethod->imaging.SegParameterSet\n\n\n\n" + }, + "metadata": {}, + "execution_count": 11 + } + ], + "source": [ + "dj.Diagram(imaging)" ] }, { @@ -1106,9 +682,9 @@ "hash": "ad2050938946b9745fdc6507c7561603613194cae6ce583d2036ae212150e70f" }, "kernelspec": { - "display_name": "Python 3.7.0 64-bit ('u19_datajoint': conda)", + "display_name": "Python 3.7.11 64-bit", "language": "python", - "name": "python37064bitu19datajointcondaf4e03803b61e40799835623898ea8018" + "name": "python371164bit8ce75340c4e34f47b474ca718361721b" }, "language_info": { "codemirror_mode": { @@ -1120,7 +696,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0-final" + "version": "3.7.11-final" } }, "nbformat": 4, diff --git a/notebooks/imaging_element/02-process-imaging-workflow.ipynb b/notebooks/imaging_element/02-process-imaging-workflow.ipynb index ea293a8c..853a3a86 100644 --- a/notebooks/imaging_element/02-process-imaging-workflow.ipynb +++ b/notebooks/imaging_element/02-process-imaging-workflow.ipynb @@ -155,8 +155,8 @@ } ], "source": [ - "subject = 'emdiamanti_gps8'\n", - "date = '2021-02-08'\n", + "subject = 'jeremyjc_j016'\n", + "date = '2021-11-21'\n", "\n", "key = (imaging.Scan & dict(session_date =date, subject_fullname=subject)).fetch1('KEY')\n", "key" @@ -2705,4 +2705,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 75535af1..e1d549c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ numpy astropy jupyterlab bitstring +element-interface @ git+https://github.com/datajoint/element-interface.git \ No newline at end of file diff --git a/slurm_files/slurm_test copy.slurm b/slurm_files/slurm_test copy.slurm new file mode 100644 index 00000000..e69de29b diff --git a/slurm_files/slurm_test.slurm b/slurm_files/slurm_test.slurm new file mode 100644 index 00000000..e69de29b diff --git a/u19_pipeline/automatic_job/clusters_paths_and_transfers.py b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py new file mode 100644 index 00000000..0369c6a6 --- /dev/null +++ b/u19_pipeline/automatic_job/clusters_paths_and_transfers.py @@ -0,0 +1,101 @@ +import datajoint as dj +import pathlib +import subprocess +import json +import re +from element_interface.utils import dict_to_uuid + +#Functions to transfer files (globus, scp, smbclient) + +#FOR PNI endpoint +pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' +#pni_ephys_sorted_data_dir = '/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/' + +#PNI directories +pni_root_data_dir = dj.config['custom']['root_data_dir'] + +#For tiger endpoint +default_user = 'alvaros' # This will change to our automatic client for globus transfers +tiger_gpu_host = 'tigergpu.princeton.edu' +tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' + + + + +tiger_home_dir = '/scratch/gpfs/BRAINCOGS' +spock_home_dir = '/usr/people/alvaros/BrainCogsProjects/ProcessJobsPipeline' +#Cluster directories +cluster_vars = { + "tiger": { + "home_dir": tiger_home_dir, + "root_data_dir": tiger_home_dir + "/Data", + "sorted_data_dir": tiger_home_dir + "/DataSorted", + "slurm_files_dir": tiger_home_dir + "/slurm_files", + "log_files_dir": tiger_home_dir + "/job_log", + "user": default_user, + "hostname": 'tigergpu.princeton.edu', + }, + "spock": { + "home_dir": spock_home_dir, + "root_data_dir": dj.config['custom']['root_data_dir'], + "slurm_files_dir": spock_home_dir + "/slurm_files", + "log_files_dir": spock_home_dir + "/job_log", + "user": default_user, + "hostname": 'spock.princeton.edu', + } +} + + +def get_cluster_vars(cluster): + + if cluster in cluster_vars: + return cluster_vars[cluster] + else: + raise('Non existing cluster') + + +def scp_file_transfer(source, dest): + + print("scp", source, dest) + p = subprocess.Popen(["scp", "-i", "~/.ssh/id_rsa_alvaros_tiger.pub", source, dest]) + transfer_status = p.wait() + return transfer_status + + +def request_globus_transfer(source, destination): + + globus_command = ["globus", "transfer", source, destination, '--recursive', '--format', 'json'] + #print(globus_command) + #s = subprocess.run(globus_command, capture_output=True) + #transfer_request = json.loads(s.stdout.decode('UTF-8')) + transfer_request = dict() + transfer_request['code'] = 'Accepted' + transfer_request['task_id'] = dict_to_uuid({'test':1}) + return transfer_request + + +def request_globus_transfer_status(id_task): + + globus_command = ["globus", "task", "show", id_task, '--format', 'json'] + #print(globus_command) + #s = subprocess.run(globus_command, capture_output=True) + #transfer_request = json.loads(s.stdout.decode('UTF-8')) + transfer_request = dict() + transfer_request['status'] = 'SUCCEEDED' + return transfer_request + + +def globus_transfer_to_tiger(raw_rel_path): + + source_ep = pni_ep_id + ':' + pni_root_data_dir + raw_rel_path + dest_ep = tiger_ep_dir + ':' + cluster_vars['tiger']['root_data_dir'] + raw_rel_path + transfer_request = request_globus_transfer(source_ep, dest_ep) + return transfer_request + + +def globus_transfer_to_pni(sorted_rel_path): + + source_ep = tiger_ep_dir + ':' + cluster_vars['tiger']['sorted_data_dir'] + sorted_rel_path + dest_ep = pni_ep_id + ':' + pni_root_data_dir + sorted_rel_path + transfer_request = request_globus_transfer(source_ep, dest_ep) + return transfer_request diff --git a/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh b/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh index 78f99e55..e4d41ea4 100755 --- a/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh +++ b/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh @@ -1,8 +1,14 @@ +#!/bin/bash - +# 1st Argument is directory where matlab script is located cd $1 -module load matlab/R2020b +# 2nd Argument is string_key for given recording +key=$2 +matlab_command="populate_ScanInfo_spock('" +matlab_command+=$2 +matlab_command+="');exit;" -key = $2 -matlab -singleCompThread -nodisplay -nosplash -r populate_ScanInfo_spock(key); \ No newline at end of file +# Load module and execute string +module load matlab/R2020b +matlab -singleCompThread -nodisplay -nosplash -r $matlab_command \ No newline at end of file diff --git a/u19_pipeline/automatic_job/params_config.py b/u19_pipeline/automatic_job/params_config.py index 0d4dccc4..7012e16a 100644 --- a/u19_pipeline/automatic_job/params_config.py +++ b/u19_pipeline/automatic_job/params_config.py @@ -10,6 +10,8 @@ 'RecordingFilePattern': np.asarray(['/*g[0-9]/*imec[0-9]']), 'ProcessUnitFilePattern': np.asarray(['/*imec[0-9]/']), 'ProcessUnitDirectoryField': 'probe_directory', + 'ProcessUnitField': 'probe', + 'ProcessingRepository': 'BrainCogsEphysSorters', }, { 'RecordingModality': 'imaging', @@ -19,6 +21,8 @@ 'RecordingFilePattern': np.asarray(['']), 'ProcessUnitFilePattern': np.asarray(['']), 'ProcessUnitDirectoryField': 'fov_directory', + 'ProcessUnitField': 'fov', + 'ProcessingRepository': 'BrainCogsImagingSegmentation', }, { 'RecordingModality': 'video_acquisition', @@ -27,7 +31,9 @@ 'FileExtensions': np.asarray(['.avi', '.mp4']), 'RecordingFilePattern': np.asarray(['']), 'ProcessUnitFilePattern': np.asarray(['']), - 'ProcessUnitDirectoryField': 'video_directory' + 'ProcessUnitDirectoryField': 'video_directory', + 'ProcessUnitField': '', + 'ProcessingRepository': 'None', }, ] @@ -103,7 +109,7 @@ 'Value': 1, 'Key': 'RAW_FILE_TRANSFER_REQUEST', 'Label': 'Raw file transfer requested', - 'UpdateField': 'task_copy_id_pre_path', + 'UpdateField': 'task_copy_id_pre', 'ProcessFunction': 'transfer_request', 'FunctionField': 'recording_process_pre_path', }, @@ -113,7 +119,7 @@ 'Label': 'Raw file transferred to cluster', 'UpdateField': None, 'ProcessFunction': 'transfer_check', - 'FunctionField': 'task_copy_id_pre_path', + 'FunctionField': 'task_copy_id_pre', }, { 'Value': 3, @@ -166,11 +172,28 @@ } ] +all_preprocess_params = { +"process_cluster": [ + "tiger", + "spock"], +"dj_element_processing":[ + "trigger", + "load", +], +"processing_algorithm": [ + "kilosort2", + "suite2p", +] +} + + recording_process_status_list = [[i['Value'], i['Label']] for i in recording_process_status_dict] +recording_process_status_df = pd.DataFrame(recording_process_status_dict) system_process = { 'SUCCESS': 0 } -startup_pipeline_matlab_dir = '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts' \ No newline at end of file +startup_pipeline_matlab_dir = '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts' +ingest_scaninfo_script = '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh' \ No newline at end of file diff --git a/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example.json b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example.json new file mode 100644 index 00000000..94a104cf --- /dev/null +++ b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example.json @@ -0,0 +1,2 @@ +{"cat_gt": 2, + "sorting_algorithm": "kilosort2"} \ No newline at end of file diff --git a/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example2.json b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example2.json new file mode 100644 index 00000000..9f2e6e5f --- /dev/null +++ b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example2.json @@ -0,0 +1,4 @@ +{ +"process_cluster": "spock", +"dj_element_processing": "trigger", +"sorting_algorithm": "suite2p"} \ No newline at end of file diff --git a/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example3.json b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example3.json new file mode 100644 index 00000000..d4c6e33d --- /dev/null +++ b/u19_pipeline/automatic_job/process_params_examples/preprocess_params_example3.json @@ -0,0 +1,4 @@ +{ +"process_cluster": "tiger", +"dj_element_processing": "load", + "sorting_algorithm": "suite2p"} \ No newline at end of file diff --git a/u19_pipeline/automatic_job/process_params_examples/process_params_example.json b/u19_pipeline/automatic_job/process_params_examples/process_params_example.json new file mode 100644 index 00000000..3e4bf17b --- /dev/null +++ b/u19_pipeline/automatic_job/process_params_examples/process_params_example.json @@ -0,0 +1,22 @@ +{"fs": 30000, + "fshigh": 150, + "minfr_goodchannels": 0.1, + "Th": [10, 4], + "lam": 10, + "AUCsplit": 0.9, + "minFR": 0.02, + "momentum": [20, 400], + "sigmaMask": 30, + "ThPr": 8, + "spkTh": -6, + "reorder": 1, + "nskip": 25, + "GPU": 1, + "Nfilt": 1024, + "nfilt_factor": 4, + "ntbuff": 64, + "whiteningRange": 32, + "nSkipCov": 25, + "scaleproc": 200, + "nPCs": 3, + "useRAM": 0} \ No newline at end of file diff --git a/u19_pipeline/automatic_job/process_params_examples/process_params_example2.json b/u19_pipeline/automatic_job/process_params_examples/process_params_example2.json new file mode 100644 index 00000000..2900a09d --- /dev/null +++ b/u19_pipeline/automatic_job/process_params_examples/process_params_example2.json @@ -0,0 +1,22 @@ +{"fs": 35000, + "fshigh": 150, + "minfr_goodchannels": 0.1, + "Th": [10, 4], + "lam": 10, + "AUCsplit": 1, + "minFR": 0.02, + "momentum": [20, 400], + "sigmaMask": 30, + "ThPr": 8, + "spkTh": -6, + "reorder": 1, + "nskip": 25, + "GPU": 1, + "Nfilt": 1024, + "nfilt_factor": 4, + "ntbuff": 64, + "whiteningRange": 32, + "nSkipCov": 25, + "scaleproc": 200, + "nPCs": 3, + "useRAM": 0} \ No newline at end of file diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 4d244784..70215e43 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -9,12 +9,10 @@ import datajoint as dj from u19_pipeline import recording, ephys, imaging_rec import u19_pipeline.utils.dj_shortcuts as dj_short -import u19_pipeline.automatic_job.file_transfers as ft +import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft import u19_pipeline.automatic_job.slurm_creator as slurmlib import u19_pipeline.automatic_job.params_config as config -recording_status_df = pd.DataFrame(config.recording_status_dict) - class RecordingHandler(): default_update_value_dict ={ @@ -40,8 +38,8 @@ def pipeline_handler_main(): #Filter current status info current_status = recording_series['status_recording_idx'] - current_status_series = recording_status_df.loc[recording_status_df['Value'] == current_status, :].squeeze() - next_status_series = recording_status_df.loc[recording_status_df['Value'] == current_status+1, :].squeeze() + current_status_series = config.recording_status_df.loc[config.recording_status_df['Value'] == current_status, :].squeeze() + next_status_series = config.recording_status_df.loc[config.recording_status_df['Value'] == current_status+1, :].squeeze() print('function to apply:', next_status_series['ProcessFunction']) @@ -87,7 +85,7 @@ def local_transfer_request(rec_series, status_series): """ status_update = False - update_value_dict = RecordingHandler.default_update_value_dict + update_value_dict = RecordingHandler.default_update_value_dict.copy() directory_path = status_series['FunctionField'] status_update = True @@ -119,7 +117,7 @@ def local_transfer_check(rec_series, status_series): """ status_update = False - update_value_dict = RecordingHandler.default_update_value_dict + update_value_dict = RecordingHandler.default_update_value_dict.copy() id_task = status_series['FunctionField'] status_update = True @@ -143,12 +141,15 @@ def modality_preingestion(rec_series, status_series): """ status_update = False - update_value_dict = RecordingHandler.default_update_value_dict + update_value_dict = RecordingHandler.default_update_value_dict.copy() - # Get fieldname for processing unit for current recording modality - process_unit_dir_fieldname = \ + # Get fieldname directory for processing unit for current recording modality + process_unit_fieldnames = \ config.recording_modality_df.loc[config.recording_modality_df['RecordingModality'] == rec_series['recording_modality'], - 'ProcessUnitDirectoryField'].squeeze() + ['ProcessUnitDirectoryField', 'ProcessUnitField']].squeeze() + process_unit_dir_fieldname = process_unit_fieldnames['ProcessUnitDirectoryField'] + process_unit_fieldname = process_unit_fieldnames['ProcessUnitField'] + if rec_series['recording_modality'] == 'electrophysiology': @@ -158,21 +159,22 @@ def modality_preingestion(rec_series, status_series): print('this_modality_recording_table', this_modality_recording_table) - # Insert this modality recording and recording "unit" - this_modality_recording_table.populate(rec_series['query_key']) - this_modality_recording_unit_table.populate(rec_series['query_key']) - elif rec_series['recording_modality'] == 'imaging': this_modality_recording_table = imaging_rec.Scan this_modality_recording_unit_table = imaging_rec.FieldOfView this_modality_processing_unit_table = imaging_rec.ImagingProcessing - print('imaging scan insert', rec_series) - this_modality_recording_table.populate(rec_series['query_key']) + #print('imaging scan insert', rec_series) + #this_modality_recording_table.populate(rec_series['query_key']) # Insert this modality recording and recording "unit" - # ------------- Call matlab insertion --------- + #imaging_rec.ScanInfo.populate(rec_series['query_key']) + + + # Insert this modality recording and recording "unit" + this_modality_recording_table.populate(rec_series['query_key']) + this_modality_recording_unit_table.populate(rec_series['query_key']) # In imaging this is to call imaging.ScanInfo populate Script # Get all recording probes ("units") from this recording recording_units = (this_modality_recording_unit_table & rec_series['query_key']).fetch(as_dict=True) @@ -180,24 +182,29 @@ def modality_preingestion(rec_series, status_series): print('recording_units', recording_units) if len(recording_units) > 0: - # Select only primary keys for recording unit + # Select only primary keys for recording unit (EphysRecordingProbes, FieldOfView, etc) rec_units_primary_key_fields = dj_short.get_primary_key_fields(this_modality_recording_unit_table) + + rec_process_table = recording.RecordingProcess() + + # rename default process params (from recording) to process params + rec_series['preprocess_paramset_idx'] = rec_series.pop('def_preprocess_paramset_idx') + rec_series['process_paramset_idx'] = rec_series.pop('def_process_paramset_idx') #Insert recording Process for all ("units") (one by one to get matching recording process id) connection = recording.RecordingProcess.connection with connection.transaction: for rec_unit in recording_units: - #Insert recording process and associated ephysProcessing records for this probe - rec_process_table = recording.RecordingProcess() - rec_process_table.insert_recording_process(rec_series, rec_unit, process_unit_dir_fieldname) + #Insert recording process and associated ephysProcessing records for this probe + rec_process_table.insert_recording_process(rec_series, rec_unit, process_unit_dir_fieldname, process_unit_fieldname) recording_process = recording.RecordingProcess.fetch('recording_process_id', order_by='recording_process_id DESC', limit=1) - # Get recording unit key fields - recording_unit_key = {k: v for k, v in rec_unit.items() if k in rec_units_primary_key_fields} - print('recording_process', recording_process) + # Get recording unit key fields + recording_unit_key = {k: v for k, v in rec_unit.items() if k in rec_units_primary_key_fields} + #Insert modality processing unit this_mod_processing = recording_unit_key.copy() this_mod_processing['recording_process_id'] = recording_process[0] @@ -218,8 +225,8 @@ def get_active_recordings(): df_recordings (pd.DataFrame): all recordings that are going to be processed in the pipeline ''' - status_query = 'status_recording_idx > ' + str(recording_status_df['Value'].min()) - status_query += ' and status_recording_idx < ' + str(recording_status_df['Value'].max()) + status_query = 'status_recording_idx > ' + str(config.recording_status_df['Value'].min()) + status_query += ' and status_recording_idx < ' + str(config.recording_status_df['Value'].max()) recordings_active = recording.Recording & status_query df_recordings = pd.DataFrame(recordings_active.fetch(as_dict=True)) diff --git a/u19_pipeline/automatic_job/recording_process_handler.py b/u19_pipeline/automatic_job/recording_process_handler.py index 82052c66..cdad6f58 100644 --- a/u19_pipeline/automatic_job/recording_process_handler.py +++ b/u19_pipeline/automatic_job/recording_process_handler.py @@ -4,13 +4,15 @@ import json import time import re +import traceback import pandas as pd import datajoint as dj import u19_pipeline.recording as recording import u19_pipeline.utils.dj_shortcuts as dj_short -import u19_pipeline.automatic_job.file_transfers as ft +import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft import u19_pipeline.automatic_job.slurm_creator as slurmlib import u19_pipeline.automatic_job.params_config as config +from u19_pipeline.utility import create_str_from_dict recording_process_status_df = pd.DataFrame(config.recording_process_status_dict) @@ -37,38 +39,53 @@ def pipeline_handler_main(): #Filter current process job rec_process_series = df_all_process_job.loc[i, :] - preprocess_paramset = recording.PreprocessParamSet.get_preprocess_params(rec_process_series['preprocess_paramset_idx']) - process_paramset = recording.ProcessParamSet.get_process_params(rec_process_series['process_paramset_idx']) + preprocess_paramset = recording.PreprocessParamSet().get_preprocess_params({'preprocess_paramset_idx':rec_process_series['preprocess_paramset_idx']}) + process_paramset = recording.ProcessParamSet().get_process_params({'process_paramset_idx': rec_process_series['process_paramset_idx']}) + + #ALS, correct preprocess params if OLD or outdated #Get params inside the recording process series rec_process_series['preprocess_paramset'] = preprocess_paramset rec_process_series['process_paramset'] = process_paramset #Filter current status info - curent_status = rec_process_series['status_pipeline_idx'] - current_status_series = df_status_pipeline.loc[df_status_pipeline['Value'] == current_status, :].squeeze() - next_status_series = df_status_pipeline.loc[df_status_pipeline['Value'] == current_status+1, :].squeeze() + current_status = rec_process_series['status_pipeline_idx'] + current_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status, :].squeeze() + next_status_series = config.recording_process_status_df.loc[config.recording_process_status_df['Value'] == current_status+1, :].squeeze() # Get processing function - function_status_process = next_status_series['ProcessFunction'] + function_status_process = getattr(RecProcessHandler, next_status_series['ProcessFunction']) #Trigger process, if success update recording process record - success_process, update_dict = function_status_process(rec_process_series, next_status_series, preprocess_paramset, process_paramset) - - if success_process: - #Get dictionary of record process - key = rec_process_series['query_key'] - #Get values to update - next_status = next_status_series['Value'] - value_update = update_dict['value_update'] - field_update = next_status_series['UpdateField'] - - RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) + try: + success_process, update_dict = function_status_process(rec_process_series, next_status_series) + + print('update_dict', update_dict) + + if success_process: + #Get dictionary of record process + key = rec_process_series['query_key'] + #Get values to update + next_status = next_status_series['Value'] + value_update = update_dict['value_update'] + field_update = next_status_series['UpdateField'] + + print('key to update', key) + print('old status', current_status, 'new status', next_status) + print('value_update', value_update, 'field_update', field_update) + print('function executed:', next_status_series['ProcessFunction']) + + RecProcessHandler.update_status_pipeline(key, next_status, field_update, value_update) + except Exception as err: + raise(err) + print(traceback.format_exc()) + ## Send notification error, update recording to error time.sleep(2) + @staticmethod - def transfer_request(rec_series, status_series, preprocess_paramset, process_paramset) + def transfer_request(rec_series, status_series): """ Request a transfer from PNI to Tiger Cluster Input: @@ -84,18 +101,27 @@ def transfer_request(rec_series, status_series, preprocess_paramset, process_par """ status_update = False - update_value_dict = RecProcessHandler.default_update_value_dict + update_value_dict = RecProcessHandler.default_update_value_dict.copy() directory_path = status_series['FunctionField'] - if status_series['Key'] is 'RAW_FILE_TRANSFER_REQUEST': - transfer_request = globus_transfer_raw_ephys_pni_tiger(directory_path) - elif status_series['Key'] is 'PROC_FILE_TRANSFER_REQUEST': - #ALS, which recording directory for processed file - transfer_request = globus_transfer_sorted_ephys_tiger_pni(directory_path) + print('process_cluter:', rec_series['preprocess_paramset']['process_cluster']) + + # If tiger, we trigger globus transfer + if rec_series['preprocess_paramset']['process_cluster'] == "tiger": + print('si fue tiger') + if status_series['Key'] is 'RAW_FILE_TRANSFER_REQUEST': + transfer_request = ft.globus_transfer_to_tiger(directory_path) + elif status_series['Key'] is 'PROC_FILE_TRANSFER_REQUEST': + #ALS, which recording directory for processed file + transfer_request = ft.globus_transfer_to_pni(directory_path) - if transfer_request['code'] == 'Accepted': + if transfer_request['code'] == 'Accepted': + status_update = True + update_value_dict['value_update'] = transfer_request['task_id'] + # If not tiger let's go to next status + else: + print('si fue spock') status_update = True - update_value_dict['value_update'] = transfer_request['task_id'] return (status_update, update_value_dict) @@ -116,14 +142,15 @@ def transfer_check(rec_series, status_series): """ status_update = False - value_update = RecProcessHandler.default_update_value_dict + update_value_dict = RecProcessHandler.default_update_value_dict.copy() id_task = status_series['FunctionField'] - transfer_request = request_globus_transfer_status(str(id_task)) - if transfer_request['status'] == 'SUCCEEDED': - status_update = True + transfer_request = ft.request_globus_transfer_status(str(id_task)) + if transfer_request['status'] == 'SUCCEEDED': + status_update = True return (status_update, update_value_dict) + @staticmethod def slurm_job_queue(rec_series, status_series): @@ -142,24 +169,26 @@ def slurm_job_queue(rec_series, status_series): """ status_update = False - update_value_dict = RecProcessHandler.default_update_value_dict + update_value_dict = RecProcessHandler.default_update_value_dict.copy() - slurm_text = slurmlib.generate_slurm_file(rec_series) - slurm_file_name = 'slurm_' + create_str_from_dict(key) + '.slurm' - slurm_file_path = str(pathlib.Path("slurm_files",slurm_file_name)) + status = slurmlib.generate_slurm_file(rec_series) + #slurm_file_name = 'slurm_' + create_str_from_dict(rec_series['query_key']) + '.slurm' + #slurm_file_path = str(pathlib.Path("slurm_files",slurm_file_name)) - write_slurm_file(slurm_file_path, slurm_text) + #slurmlib.write_slurm_file(slurm_file_path, slurm_text) - tiger_slurm_user = default_user+'@'+tiger_gpu_host - tiger_slurm_location = tiger_slurm_user+':'+tiger_slurm_files_dir+slurm_file_name - transfer_request = scp_file_transfer(slurm_file_path, tiger_slurm_location) + #tiger_slurm_user = default_user+'@'+tiger_gpu_host + #tiger_slurm_location = tiger_slurm_user+':'+tiger_slurm_files_dir+slurm_file_name + #transfer_request = scp_file_transfer(slurm_file_path, tiger_slurm_location) - if transfer_request == config.system_process['SUCCESS']: - slurm_queue_status, slurm_jobid = queue_slurm_file(tiger_slurm_user, tiger_slurm_files_dir+slurm_file_name) + if status == config.system_process['SUCCESS']: + #slurm_queue_status, slurm_jobid = slurmlib.queue_slurm_file(tiger_slurm_user, tiger_slurm_files_dir+slurm_file_name) + slurm_jobid = 0 - if transfer_request == config.system_process['SUCCESS']: + if status == config.system_process['SUCCESS']: status_update = True - update_status_pipeline(key, status_dict['JOB_QUEUE']['Task_Field'], slurm_jobid, status_dict['JOB_QUEUE']['Value']) + update_value_dict['value_update'] = slurm_jobid + #update_status_pipeline(key, status_dict['JOB_QUEUE']['Task_Field'], slurm_jobid, status_dict['JOB_QUEUE']['Value']) return (status_update, update_value_dict) @@ -180,10 +209,10 @@ def slurm_job_check(rec_series, status_series): """ status_update = False - value_update = RecProcessHandler.default_update_value_dict + update_value_dict = RecProcessHandler.default_update_value_dict.copy() slurm_jobid = status_series['FunctionField'] - ssh_user = tiger_slurm_user = default_user+'@'+tiger_gpu_host + ssh_user = default_user+'@'+tiger_gpu_host job_status = check_slurm_job(ssh_user, slurm_jobid) print('job status', job_status) @@ -217,9 +246,6 @@ def get_active_process_jobs(): return df_process_jobs - -RecProcessHandler.get_preprocess_params(rec_process_series['query_key']) - @staticmethod def update_status_pipeline(recording_process_key_dict, status, update_field=None, update_value=None): """ @@ -236,7 +262,7 @@ def update_status_pipeline(recording_process_key_dict, status, update_field=None update_task_id_dict[update_field] = update_value recording.RecordingProcess.update1(update_task_id_dict) - update_status_dict = recording_process_dict.copy() + update_status_dict = recording_process_key_dict.copy() update_status_dict['status_pipeline_idx'] = status recording.RecordingProcess.update1(update_status_dict) diff --git a/u19_pipeline/automatic_job/slurm_creator.py b/u19_pipeline/automatic_job/slurm_creator.py index 531772e9..27034ccd 100644 --- a/u19_pipeline/automatic_job/slurm_creator.py +++ b/u19_pipeline/automatic_job/slurm_creator.py @@ -3,8 +3,11 @@ #import os #import pathlib import subprocess +import pathlib import json import re +import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft +from u19_pipeline.utility import create_str_from_dict, is_this_spock # Functions to create slurm jobs @@ -30,25 +33,68 @@ default_matlab_ver = 'R2020b' def generate_slurm_file(record_process_series): + ''' + Generate and send slurm file to be queued in processing cluster + ''' + #get preprocess params (some of these will change slurm creation) + preprocess_params = record_process_series['preprocess_paramset'] - rel_path = record_process_series['acquisition_raw_rel_path'] + #relative Path where data will be located + rel_path = record_process_series['recording_process_pre_path'] + + #get processing_id string key = record_process_series['query_key'] - + str_key = create_str_from_dict(key) + + # Start with default values slurm_dict = slurm_dict_default.copy() - slurm_dict['job-name'] = default_preprocessing_tool + "_" + create_str_from_dict(key) - slurm_dict['output'] = str(pathlib.Path(tiger_log_files_dir,default_preprocessing_tool + create_str_from_dict(key) + '.log')) + print(preprocess_params['sorting_algorithm']) + print(str_key) + print(type(str_key)) + print(type(preprocess_params['sorting_algorithm'])) + slurm_dict['job-name'] = preprocess_params['sorting_algorithm'] + "_" + str_key + + #Get all associated directories given the selected processing cluster + cluster_vars = ft.get_cluster_vars(preprocess_params['process_cluster']) + slurm_dict['output'] = str(pathlib.Path(cluster_vars['log_files_dir'],str_key + '.log')) + + if preprocess_params['process_cluster'] == 'spock' and preprocess_params["dj_element_processing"] == "trigger": + slurm_text = generate_slurm_dj_trigger(slurm_dict) + else: + slurm_text = generate_slurm_kilosort_text(slurm_dict, default_matlab_ver, ft.default_user, rel_path) + slurm_file_name = 'slurm_' + str_key + '.slurm' + slurm_file_local_path = str(pathlib.Path("slurm_files",slurm_file_name)) + + write_slurm_file(slurm_file_local_path, slurm_text) + + if preprocess_params['process_cluster'] == 'spock' and is_this_spock(): + status = copy_spock_local_slurm_file(slurm_file_local_path, slurm_file_name, cluster_vars) + else: + status = transfer_slurm_file(slurm_file_local_path, slurm_file_name, cluster_vars) + + return status - slurm_text = generate_slurm_kilosort_text(slurm_dict, default_matlab_ver, default_user, raw_rel_path) - slurm_file_name = 'slurm_' + create_str_from_dict(key) + '.slurm' - slurm_file_path = str(pathlib.Path("slurm_files",slurm_file_name)) +def transfer_slurm_file(slurm_file_local_path, slurm_file_name, cluster_vars): + ''' + Create scp command from cluster directories and local slurm file + ''' - write_slurm_file(slurm_file_path, slurm_text) - - tiger_slurm_user = default_user+'@'+tiger_gpu_host - tiger_slurm_location = tiger_slurm_user+':'+tiger_slurm_files_dir+slurm_file_name - transfer_request = scp_file_transfer(slurm_file_path, tiger_slurm_location) + user_host = cluster_vars['user']+'@'+cluster_vars['hostname'] + slurm_destination = user_host+':'+str(pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name)) + status = ft.scp_file_transfer(slurm_file_local_path, slurm_destination) + + return status + +def copy_spock_local_slurm_file(slurm_file_local_path, slurm_file_name, cluster_vars): + ''' + Copy local spock slurm file + ''' + slurm_destination = str(pathlib.Path(cluster_vars['slurm_files_dir'], slurm_file_name)) + p = subprocess.Popen(["cp", slurm_file_local_path, slurm_destination]) + status = p.wait() + return status def create_slurm_params_file(slurm_dict): @@ -84,6 +130,40 @@ def generate_slurm_kilosort_text(slurm_dict, matlab_ver, user_run, raw_file_path return slurm_text +def generate_slurm_text(slurm_dict, matlab_ver, user_run, raw_file_path): + + slurm_text = '#!/bin/bash\n' + slurm_text += create_slurm_params_file(slurm_dict) + slurm_text += 'module load matlab/' + matlab_ver + '\n' + slurm_text += 'cd /tigress/' + user_run + '\n' + slurm_text += 'matlab -singleCompThread -nodisplay -nosplash -r "pause(1); ' + "disp('aqui la chides'); exit" + '"' + #slurm_text += 'matlab -singleCompThread -nodisplay -nosplash -r "addpath(''/tigress/' + user_run + "/run_kilosort/spikesorters/'); " + #slurm_text += "run_ks2('/tigress/" + user_run + "/ephys_raw" + raw_file_path + "','/tigress/" + user_run + "/run_kilosort/tmp/'); exit" + '"' + + return slurm_text + + +def generate_slurm_dj_trigger(slurm_dict): + + slurm_text = '#!/bin/bash\n' + slurm_text += create_slurm_params_file(slurm_dict) + slurm_text += ''' + echo "SLURM_JOB_ID: ${SLURM_JOB_ID}" + echo "SLURM_SUBMIT_DIR: ${SLURM_SUBMIT_DIR}" + echo "SESSID: ${sessid}" + + module load anacondapy/2021.11 + + conda activate U19-pipeline_python + + cd /usr/people/kg7524/U19-pipeline_python + + python /usr/people/kg7524/slurm/test.py + ''' + + return slurm_text + + def queue_slurm_file(ssh_user, slurm_file): id_slurm_job = -1 diff --git a/u19_pipeline/imaging_element.py b/u19_pipeline/imaging_element.py index b950743c..97f5e32e 100644 --- a/u19_pipeline/imaging_element.py +++ b/u19_pipeline/imaging_element.py @@ -63,14 +63,17 @@ def get_scan_image_files(scan_key): #Replace scan_id with fov, we are going to search files by fov if 'scan_id' in fov_key: fov_key['fov'] = fov_key.pop('scan_id') - scan_filepaths_ori = list((imaging.FieldOfView.File * imaging.FieldOfView & fov_key).proj( - full_path='concat(relative_fov_directory, fov_filename)').fetch('full_path')) + scan_filepaths_ori = (imaging.FieldOfView.File * imaging.FieldOfView & fov_key).fetch('relative_fov_directory', 'fov_filename', as_dict=True) + + scan_filepaths_conc = list() + for i in range(len(scan_filepaths_ori)): + scan_filepaths_conc.append((pathlib.Path(scan_filepaths_ori[i]['relative_fov_directory']) / scan_filepaths_ori[i]['fov_filename']).as_posix()) # if rel paths start with / remove it for Pathlib library - scan_filepaths_ori = [x[1:] if x[0] == '/' else x for x in scan_filepaths_ori] - + scan_filepaths_conc = [x[1:] if x[0] == '/' else x for x in scan_filepaths_conc] + data_dir = get_imaging_root_data_dir() - tiff_filepaths = [(data_dir / x).as_posix() for x in scan_filepaths_ori] + tiff_filepaths = [(pathlib.Path(data_dir) / x).as_posix() for x in scan_filepaths_conc] if tiff_filepaths: return tiff_filepaths @@ -90,9 +93,12 @@ def get_suite2p_dir(processing_task_key): sess_dir = data_dir / bucket_scan_dir / 'suite2p' relative_suite2p_dir = (pathlib.Path(bucket_scan_dir) / 'suite2p').as_posix() + print(bucket_scan_dir) + + # Check if suite2p dir exists if not sess_dir.exists(): - raise FileNotFoundError(f'Session directory not found ({scan_dir})') + raise FileNotFoundError(f'Session directory not found ({sess_dir})') # Check if ops.npy is inside suite2pdir suite2p_dirs = set([fp.parent.parent for fp in sess_dir.rglob('*ops.npy')]) diff --git a/u19_pipeline/imaging_rec.py b/u19_pipeline/imaging_rec.py index 7c41c377..d4327f31 100644 --- a/u19_pipeline/imaging_rec.py +++ b/u19_pipeline/imaging_rec.py @@ -1,6 +1,9 @@ import datajoint as dj from u19_pipeline import acquisition, subject, recording +import u19_pipeline.automatic_job.params_config as config +import u19_pipeline.utils.dj_shortcuts as dj_short +import subprocess schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'imaging_rec') @@ -79,9 +82,22 @@ class FieldOfView(dj.Imported): fov_rotation_degrees : float # rotation of the FOV with respect to cardinal axes in degrees. One for each FOV in scan fov_pixel_resolution_xy : blob # number of pixels for rows and columns of the FOV. One for each FOV in scan fov_discrete_plane_mode : tinyint # true if FOV is only defined (acquired) at a single specifed depth in the volume. One for each FOV in scan should this be boolean? - power_percent : float # percentage of power used for this field of view + power_percent : float # percentage of power used for this field of view """ + def populate(self, key): + + str_key = dj_short.get_string_key(key) + command = [config.ingest_scaninfo_script, config.startup_pipeline_matlab_dir, str_key] + print(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.wait() + print('aftercommand before comm') + stdout, stderr = p.communicate() + print('aftercommand after comm') + print(stdout.decode('UTF-8')) + print(stderr.decode('UTF-8')) + class File(dj.Part): definition = """ # list of files per FOV diff --git a/u19_pipeline/ingest/imaging_element_ingest.py b/u19_pipeline/ingest/imaging_element_ingest.py index ded9c2df..b1b6b0d0 100644 --- a/u19_pipeline/ingest/imaging_element_ingest.py +++ b/u19_pipeline/ingest/imaging_element_ingest.py @@ -34,6 +34,7 @@ def process_scan(scan_key): loaded_scan = scanreader.read_scan(scan_filepaths) header = parse_scanimage_header(loaded_scan) scanner = header['SI_imagingSystem'].strip('\'') + print(scanner) except Exception as e: print(f'ScanImage loading error: {scan_filepaths}\n{str(e)}') return diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 58604867..300fdcfc 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -2,6 +2,8 @@ import numpy as np from u19_pipeline import lab, task, subject, acquisition import u19_pipeline.automatic_job.params_config as config +from element_interface.utils import dict_to_uuid +from u19_pipeline.utility import numpy_array_to_dict schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'recording') @@ -16,6 +18,7 @@ class RecordingModality(dj.Lookup): recording_file_pattern: blob # directory pattern to find recordings in path process_unit_file_pattern: blob # process "unit" pattern to find in path process_unit_dir_fieldname: varchar(64) # FieldName that stores process unit directory for specific modality + process_unit_fieldname: varchar(32) # FieldName that stores process unit for specific modality (fov, probe, etc) """ contents = config.recording_modality_list @@ -63,15 +66,21 @@ def insert_new_params(cls, recording_modality: str, preprocess_paramset_idx: int else: cls.insert1(paramset_dict) - @classmethod - def get_preprocess_params(preprocess_param_idx): + + def get_preprocess_params(self, preprocess_param_idx): ''' Get process params for current recording process Return: preprocess_paramset (dict): preprocess params associated with recording process ''' - preprocess_paramset = (recording.ProcessParamSet & preprocess_param_idx).fetch1('preprocess_paramset') + preprocess_paramset = (self & preprocess_param_idx).fetch1('preprocess_paramset') + + #If stored in MATLAB this is a numpy array to be converted to dictionary + if isinstance(preprocess_paramset, np.ndarray): + preprocess_paramset = numpy_array_to_dict(preprocess_paramset) + + return preprocess_paramset @@ -109,15 +118,20 @@ def insert_new_params(cls, recording_modality: str, paramset_idx: int, else: cls.insert1(paramset_dict) - @staticmethod - def get_process_params(process_param_idx): + + def get_process_params(self, process_param_idx): ''' Get process params for current recording process Return: process_paramset (dict): process params associated with recording process ''' - process_paramset = (recording.ProcessParamSet & process_param_idx).fetch1('process_paramset') + process_paramset = (self & process_param_idx).fetch1('process_paramset') + + #If stored in MATLAB this is a numpy array to be converted to dictionary + if isinstance(process_paramset, np.ndarray): + process_paramset = numpy_array_to_dict(process_paramset) + return process_paramset @@ -129,8 +143,8 @@ class Recording(dj.Manual): -> RecordingModality -> lab.Location -> StatusRecordingDefinition # current status for recording in the pipeline - -> PreprocessParamSet # reference to params to preprocess recording (possible to inherit to recordigprocess) - -> ProcessParamSet # reference to params to process recording (possible to inherit to recordigprocess) + -> PreprocessParamSet.proj(def_preprocess_paramset_idx='preprocess_paramset_idx') # reference to params to default preprocess recording (possible to inherit to recordigprocess) + -> ProcessParamSet.proj(def_process_paramset_idx='process_paramset_idx') # reference to params to default process recording (possible to inherit to recordigprocess) task_copy_id_pni=null: UUID # id for globus transfer task raw file local->cup inherit_params_recording=1: boolean # all RecordingProcess from a recording will have same paramSets recording_directory: varchar(255) # relative directory where the recording will be stored on cup @@ -172,14 +186,15 @@ class RecordingProcess(dj.Manual): -> StatusProcessDefinition # current status in the pipeline -> PreprocessParamSet # reference to params to preprocess recording -> ProcessParamSet # reference to params to process recording + fragment_number: TINYINT(1) # fov# or probe#, etc. reference from the corresponding modality recording_process_pre_path=null: VARCHAR(200) # relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview) recording_process_post_path=null: VARCHAR(200) # relative path for processed data recording task_copy_id_pre=null: UUID # id for globus transfer task raw file cup->tiger - task_copy_id_post=null: UUID # id for globus transfer task sorted file tiger->cup + task_copy_id_post=null: UUID # id for globus transfer task sorted file tiger->cup slurm_id=null: VARCHAR(16) # id for slurm process in tiger """ - def insert_recording_process(self, recording_key, rec_unit, unit_directory_fieldname): + def insert_recording_process(self, recording_key, rec_unit, unit_directory_fieldname, unit_fieldname): ''' #Insert RecordingProcess(es) from recording. # For each processing "unit" of a recording add a new recordingProcess (imaging ->field of view, electrophysiology->probe) @@ -187,6 +202,7 @@ def insert_recording_process(self, recording_key, rec_unit, unit_directory_field recording_key (dict) = Dictionary with recording record rec_unit (dict) = Dictionary of recording "unit" to be processed unit_directory_fieldname (str) = Unit directory fieldname to be read (ephys-> probe_directory, imaging->fov_directory) + unit_fieldname (str) = Unit fieldname to be read (ephys-> probe_, imaging->fov) ''' # Get directory fieldname for specific modality (probe_directory, fov_directory, etc.) @@ -196,6 +212,7 @@ def insert_recording_process(self, recording_key, rec_unit, unit_directory_field this_recprocess_key['recording_id'] = recording_key['recording_id'] this_recprocess_key['preprocess_paramset_idx'] = recording_key['preprocess_paramset_idx'] this_recprocess_key['process_paramset_idx'] = recording_key['process_paramset_idx'] + this_recprocess_key['fragment_number'] = rec_unit[unit_fieldname] this_recprocess_key['recording_process_pre_path'] = rec_unit[unit_directory_fieldname] this_recprocess_key['status_pipeline_idx'] = 0 diff --git a/u19_pipeline/utility.py b/u19_pipeline/utility.py index 64f9242f..ecdaffd2 100644 --- a/u19_pipeline/utility.py +++ b/u19_pipeline/utility.py @@ -3,6 +3,7 @@ import sys import os import pandas as pd +from pandas.api.types import is_numeric_dtype import numpy as np from scipy.optimize import curve_fit from astropy.stats import binom_conf_interval @@ -280,4 +281,31 @@ def create_str_from_dict(key_dict): slurm_file_name = '' for i in key_dict.keys(): slurm_file_name += str(i) + '_' + str(key_dict[i]) - return slurm_file_name \ No newline at end of file + return slurm_file_name + + +def numpy_array_to_dict(np_array, as_int=True): + ''' + Transform a numpy array to dictionary: + (numpy array are stored when saving Blobs in MATLAB Datajoint, normally a dictionary will be the fit) + ''' + + # Transform numpy array to DF + out_dict = pd.DataFrame(np_array.flatten()) + + # Flatten each column and get the "real value of it" + out_dict = out_dict.applymap(lambda x: x.flatten()) + columns = out_dict.columns.copy() + + out_dict = out_dict.applymap(lambda x: x[0]).squeeze() + + #Transform numeric columns to int (normally for params) + if as_int: + for i in columns: + + if (is_numeric_dtype(out_dict[i].dtype)): + out_dict[i] = out_dict[i].astype('int') + + out_dict = out_dict.to_dict() + + return out_dict \ No newline at end of file diff --git a/u19_pipeline/utils/dj_shortcuts.py b/u19_pipeline/utils/dj_shortcuts.py index 61ae5081..0e87490f 100644 --- a/u19_pipeline/utils/dj_shortcuts.py +++ b/u19_pipeline/utils/dj_shortcuts.py @@ -49,3 +49,22 @@ def smart_dj_join(t1, t2): t = t1 * t2 return t + + +def get_string_key(key): + """ + Translate list or dict key to string + """ + + str_key = '' + if isinstance(key, list): + str_key = [[k + '=' + str(v) for k,v in x.items()] for x in key] + str_key = ['('+' and '.join(sublist)+')' for sublist in str_key] + str_key = ' or '.join(str_key) + elif isinstance(key, dict): + str_key = [k + '=' + str(v) for k,v in key.items()] + str_key = ' and '.join(str_key) + elif isinstance(key,str): + str_key = key + + return str_key \ No newline at end of file diff --git a/u19_pipeline/utils/ephys_utils.py b/u19_pipeline/utils/ephys_utils.py index 482c5ce3..d4413501 100644 --- a/u19_pipeline/utils/ephys_utils.py +++ b/u19_pipeline/utils/ephys_utils.py @@ -173,7 +173,13 @@ def assert_iteration_samples_count(iteration_sample_idx_output, behavior_time_ve if iteration_sample_idx_output.shape[0] != (behavior_time_vector.shape[0]): status = False return status + + count = 0 for idx_trial, iter_trials in enumerate(iteration_sample_idx_output): + count += 1 + print(count) + print(iter_trials.shape[0]) + print(behavior_time_vector[idx_trial].shape[0]) #For each trial iteration # should be equal to the behavioral file iterations if iter_trials.shape[0] != behavior_time_vector[idx_trial].shape[0]: status = False From 51a7c5eba4122aea8428ed2981edc9096112fcac Mon Sep 17 00:00:00 2001 From: Alvalunasan Date: Wed, 16 Feb 2022 09:21:31 -0500 Subject: [PATCH 32/34] Deleted old file transfers file --- u19_pipeline/automatic_job/file_transfers.py | 69 -------------------- 1 file changed, 69 deletions(-) delete mode 100644 u19_pipeline/automatic_job/file_transfers.py diff --git a/u19_pipeline/automatic_job/file_transfers.py b/u19_pipeline/automatic_job/file_transfers.py deleted file mode 100644 index 0cc50901..00000000 --- a/u19_pipeline/automatic_job/file_transfers.py +++ /dev/null @@ -1,69 +0,0 @@ - -import pathlib -import subprocess -import json -import re - - -#Functions to transfer files (globus, scp, smbclient) - - -#FOR PNI endpoint -pni_ep_id = '6ce834d6-ff8a-11e6-bad1-22000b9a448b' -#pni_ephys_sorted_data_dir = '/mnt/cup/labs/brody/RATTER/PhysData/Test_ephys_pipeline_NP_sorted/' - -#PNI directories -pni_root_data_dir = '/braininit/Data/' - -#For tiger endpoint -default_user = 'alvaros' # This will change to our automatic client for globus transfers -tiger_gpu_host = 'tigergpu.princeton.edu' -tiger_ep_dir = 'a9df83d2-42f0-11e6-80cf-22000b1701d1' - -# Tiger directories -tiger_home_dir = '/tigress/alvaros' # This will be changed to /scratch/gpfs/BRAINCOGS when permissions are granted -tiger_raw_root_data_dir = tiger_home_dir + '/DataRaw' -tiger_sorted_root_data_dir = tiger_home_dir + '/DataProcessed' -tiger_slurm_files_dir = tiger_home_dir + '/slurm_files/' -tiger_log_files_dir = tiger_home_dir + '/job_log/' - - -def scp_file_transfer(source, dest): - - p = subprocess.Popen(["scp", source, dest]) - transfer_status = p.wait() - return transfer_status - - -def request_globus_transfer(source, destination): - - globus_command = ["globus", "transfer", source, destination, '--recursive', '--format', 'json'] - print(globus_command) - s = subprocess.run(globus_command, capture_output=True) - transfer_request = json.loads(s.stdout.decode('UTF-8')) - return transfer_request - - -def request_globus_transfer_status(id_task): - - globus_command = ["globus", "task", "show", id_task, '--format', 'json'] - print(globus_command) - s = subprocess.run(globus_command, capture_output=True) - transfer_request = json.loads(s.stdout.decode('UTF-8')) - return transfer_request - - -def globus_transfer_to_tiger(raw_rel_path): - - source_ep = pni_ep_id + ':' + pni_root_data_dir + raw_rel_path - dest_ep = tiger_ep_dir + ':' + tiger_raw_root_data_dir + raw_rel_path - transfer_request = request_globus_transfer(source_ep, dest_ep) - return transfer_request - - -def globus_transfer_to_pni(sorted_rel_path): - - source_ep = tiger_ep_dir + ':' + tiger_sorted_root_data_dir + sorted_rel_path - dest_ep = pni_ep_id + ':' + pni_root_data_dir + sorted_rel_path - transfer_request = request_globus_transfer(source_ep, dest_ep) - return transfer_request \ No newline at end of file From eb0f571c0d859c35f5257c81cc99e0e4da693d10 Mon Sep 17 00:00:00 2001 From: Alvaro Luna Date: Wed, 16 Feb 2022 19:02:06 -0500 Subject: [PATCH 33/34] Ephys_rec schema --- .../automatic_job/recording_handler.py | 8 +- u19_pipeline/ephys.py | 64 +--- u19_pipeline/ephys_rec.py | 298 ++++++++++++++++++ u19_pipeline/recording.py | 1 + 4 files changed, 310 insertions(+), 61 deletions(-) create mode 100644 u19_pipeline/ephys_rec.py diff --git a/u19_pipeline/automatic_job/recording_handler.py b/u19_pipeline/automatic_job/recording_handler.py index 70215e43..aabdcc1b 100644 --- a/u19_pipeline/automatic_job/recording_handler.py +++ b/u19_pipeline/automatic_job/recording_handler.py @@ -7,7 +7,7 @@ import re import pandas as pd import datajoint as dj -from u19_pipeline import recording, ephys, imaging_rec +from u19_pipeline import recording, ephys_rec, imaging_rec import u19_pipeline.utils.dj_shortcuts as dj_short import u19_pipeline.automatic_job.clusters_paths_and_transfers as ft import u19_pipeline.automatic_job.slurm_creator as slurmlib @@ -153,9 +153,9 @@ def modality_preingestion(rec_series, status_series): if rec_series['recording_modality'] == 'electrophysiology': - this_modality_recording_table = ephys.EphysRecording - this_modality_recording_unit_table = ephys.EphysRecordingProbes - this_modality_processing_unit_table = ephys.EphysProcessing + this_modality_recording_table = ephys_rec.EphysRecording + this_modality_recording_unit_table = ephys_rec.EphysRecordingProbes + this_modality_processing_unit_table = ephys_rec.EphysProcessing print('this_modality_recording_table', this_modality_recording_table) diff --git a/u19_pipeline/ephys.py b/u19_pipeline/ephys.py index 2074bb23..c50b3f34 100644 --- a/u19_pipeline/ephys.py +++ b/u19_pipeline/ephys.py @@ -2,15 +2,13 @@ import pathlib import numpy as np -from u19_pipeline import behavior, recording +from u19_pipeline import ephys, behavior from element_array_ephys import probe as probe_element from element_array_ephys import ephys as ephys_element import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX import u19_pipeline.utils.ephys_utils as ephys_utils -import u19_pipeline.utils.path_utils as pu -import u19_pipeline.automatic_job.params_config as config from u19_pipeline.utils.DemoReadSGLXData.readSGLX import readMeta @@ -38,7 +36,7 @@ # 2. Upstream tables schema_reference = dj.schema(dj.config['custom']['database.prefix'] + 'reference') -schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'ephys') +schema = dj.schema(dj.config['custom']['database.prefix'] + 'ephys') @schema_reference @@ -50,65 +48,17 @@ class SkullReference(dj.Lookup): @schema -class EphysRecording(dj.Computed): +class EphysSession(dj.Manual): definition = """ # General information of an ephys session - -> recording.Recording + -> acquisition.Session --- + ephys_directory: varchar(255) # the absolute directory where the ephys data for this session will be stored in bucket """ - @property - def key_source(self): - return recording.Recording & {'recording_modality': 'electrophysiology'} - - def make(self, key): - self.insert1(key) - - -@schema -class EphysRecordingProbes(dj.Computed): - definition = """ - # General information of an ephys session - -> EphysRecording - probe : tinyint # probe number for the recording - --- - probe_directory : varchar(255) # probe specific directory - """ - - def make(self, key): - - root_dir = get_ephys_root_data_dir() - rec_diro = (recording.Recording & key).fetch1('recording_directory') - rec_dir = str(pathlib.Path(root_dir, rec_diro)) - - ephys_probe_pattern = \ - config.recording_modality_df.loc[config.recording_modality_df['RecordingModality'] == 'electrophysiology', 'ProcessUnitFilePattern'].squeeze() - - probe_dirs = pu.get_filepattern_paths(rec_dir, ephys_probe_pattern[0]) - - probe_keys = [] - for idx, probe_dir in enumerate(probe_dirs): - probe_dir = probe_dir.replace(root_dir, "") - this_key = key.copy() - this_key['probe'] = idx - this_key['probe_directory'] = probe_dir - probe_keys.append(this_key) - - if len(probe_keys) > 0: - self.insert(probe_keys) - - -@schema -class EphysProcessing(dj.Manual): - definition = """ - -> recording.RecordingProcess - ----- - -> EphysRecordingProbes - """ - # ephys element requires table with name Session -Session = EphysProcessing +Session = EphysSession # 3. Utility functions @@ -142,7 +92,7 @@ def get_session_directory(session_key): @schema class BehaviorSync(dj.Imported): definition = """ - -> EphysRecording + -> ephys.EphysSession --- nidq_sampling_rate : float # sampling rate of behavioral iterations niSampRate in nidq meta file iteration_index_nidq : longblob # Virmen index time series. Length of this longblob should be the number of samples in the nidaq file. diff --git a/u19_pipeline/ephys_rec.py b/u19_pipeline/ephys_rec.py new file mode 100644 index 00000000..ca0aa6d5 --- /dev/null +++ b/u19_pipeline/ephys_rec.py @@ -0,0 +1,298 @@ +import datajoint as dj +import pathlib +import numpy as np + +from u19_pipeline import behavior, recording + +from element_array_ephys import probe as probe_element +from element_array_ephys import ephys as ephys_element + +import u19_pipeline.utils.DemoReadSGLXData.readSGLX as readSGLX +import u19_pipeline.utils.ephys_utils as ephys_utils +import u19_pipeline.utils.path_utils as pu +import u19_pipeline.automatic_job.params_config as config + +from u19_pipeline.utils.DemoReadSGLXData.readSGLX import readMeta + +""" +------ Gathering requirements to activate the ephys elements ------ +To activate the ephys elements, we need to provide: +1. Schema names + + schema name for the probe module + + schema name for the ephys module +2. Upstream tables + + Session table + + SkullReference table - Reference table for InsertionLocation, specifying the skull reference + used for probe insertion location (e.g. Bregma, Lambda) +3. Utility functions + + get_ephys_root_data_dir() + + get_session_directory() +For more detail, check the docstring of the imaging element: + help(probe_element.activate) + help(ephys_element.activate) +""" + +# 1. Schema names +probe_schema_name = dj.config['custom']['database.prefix'] + 'probe_element' +ephys_schema_name = dj.config['custom']['database.prefix'] + 'ephys_element' + +# 2. Upstream tables +schema_reference = dj.schema(dj.config['custom']['database.prefix'] + 'reference') +schema = dj.schema(dj.config['custom']['database.test.prefix'] + 'ephys_rec') + + +@schema_reference +class SkullReference(dj.Lookup): + definition = """ + skull_reference : varchar(60) + """ + contents = zip(['Bregma', 'Lambda']) + + +@schema +class EphysRecording(dj.Computed): + definition = """ + # General information of an ephys session + -> recording.Recording + --- + """ + @property + def key_source(self): + return recording.Recording & {'recording_modality': 'electrophysiology'} + + def make(self, key): + self.insert1(key) + + +@schema +class EphysRecordingProbes(dj.Computed): + definition = """ + # General information of an ephys session + -> EphysRecording + probe : tinyint # probe number for the recording + --- + probe_directory : varchar(255) # probe specific directory + """ + + def make(self, key): + + root_dir = get_ephys_root_data_dir() + rec_diro = (recording.Recording & key).fetch1('recording_directory') + rec_dir = str(pathlib.Path(root_dir, rec_diro)) + + ephys_probe_pattern = \ + config.recording_modality_df.loc[config.recording_modality_df['RecordingModality'] == 'electrophysiology', 'ProcessUnitFilePattern'].squeeze() + + probe_dirs = pu.get_filepattern_paths(rec_dir, ephys_probe_pattern[0]) + + probe_keys = [] + for idx, probe_dir in enumerate(probe_dirs): + probe_dir = probe_dir.replace(root_dir, "") + this_key = key.copy() + this_key['probe'] = idx + this_key['probe_directory'] = probe_dir + probe_keys.append(this_key) + + if len(probe_keys) > 0: + self.insert(probe_keys) + + +@schema +class EphysProcessing(dj.Manual): + definition = """ + -> recording.RecordingProcess + ----- + -> EphysRecordingProbes + """ + + + +# ephys element requires table with name Session +Session = EphysProcessing + + +# 3. Utility functions + +def get_ephys_root_data_dir(): + data_dir = dj.config.get('custom', {}).get('ephys_root_data_dir', None) + data_dir = pathlib.Path(data_dir) + data_dir = data_dir.as_posix() + return data_dir if data_dir else None + + +def get_session_directory(session_key): + + data_dir = str(get_ephys_root_data_dir()) + sess_dir = (ephys_rec.EphysSession & session_key).fetch1('ephys_directory') + session_dir = pathlib.Path(data_dir + sess_dir) + return session_dir.as_posix() + + +# ------------- Activate "ephys" schema ------------- +ephys_element.activate(ephys_schema_name, probe_schema_name, linking_module=__name__) + + +# ------------- Create Neuropixels probe entries ------------- +for probe_type in ('neuropixels 1.0 - 3A', 'neuropixels 1.0 - 3B', + 'neuropixels 2.0 - SS', 'neuropixels 2.0 - MS'): + probe_element.ProbeType.create_neuropixels_probe(probe_type) + + +# downstream tables for ephys element +@schema +class BehaviorSync(dj.Imported): + definition = """ + -> EphysRecording + --- + nidq_sampling_rate : float # sampling rate of behavioral iterations niSampRate in nidq meta file + iteration_index_nidq : longblob # Virmen index time series. Length of this longblob should be the number of samples in the nidaq file. + trial_index_nidq=null : longblob # Trial index time series. length of this longblob should be the number of samples in the nidaq file. + """ + + class ImecSamplingRate(dj.Part): + definition = """ + -> master + -> ephys_element.ProbeInsertion + --- + ephys_sampling_rate: float # sampling rate of the headstage of a probe, imSampRate in imec meta file + """ + + def make(self, key): + # Pull the Nidaq file/record + session_dir = pathlib.Path(get_session_directory(key)) + nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0] + # And get the datajoint record + behavior = dj.create_virtual_module('behavior', 'u19_behavior') + thissession = behavior.TowersBlock().Trial() & key + behavior_time, iterstart = thissession.fetch('trial_time', 'vi_start') + + # 1: load meta data, and the content of the NIDAQ file. Its content is digital. + nidq_meta = readSGLX.readMeta(nidq_bin_full_path) + nidq_sampling_rate = readSGLX.SampRate(nidq_meta) + digital_array = ephys_utils.spice_glx_utility.load_spice_glx_digital_file(nidq_bin_full_path, nidq_meta) + + # Synchronize between pulses and get iteration # vector for each sample + mode='counter_bit0' + iteration_dict = ephys_utils.get_iteration_sample_vector_from_digital_lines_pulses(digital_array[1,:], digital_array[2,:], nidq_sampling_rate, behavior_time.shape[0], mode) + # Check # of trials and iterations match + status = ephys_utils.assert_iteration_samples_count(iteration_dict['iter_start_idx'], behavior_time) + + #They didn't match, try counter method (if available) + if (not status) and (digital_array.shape[0] > 3): + [framenumber_in_trial, trialnumber] = ephys_utils.behavior_sync_frame_counter_method(digital_array, behavior_time, thissession, nidq_sampling_rate, 3, 5) + iteration_dict['framenumber_vector_samples'] = framenumber_in_trial + iteration_dict['trialnumber_vector_samples'] = trialnumber + + + final_key = dict(key, nidq_sampling_rate = nidq_sampling_rate, + iteration_index_nidq = iteration_dict['framenumber_vector_samples'], + trial_index_nidq = iteration_dict['trialnumber_vector_samples']) + + print(final_key) + + ephys_rec.BehaviorSync.insert1(final_key,allow_direct_insert=True) + + self.insert_imec_sampling_rate(key, session_dir) + + + + def insert_imec_sampling_rate(self, key, session_dir): + + # get the imec sampling rate for a particular probe + here = ephys_element.ProbeInsertion & key + for probe_insertion in here.fetch('KEY'): + #imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.bin'.format(probe_insertion['insertion_number']))) + imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number']))) + + if len(imec_bin_filepath) == 1: # find the binary file to get meta data + imec_bin_filepath = imec_bin_filepath[0] + else: # if this fails, get the ap.meta file. + imec_bin_filepath = list(session_dir.glob('*imec{}/*.ap.meta'.format(probe_insertion['insertion_number']))) + if len(imec_bin_filepath) == 1: + s = str(imec_bin_filepath[0]) + imec_bin_filepath = pathlib.Path(s.replace(".meta", ".bin")) + else: # If this fails too, no imec file exists at the path. + raise NameError("No imec meta file found.") + + imec_meta = readMeta(imec_bin_filepath) + self.ImecSamplingRate.insert1( + dict(probe_insertion, + ephys_sampling_rate=imec_meta['imSampRate'])) + + +@schema +class CuratedClustersIteration(dj.Computed): + definition = """ + -> ephys_element.CuratedClustering + -> BehaviorSync + """ + + class Unit(dj.Part): + definition = """ + -> master + -> ephys_element.CuratedClustering.Unit + --- + spike_counts_iteration: longblob # number of spikes during each iteration. have length as the number of iterations - 1 + firing_rate_before_first_iteration: float + firing_rate_after_last_iteration: float + """ + + def make(self, key): + + self.insert1(key) + + nidq_sampling_rate, iteration_index_nidq = \ + (BehaviorSync * BehaviorSync.ImecSamplingRate & key).fetch1( + 'nidq_sampling_rate', 'iteration_index_nidq') + + key_session = key.copy() + del key_session["insertion_number"] + + thissession = behavior.TowersBlock().Trial() & key + iterstart = thissession.fetch('vi_start') + + first_vr_iteration = iterstart[0] + + # Obtain the precise times when the frames transition. + # This is obtained from iteration_index_nidq + ls = np.diff(iteration_index_nidq) + ls[ls<0] = 1 # These are the trial transitions (see definition above). To get total number of frames, we define this as a transition like all others. + ls[np.isnan(ls)] = 0 + iteration_transition_indexes = np.where(ls)[0] + + # First iterations captured not in virmen because vr was not started yet + #for i in range(first_vr_iteration): + + # if iteration_index_nidq[iteration_transition_indexes[i]] <= first_vr_iteration: + # ls[iteration_transition_indexes[i]] = 0 + + print('sum_iterationtrans', np.sum(ls)) + + iteration_times = np.where(ls)[0]/nidq_sampling_rate + + # get end of time from nidq metadata + session_dir = pathlib.Path(get_session_directory(key)) + nidq_bin_full_path = list(session_dir.glob('*nidq.bin*'))[0] + nidq_meta = readMeta(nidq_bin_full_path) + t_end = np.float(nidq_meta['fileTimeSecs']) + + unit_spike_counts = [] + + for unit_key in (ephys_element.CuratedClustering.Unit & key).fetch('KEY'): + spike_times = (ephys_element.CuratedClustering.Unit & unit_key).fetch1('spike_times') + # vector with length n_iterations + 1 + spike_counts_iteration = np.bincount(np.digitize(spike_times, iteration_times)) + + firing_rate_before_first_iteration = spike_counts_iteration[0]/iteration_times[0] + firing_rate_after_last_iteration = spike_counts_iteration[-1]/(t_end - iteration_times[-1]) + + # remove the first and last entries + spike_counts_iteration = np.delete(spike_counts_iteration, [0, -1]) + + unit_spike_counts.append( + dict(unit_key, + spike_counts_iteration=spike_counts_iteration, + firing_rate_before_first_iteration=firing_rate_before_first_iteration, + firing_rate_after_last_iteration=firing_rate_after_last_iteration) + ) + self.Unit.insert(unit_spike_counts) diff --git a/u19_pipeline/recording.py b/u19_pipeline/recording.py index 300fdcfc..b9f625c6 100644 --- a/u19_pipeline/recording.py +++ b/u19_pipeline/recording.py @@ -19,6 +19,7 @@ class RecordingModality(dj.Lookup): process_unit_file_pattern: blob # process "unit" pattern to find in path process_unit_dir_fieldname: varchar(64) # FieldName that stores process unit directory for specific modality process_unit_fieldname: varchar(32) # FieldName that stores process unit for specific modality (fov, probe, etc) + process_repository: varchar(64) # Name of the repository that handles the processing of these modality """ contents = config.recording_modality_list From 21172aa357e0c76236fbade39801e84eacffcd35 Mon Sep 17 00:00:00 2001 From: Alvalunasan Date: Thu, 17 Feb 2022 08:33:53 -0500 Subject: [PATCH 34/34] Last notebook tests --- notebooks/TestAutomaticPipeline.ipynb | 570 ++++++++++++++++++++++---- 1 file changed, 501 insertions(+), 69 deletions(-) diff --git a/notebooks/TestAutomaticPipeline.ipynb b/notebooks/TestAutomaticPipeline.ipynb index c532d4ea..930adad0 100644 --- a/notebooks/TestAutomaticPipeline.ipynb +++ b/notebooks/TestAutomaticPipeline.ipynb @@ -27,9 +27,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "metadata": {}, "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Connecting alvaros@datajoint00.pni.princeton.edu:3306\n" + ] + }, { "output_type": "execute_result", "data": { @@ -38,7 +45,7 @@ ] }, "metadata": {}, - "execution_count": 6 + "execution_count": 2 } ], "source": [ @@ -52,6 +59,8 @@ "import pathlib\n", "import u19_pipeline.automatic_job as auto_pipeline\n", "import u19_pipeline.automatic_job.recording_handler as rec_handler\n", + "import u19_pipeline.automatic_job.recording_process_handler as rec_process_handler\n", + "\n", "\n", "from u19_pipeline import recording, imaging_rec, ephys\n", "\n", @@ -92,6 +101,30 @@ "cell_type": "code", "execution_count": 4, "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Empty DataFrame\n", + "Columns: []\n", + "Index: []" + ], + "text/html": "
\n\n\n \n \n \n \n \n \n \n
\n
" + }, + "metadata": {}, + "execution_count": 4 + } + ], + "source": [ + "df_params = pd.DataFrame(recording.PreprocessParamSet.fetch(as_dict=True))\n", + "df_params" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -100,7 +133,7 @@ "fake_recording['location'] = 'Bezos1'\n", "fake_recording['status_recording_idx'] = 0\n", "fake_recording['process_paramset_idx'] = 1\n", - "fake_recording['preprocess_paramset_idx'] = 1\n", + "fake_recording['preprocess_paramset_idx'] = 3\n", "fake_recording['recording_directory'] = 'ms81/ms81_M004/20210507/towersTask_g0'\n", "fake_recording['local_directory'] = 'ms81/ms81_M004/20210507/towersTask_g0'\n", "\n", @@ -118,7 +151,7 @@ "fake_recording['location'] = 'Bezos1'\n", "fake_recording['status_recording_idx'] = 0\n", "fake_recording['process_paramset_idx'] = 1\n", - "fake_recording['preprocess_paramset_idx'] = 2\n", + "fake_recording['preprocess_paramset_idx'] = 3\n", "fake_recording['recording_directory'] = 'emdia/emdia_gps24/20220111'\n", "fake_recording['local_directory'] = 'emdia/emdia_gps24/20220111'\n", "\n", @@ -127,14 +160,205 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, + "execution_count": 9, + "metadata": { + "tags": [] + }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ - "function to apply: modality_preingestion\nimaging scan insert recording_id 4\nrecording_modality imaging\nlocation Bezos1\nstatus_recording_idx 2\npreprocess_paramset_idx 2\nprocess_paramset_idx 1\ntask_copy_id_pni None\ninherit_params_recording 1\nrecording_directory emdia/emdia_gps24/20220111\nlocal_directory emdia/emdia_gps24/20220111\nquery_key {'recording_id': 4}\nName: 0, dtype: object\nrecording_units [{'recording_id': 4, 'fov': 1, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'fov_name': 'mM2_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[-1662.46125877, 269.83897896]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 2, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'fov_name': 'RSP_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1142.3930918 , -99.24825482]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}, {'recording_id': 4, 'fov': 3, 'fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'relative_fov_directory': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'fov_name': 'AM_z1', 'fov_depth': 1120.0, 'fov_center_xy': array([[1555.00006185, 1097.92718087]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 75.0}]\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI01_z1/', 'status_pipeline_idx': 0}\nrecording_process [24]\nthis_mod_processing {'recording_id': 4, 'fov': 1, 'recording_process_id': 24}\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI02_z1/', 'status_pipeline_idx': 0}\nrecording_process [25]\nthis_mod_processing {'recording_id': 4, 'fov': 2, 'recording_process_id': 25}\nthis_recprocess_key {'recording_id': 4, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'recording_process_pre_path': 'emdia/emdia_gps24/20220111/ROI03_z1/', 'status_pipeline_idx': 0}\nrecording_process [26]\nthis_mod_processing {'recording_id': 4, 'fov': 3, 'recording_process_id': 26}\nkey to update {'recording_id': 4}\nrecording_key_dict {'recording_id': 4}\nstatus 3\nupdate_field None\nupdate_value None\nupdate_status_dict {'recording_id': 4, 'status_recording_idx': 3}\n" + "function to apply: modality_preingestion\n", + "['/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh', '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts', 'recording_id=1']\n", + "aftercommand before comm\n", + "aftercommand after comm\n", + "\n", + " < M A T L A B (R) >\n", + " Copyright 1984-2020 The MathWorks, Inc.\n", + " R2020b Update 1 (9.9.0.1495850) 64-bit (glnxa64)\n", + " September 30, 2020\n", + "\n", + " \n", + "To get started, type doc.\n", + "For product information, visit www.mathworks.com.\n", + " \n", + " ----------------------------------------------------\n", + "\tYour license will expire in 14 days.\n", + "\tPlease contact your system administrator or\n", + "\tMathWorks to renew this license.\n", + " ----------------------------------------------------\n", + "\n", + "s =\n", + "\n", + " 0\n", + "\n", + " 0: datajoint00.pni.princeton.edu via TCP/IP Server version 5.5.5-10.2.33-MariaDB (encrypted)\n", + "database connection id: 67647\n", + "\n", + "**imaging_rec.ScanInfo: Found 1 unpopulated keys\n", + "\n", + "Populating imaging_rec.ScanInfo for:\n", + " recording_id: 1\n", + "\n", + "------------ preparing /mnt/cup/braininit/Data/imaging/lpinto/lpinto_SP6/20191011 --------------\n", + "\tgetting headers...\n", + "Starting parallel pool (parpool) using the 'local' profile ...\n", + "Connected to the parallel pool (number of workers: 16).\n", + "estimating bleaching......................... done after 0.2 min\n", + "\tdone after 0.8 min\n", + "closing DataJoint connection #0\n", + "\n", + "\n", + "recording_units [{'recording_id': 1, 'fov': 1, 'fov_directory': 'lpinto/lpinto_SP6/20191011/ROI01_z1/', 'fov_name': 'ROI01_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[-1103.73353235, -99.46859652]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 80.0}, {'recording_id': 1, 'fov': 2, 'fov_directory': 'lpinto/lpinto_SP6/20191011/ROI02_z1/', 'fov_name': 'ROI02_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[ 754.37518278, -157.18221563]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 80.0}, {'recording_id': 1, 'fov': 3, 'fov_directory': 'lpinto/lpinto_SP6/20191011/ROI03_z1/', 'fov_name': 'ROI03_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[1436.87351828, 673.17742791]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 80.0}]\n", + "this_recprocess_key {'recording_id': 1, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 1, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191011/ROI01_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [1]\n", + "this_mod_processing {'recording_id': 1, 'fov': 1, 'recording_process_id': 1}\n", + "this_recprocess_key {'recording_id': 1, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 2, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191011/ROI02_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [2]\n", + "this_mod_processing {'recording_id': 1, 'fov': 2, 'recording_process_id': 2}\n", + "this_recprocess_key {'recording_id': 1, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 3, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191011/ROI03_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [3]\n", + "this_mod_processing {'recording_id': 1, 'fov': 3, 'recording_process_id': 3}\n", + "key to update {'recording_id': 1}\n", + "recording_key_dict {'recording_id': 1}\n", + "status 3\n", + "update_field None\n", + "update_value None\n", + "update_status_dict {'recording_id': 1, 'status_recording_idx': 3}\n", + "function to apply: modality_preingestion\n", + "['/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh', '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts', 'recording_id=2']\n", + "aftercommand before comm\n", + "aftercommand after comm\n", + "\n", + " < M A T L A B (R) >\n", + " Copyright 1984-2020 The MathWorks, Inc.\n", + " R2020b Update 1 (9.9.0.1495850) 64-bit (glnxa64)\n", + " September 30, 2020\n", + "\n", + " \n", + "To get started, type doc.\n", + "For product information, visit www.mathworks.com.\n", + " \n", + " ----------------------------------------------------\n", + "\tYour license will expire in 14 days.\n", + "\tPlease contact your system administrator or\n", + "\tMathWorks to renew this license.\n", + " ----------------------------------------------------\n", + "\n", + "s =\n", + "\n", + " 0\n", + "\n", + " 0: datajoint00.pni.princeton.edu via TCP/IP Server version 5.5.5-10.2.33-MariaDB (encrypted)\n", + "database connection id: 67651\n", + "\n", + "**imaging_rec.ScanInfo: Found 1 unpopulated keys\n", + "\n", + "Populating imaging_rec.ScanInfo for:\n", + " recording_id: 2\n", + "\n", + "------------ preparing /mnt/cup/braininit/Data/imaging/lpinto/lpinto_SP6/20191010 --------------\n", + "\tgetting headers...\n", + "Starting parallel pool (parpool) using the 'local' profile ...\n", + "Connected to the parallel pool (number of workers: 16).\n", + "estimating bleaching........................ done after 0.2 min\n", + "\tdone after 0.8 min\n", + "closing DataJoint connection #0\n", + "\n", + "\n", + "recording_units [{'recording_id': 2, 'fov': 1, 'fov_directory': 'lpinto/lpinto_SP6/20191010/ROI01_z1/', 'fov_name': 'ROI01_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[-1103.73353235, -99.46859652]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}, {'recording_id': 2, 'fov': 2, 'fov_directory': 'lpinto/lpinto_SP6/20191010/ROI02_z1/', 'fov_name': 'ROI02_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[ 754.37518278, -157.18221563]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}, {'recording_id': 2, 'fov': 3, 'fov_directory': 'lpinto/lpinto_SP6/20191010/ROI03_z1/', 'fov_name': 'ROI03_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[1436.87351828, 673.17742791]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}]\n", + "this_recprocess_key {'recording_id': 2, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 1, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191010/ROI01_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [4]\n", + "this_mod_processing {'recording_id': 2, 'fov': 1, 'recording_process_id': 4}\n", + "this_recprocess_key {'recording_id': 2, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 2, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191010/ROI02_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [5]\n", + "this_mod_processing {'recording_id': 2, 'fov': 2, 'recording_process_id': 5}\n", + "this_recprocess_key {'recording_id': 2, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 3, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191010/ROI03_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [6]\n", + "this_mod_processing {'recording_id': 2, 'fov': 3, 'recording_process_id': 6}\n", + "key to update {'recording_id': 2}\n", + "recording_key_dict {'recording_id': 2}\n", + "status 3\n", + "update_field None\n", + "update_value None\n", + "update_status_dict {'recording_id': 2, 'status_recording_idx': 3}\n", + "function to apply: modality_preingestion\n", + "['/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh', '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts', 'recording_id=3']\n", + "aftercommand before comm\n", + "aftercommand after comm\n", + "\n", + " < M A T L A B (R) >\n", + " Copyright 1984-2020 The MathWorks, Inc.\n", + " R2020b Update 1 (9.9.0.1495850) 64-bit (glnxa64)\n", + " September 30, 2020\n", + "\n", + " \n", + "To get started, type doc.\n", + "For product information, visit www.mathworks.com.\n", + " \n", + " ----------------------------------------------------\n", + "\tYour license will expire in 14 days.\n", + "\tPlease contact your system administrator or\n", + "\tMathWorks to renew this license.\n", + " ----------------------------------------------------\n", + "\n", + "s =\n", + "\n", + " 0\n", + "\n", + " 0: datajoint00.pni.princeton.edu via TCP/IP Server version 5.5.5-10.2.33-MariaDB (encrypted)\n", + "database connection id: 67655\n", + "\n", + "**imaging_rec.ScanInfo: Found 1 unpopulated keys\n", + "\n", + "Populating imaging_rec.ScanInfo for:\n", + " recording_id: 3\n", + "\n", + "------------ preparing /mnt/cup/braininit/Data/imaging/lpinto/lpinto_SP6/20191008 --------------\n", + "\tgetting headers...\n", + "Starting parallel pool (parpool) using the 'local' profile ...\n", + "Connected to the parallel pool (number of workers: 16).\n", + "estimating bleaching...................... done after 0.2 min\n", + "\tdone after 0.7 min\n", + "closing DataJoint connection #0\n", + "\n", + "\n", + "recording_units [{'recording_id': 3, 'fov': 1, 'fov_directory': 'lpinto/lpinto_SP6/20191008/ROI01_z1/', 'fov_name': 'ROI01_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[-1103.73353235, -194.73742733]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}, {'recording_id': 3, 'fov': 2, 'fov_directory': 'lpinto/lpinto_SP6/20191008/ROI02_z1/', 'fov_name': 'ROI02_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[ 796.71688541, -390.06157986]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}, {'recording_id': 3, 'fov': 3, 'fov_directory': 'lpinto/lpinto_SP6/20191008/ROI03_z1/', 'fov_name': 'ROI03_z1', 'fov_depth': 0.0, 'fov_center_xy': array([[1447.45894398, 567.32317143]]), 'fov_size_xy': array([[537.60000003, 537.60000003]]), 'fov_rotation_degrees': 0.0, 'fov_pixel_resolution_xy': array([[512., 512.]]), 'fov_discrete_plane_mode': 0, 'power_percent': 87.0}]\n", + "this_recprocess_key {'recording_id': 3, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 1, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191008/ROI01_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [7]\n", + "this_mod_processing {'recording_id': 3, 'fov': 1, 'recording_process_id': 7}\n", + "this_recprocess_key {'recording_id': 3, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 2, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191008/ROI02_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [8]\n", + "this_mod_processing {'recording_id': 3, 'fov': 2, 'recording_process_id': 8}\n", + "this_recprocess_key {'recording_id': 3, 'preprocess_paramset_idx': 2, 'process_paramset_idx': 1, 'fragment_number': 3, 'recording_process_pre_path': 'lpinto/lpinto_SP6/20191008/ROI03_z1/', 'status_pipeline_idx': 0}\n", + "recording_process [9]\n", + "this_mod_processing {'recording_id': 3, 'fov': 3, 'recording_process_id': 9}\n", + "key to update {'recording_id': 3}\n", + "recording_key_dict {'recording_id': 3}\n", + "status 3\n", + "update_field None\n", + "update_value None\n", + "update_status_dict {'recording_id': 3, 'status_recording_idx': 3}\n", + "function to apply: modality_preingestion\n", + "['/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/ingest_scaninfo_shell.sh', '/usr/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline-matlab/scripts', 'recording_id=4']\n" + ] + }, + { + "output_type": "error", + "ename": "KeyboardInterrupt", + "evalue": "", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_217536/2544093602.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ml\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrec_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRecordingHandler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpipeline_handler_main\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/mnt/cup/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/recording_handler.py\u001b[0m in \u001b[0;36mpipeline_handler_main\u001b[0;34m()\u001b[0m\n\u001b[1;32m 49\u001b[0m \u001b[0;31m#Trigger process, if success update recording process record\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0msuccess_process\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdate_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunction_status_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecording_series\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_status_series\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 53\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msuccess_process\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/mnt/cup/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/automatic_job/recording_handler.py\u001b[0m in \u001b[0;36mmodality_preingestion\u001b[0;34m(rec_series, status_series)\u001b[0m\n\u001b[1;32m 175\u001b[0m \u001b[0;31m# Insert this modality recording and recording \"unit\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0mthis_modality_recording_table\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec_series\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'query_key'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 177\u001b[0;31m \u001b[0mthis_modality_recording_unit_table\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopulate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrec_series\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'query_key'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# In imaging this is to call imaging.ScanInfo populate Script\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 178\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[0;31m# Get all recording probes (\"units\") from this recording\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/mnt/cup/people/alvaros/BrainCogsProjects/Datajoint_projs/U19-pipeline_python/u19_pipeline/imaging_rec.py\u001b[0m in \u001b[0;36mpopulate\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 92\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstdout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPIPE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 95\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'aftercommand before comm'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 96\u001b[0m \u001b[0mstdout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstderr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommunicate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 1017\u001b[0m \u001b[0mendtime\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1018\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1019\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1020\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;31m# https://bugs.python.org/issue25942\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_wait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 1651\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1652\u001b[0m \u001b[0;32mbreak\u001b[0m \u001b[0;31m# Another thread waited.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1653\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1654\u001b[0m \u001b[0;31m# Check the pid and loop as waitpid has been known to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1655\u001b[0m \u001b[0;31m# return 0 even without WNOHANG in odd situations.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/subprocess.py\u001b[0m in \u001b[0;36m_try_wait\u001b[0;34m(self, wait_flags)\u001b[0m\n\u001b[1;32m 1609\u001b[0m \u001b[0;34m\"\"\"All callers to this function MUST hold self._waitpid_lock.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1610\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1611\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msts\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwait_flags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1612\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mChildProcessError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1613\u001b[0m \u001b[0;31m# This happens if SIGCLD is set to be ignored or waiting\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -142,40 +366,218 @@ "l = rec_handler.RecordingHandler.pipeline_handler_main()" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "l = rec_handler.RecordingHandler.pipeline_handler_main()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "suite2p\n", + "recording_process_id_16\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_16.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_16.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_17\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_17.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_17.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_18\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_18.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_18.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_19\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_19.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_19.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_27\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_27.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_27.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_28\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_28.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_28.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_29\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_29.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_29.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_30\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_30.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_30.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "suite2p\n", + "recording_process_id_31\n", + "\n", + "\n", + "scp slurm_files/slurm_recording_process_id_31.slurm alvaros@tigergpu.princeton.edu:/scratch/gpfs/BRAINCOGS/slurm_files/slurm_recording_process_id_31.slurm\n", + "Permission denied (publickey,keyboard-interactive).\n", + "lost connection\n", + "update_dict {'value_update': None, 'error_info': None}\n", + "process_cluter: tiger\n", + "si fue tiger\n", + "update_dict {'value_update': UUID('5a105e8b-9d40-e132-9780-d62ea2265d8a'), 'error_info': None}\n", + "key to update {'recording_process_id': 32}\n", + "old status 0 new status 1\n", + "value_update 5a105e8b-9d40-e132-9780-d62ea2265d8a field_update task_copy_id_pre\n", + "function executed: transfer_request\n", + "process_cluter: tiger\n", + "si fue tiger\n", + "update_dict {'value_update': UUID('5a105e8b-9d40-e132-9780-d62ea2265d8a'), 'error_info': None}\n", + "key to update {'recording_process_id': 33}\n", + "old status 0 new status 1\n", + "value_update 5a105e8b-9d40-e132-9780-d62ea2265d8a field_update task_copy_id_pre\n", + "function executed: transfer_request\n", + "process_cluter: tiger\n", + "si fue tiger\n", + "update_dict {'value_update': UUID('5a105e8b-9d40-e132-9780-d62ea2265d8a'), 'error_info': None}\n", + "key to update {'recording_process_id': 34}\n", + "old status 0 new status 1\n", + "value_update 5a105e8b-9d40-e132-9780-d62ea2265d8a field_update task_copy_id_pre\n", + "function executed: transfer_request\n", + "process_cluter: tiger\n", + "si fue tiger\n", + "update_dict {'value_update': UUID('5a105e8b-9d40-e132-9780-d62ea2265d8a'), 'error_info': None}\n", + "key to update {'recording_process_id': 35}\n", + "old status 0 new status 1\n", + "value_update 5a105e8b-9d40-e132-9780-d62ea2265d8a field_update task_copy_id_pre\n", + "function executed: transfer_request\n" + ] + } + ], + "source": [ + "l = rec_process_handler.RecProcessHandler.pipeline_handler_main()" + ] + }, { "cell_type": "code", "execution_count": 8, "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[[(array([[2.]]), array(['kilosort2'], dtype='\nsoy ndarray\nsoy ndarray\n" + ] + } + ], + "source": [ + "keyo = dict()\n", + "keyo['preprocess_paramset_idx'] = 1\n", + "a = recording.PreprocessParamSet()\n", + "lo = a.get_preprocess_params(keyo)\n", + "\n", + "keyo2 = dict()\n", + "keyo2['process_paramset_idx'] = 1\n", + "b = recording.ProcessParamSet()\n", + "lu = b.get_process_params(keyo2)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "*recording_pro recording_id status_pipelin preprocess_par process_params recording_proc recording_proc task_copy_id_p task_copy_id_p slurm_id \n", - "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +------------+ +----------+\n", - "16 2 0 1 1 /Volumes/brain None None None None \n", - "17 2 0 1 1 /Volumes/brain None None None None \n", - "18 3 0 1 1 /Volumes/brain None None None None \n", - "19 3 0 1 1 /Volumes/brain None None None None \n", - "24 4 0 2 1 emdia/emdia_gp None None None None \n", - "25 4 0 2 1 emdia/emdia_gp None None None None \n", - "26 4 0 2 1 emdia/emdia_gp None None None None \n", - " (Total: 7)" - ], - "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

status_pipeline_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

recording_process_pre_path

\n relative path for raw data recording subdirectory that will be processed (ephys-> probe, imaging->fieldofview)\n
\n

recording_process_post_path

\n relative path for processed data recording\n
\n

task_copy_id_pre

\n id for globus transfer task raw file cup->tiger\n
\n

task_copy_id_post

\n id for globus transfer task sorted file tiger->cup\n
\n

slurm_id

\n id for slurm process in tiger\n
162011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
172011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
183011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/NoneNoneNoneNone
193011/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/NoneNoneNoneNone
244021emdia/emdia_gps24/20220111/ROI01_z1/NoneNoneNoneNone
254021emdia/emdia_gps24/20220111/ROI02_z1/NoneNoneNoneNone
264021emdia/emdia_gps24/20220111/ROI03_z1/NoneNoneNoneNone
\n \n

Total: 7

\n " + "{'cat_gt': 2, 'sorting_algorithm': 'kilosort2'}" + ] }, "metadata": {}, - "execution_count": 8 + "execution_count": 9 } ], "source": [ - "recording.RecordingProcess()\n", - "\n" + "lo" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "diro = os.getcwd()\n", + "param_file = str(pathlib.Path(diro, 'u19_pipeline', 'automatic_job', 'process_params_examples', 'preprocess_params_example.json'))\n", + "f = open(param_file)\n", + "param_dict = json.load(f)\n", + "param_dict\n", + "\n", + "keyu = dict()\n", + "keyu['preprocess_paramset_idx'] = 1\n", + "keyu['preprocess_paramset'] = param_dict\n", + "a.update1(keyu)" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Index(['cat_gt', 'sorting_algorithm'], dtype='object')" + ] + }, + "metadata": {}, + "execution_count": 67 + } + ], + "source": [ + "ws.columns" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -192,7 +594,7 @@ "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

recording_modality

\n modalities for recording (ephys, imaging, video_recording, etc.)\n
\n

location

\n \n
\n

status_recording_idx

\n status in the automate process pipeline\n
\n

preprocess_paramset_idx

\n \n
\n

process_paramset_idx

\n \n
\n

task_copy_id_pni

\n id for globus transfer task raw file local->cup\n
\n

inherit_params_recording

\n all RecordingProcess from a recording will have same paramSets\n
\n

recording_directory

\n relative directory where the recording will be stored on cup\n
\n

local_directory

\n local directory where the recording is stored on system\n
2electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
3electrophysiologyBezos1311None1ms81/ms81_M004/20210507/towersTask_g0ms81/ms81_M004/20210507/towersTask_g0
4imagingBezos1321None1emdia/emdia_gps24/20220111emdia/emdia_gps24/20220111
\n \n

Total: 3

\n " }, "metadata": {}, - "execution_count": 9 + "execution_count": 6 } ], "source": [ @@ -201,7 +603,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -219,7 +621,7 @@ "text/html": "\n \n \n \n General information of an ephys session\n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
\n

probe_directory

\n probe specific directory\n
20/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
21/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
30/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec0/
31/Volumes/braininit/Data/electrophysiology/ms81/ms81_M004/20210507/towersTask_g0/towersTask_g0_imec1/
\n \n

Total: 4

\n " }, "metadata": {}, - "execution_count": 10 + "execution_count": 7 } ], "source": [ @@ -235,7 +637,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -253,7 +655,7 @@ "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

probe

\n probe number for the recording\n
1620
1721
1830
1931
\n \n

Total: 4

\n " }, "metadata": {}, - "execution_count": 11 + "execution_count": 8 } ], "source": [ @@ -262,7 +664,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -277,7 +679,7 @@ "text/html": "\n \n \n \n General information of an imaging session\n
\n \n \n \n
\n

recording_id

\n Unique number assigned to each recording\n
4
\n \n

Total: 1

\n " }, "metadata": {}, - "execution_count": 12 + "execution_count": 9 } ], "source": [ @@ -286,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -301,7 +703,7 @@ "text/html": "\n \n \n \n metainfo about imaging session\n
\n \n \n \n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

file_name_base

\n base name of the file\n
\n

scan_width

\n width of scanning in pixels\n
\n

scan_height

\n height of scanning in pixels\n
\n

acq_time

\n acquisition time\n
\n

n_depths

\n number of depths\n
\n

scan_depths

\n depth values in this scan\n
\n

frame_rate

\n imaging frame rate\n
\n

inter_fov_lag_sec

\n time lag in secs between fovs\n
\n

frame_ts_sec

\n frame timestamps in secs 1xnFrames\n
\n

channels

\n is this the channer number or total number of channels\n
\n

cfg_filename

\n cfg file path\n
\n

usr_filename

\n usr file path\n
\n

fast_z_lag

\n fast z lag\n
\n

fast_z_flyback_time

\n time it takes to fly back to fov\n
\n

line_period

\n scan time per line\n
\n

scan_frame_period

\n \n
\n

scan_volume_rate

\n \n
\n

flyback_time_per_frame

\n \n
\n

flyto_time_per_scan_field

\n \n
\n

fov_corner_points

\n coordinates of the corners of the full 5mm FOV, in microns\n
\n

nfovs

\n number of field of view\n
\n

nframes

\n number of frames in the scan\n
\n

nframes_good

\n number of frames in the scan before acceptable sample bleaching threshold is crossed\n
\n

last_good_file

\n number of the file containing the last good frame because of bleaching\n
\n

motion_correction_enabled

\n \n
\n

motion_correction_mode

\n \n
\n

stacks_enabled

\n \n
\n

stack_actuator

\n \n
\n

stack_definition

\n \n
4emdia/emdia_gps24/20220111/gps24_20220111_mainrecording_00001_00001.tif51215882022-01-11 00:00:001=BLOB=14.30050.001=BLOB==BLOB=C:\\Users\\admin\\Documents\\MATLAB\\MikaToo.cfgC:\\Users\\admin\\Documents\\MATLAB\\Mika.usr0.00080.0044.14754e-050.069927614.30050.0010.001=BLOB=35146836000181automated1fastZarbitrary
\n \n

Total: 1

\n " }, "metadata": {}, - "execution_count": 13 + "execution_count": 10 } ], "source": [ @@ -310,7 +712,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -319,15 +721,15 @@ "text/plain": [ "*recording_pro recording_id fov \n", "+------------+ +------------+ +-----+\n", - "24 4 1 \n", - "25 4 2 \n", - "26 4 3 \n", + "27 4 1 \n", + "28 4 2 \n", + "29 4 3 \n", " (Total: 3)" ], - "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

fov

\n number of the field of view in this scan\n
2441
2542
2643
\n \n

Total: 3

\n " + "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n\n
\n

recording_process_id

\n Unique number assigned to each processing job for a recording unit\n
\n

recording_id

\n Unique number assigned to each recording\n
\n

fov

\n number of the field of view in this scan\n
2741
2842
2943
\n \n

Total: 3

\n " }, "metadata": {}, - "execution_count": 14 + "execution_count": 11 } ], "source": [ @@ -336,19 +738,19 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 12, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "" + "" ], - "image/svg+xml": "\n\n%3\n\n\n56\n\n56\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n56->recording.acquisition.Session\n\n\n\n55\n\n55\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n55->recording.RecordingProcessStatus\n\n\n\n54\n\n54\n\n\n54->recording.RecordingProcessStatus\n\n\n\nephys.EphysProcessing\n\n\nephys.EphysProcessing\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nrecording.RecordingProcess->ephys.EphysProcessing\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nephys.EphysRecordingProbes\n\n\nephys.EphysRecordingProbes\n\n\n\n\nephys.EphysRecordingProbes->ephys.EphysProcessing\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->54\n\n\n\nrecording.StatusProcessDefinition->55\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->56\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nephys.EphysRecording->ephys.EphysRecordingProbes\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n" + "image/svg+xml": "\n\n%3\n\n\n2\n\n2\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n2->recording.acquisition.Session\n\n\n\n0\n\n0\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n0->recording.RecordingProcessStatus\n\n\n\n1\n\n1\n\n\n1->recording.RecordingProcessStatus\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nephys.EphysProcessing\n\n\nephys.EphysProcessing\n\n\n\n\nrecording.RecordingProcess->ephys.EphysProcessing\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nephys.EphysRecording\n\n\nephys.EphysRecording\n\n\n\n\nrecording.Recording->ephys.EphysRecording\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->2\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->0\n\n\n\nrecording.StatusProcessDefinition->1\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nephys.EphysRecordingProbes\n\n\nephys.EphysRecordingProbes\n\n\n\n\nephys.EphysRecordingProbes->ephys.EphysProcessing\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nephys.EphysRecording->ephys.EphysRecordingProbes\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n" }, "metadata": {}, - "execution_count": 16 + "execution_count": 12 } ], "source": [ @@ -357,19 +759,19 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ - "" + "" ], - "image/svg+xml": "\n\n%3\n\n\n110\n\n110\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n110->recording.acquisition.Session\n\n\n\n108\n\n108\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n108->recording.RecordingProcessStatus\n\n\n\n109\n\n109\n\n\n109->recording.RecordingProcessStatus\n\n\n\nimaging_rec.ImagingProcessing\n\n\nimaging_rec.ImagingProcessing\n\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingProcessing\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->108\n\n\n\nrecording.StatusProcessDefinition->109\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nimaging_rec.FieldOfView\n\n\nimaging_rec.FieldOfView\n\n\n\n\nimaging_rec.FieldOfView->imaging_rec.ImagingProcessing\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\nimaging_rec.Scan->imaging_rec.FieldOfView\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->110\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\n" + "image/svg+xml": "\n\n%3\n\n\n56\n\n56\n\n\nrecording.acquisition.Session\n\n\nrecording.acquisition.Session\n\n\n\n\n56->recording.acquisition.Session\n\n\n\n55\n\n55\n\n\nrecording.RecordingProcessStatus\n\n\nrecording.RecordingProcessStatus\n\n\n\n\n55->recording.RecordingProcessStatus\n\n\n\n54\n\n54\n\n\n54->recording.RecordingProcessStatus\n\n\n\nimaging_rec.Scan\n\n\nimaging_rec.Scan\n\n\n\n\nimaging_rec.FieldOfView\n\n\nimaging_rec.FieldOfView\n\n\n\n\nimaging_rec.Scan->imaging_rec.FieldOfView\n\n\n\nrecording.RecordingProcess\n\n\nrecording.RecordingProcess\n\n\n\n\nrecording.RecordingProcess->recording.RecordingProcessStatus\n\n\n\nimaging_rec.ImagingProcessing\n\n\nimaging_rec.ImagingProcessing\n\n\n\n\nrecording.RecordingProcess->imaging_rec.ImagingProcessing\n\n\n\nrecording.Recording\n\n\nrecording.Recording\n\n\n\n\nrecording.Recording->imaging_rec.Scan\n\n\n\nrecording.Recording->recording.RecordingProcess\n\n\n\nrecording.Recording.BehaviorSession\n\n\nrecording.Recording.BehaviorSession\n\n\n\n\nrecording.Recording->recording.Recording.BehaviorSession\n\n\n\nrecording.Recording.RecordingSession\n\n\nrecording.Recording.RecordingSession\n\n\n\n\nrecording.Recording->recording.Recording.RecordingSession\n\n\n\nrecording.StatusRecordingDefinition\n\n\nrecording.StatusRecordingDefinition\n\n\n\n\nrecording.StatusRecordingDefinition->recording.Recording\n\n\n\nrecording.ProcessParamSet\n\n\nrecording.ProcessParamSet\n\n\n\n\nrecording.ProcessParamSet->recording.RecordingProcess\n\n\n\nrecording.ProcessParamSet->recording.Recording\n\n\n\nimaging_rec.FieldOfView->imaging_rec.ImagingProcessing\n\n\n\nrecording.acquisition.Session->recording.Recording.BehaviorSession\n\n\n\nrecording.lab.Location\n\n\nrecording.lab.Location\n\n\n\n\nrecording.lab.Location->56\n\n\n\nrecording.lab.Location->recording.Recording\n\n\n\nrecording.subject.Subject\n\n\nrecording.subject.Subject\n\n\n\n\nrecording.lab.Location->recording.subject.Subject\n\n\n\nrecording.RecordingModality\n\n\nrecording.RecordingModality\n\n\n\n\nrecording.RecordingModality->recording.Recording\n\n\n\nrecording.RecordingModality->recording.ProcessParamSet\n\n\n\nrecording.PreprocessParamSet\n\n\nrecording.PreprocessParamSet\n\n\n\n\nrecording.RecordingModality->recording.PreprocessParamSet\n\n\n\nrecording.StatusProcessDefinition\n\n\nrecording.StatusProcessDefinition\n\n\n\n\nrecording.StatusProcessDefinition->54\n\n\n\nrecording.StatusProcessDefinition->55\n\n\n\nrecording.StatusProcessDefinition->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet->recording.RecordingProcess\n\n\n\nrecording.PreprocessParamSet->recording.Recording\n\n\n\nrecording.subject.Subject->recording.Recording.RecordingSession\n\n\n\nrecording.acquisition.SessionStarted\n\n\nrecording.acquisition.SessionStarted\n\n\n\n\nrecording.subject.Subject->recording.acquisition.SessionStarted\n\n\n\nrecording.acquisition.SessionStarted->recording.acquisition.Session\n\n\n\n" }, "metadata": {}, - "execution_count": 17 + "execution_count": 13 } ], "source": [ @@ -378,7 +780,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -389,7 +791,7 @@ ] }, "metadata": {}, - "execution_count": 5 + "execution_count": 14 } ], "source": [ @@ -399,7 +801,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -608,14 +1010,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "imaging = dj.create_virtual_module('imaging', 'u19_imaging')" + ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -626,26 +1030,38 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [ { - "output_type": "execute_result", - "data": { - "text/plain": [ - "*subject_fulln *session_date *session_numbe scan_directory relative_scan_ acquisition_ty\n", - "+------------+ +------------+ +------------+ +------------+ +------------+ +------------+\n", - "emdia_teto6s_1 2021-11-12 0 /mnt/cup/brain /teto6s_12/202 mesoscope \n", - " (Total: 1)" - ], - "text/html": "\n \n \n \n \n
\n \n \n \n\n\n\n\n\n
\n

subject_fullname

\n username_mouse_nickname\n
\n

session_date

\n date of experiment\n
\n

session_number

\n number\n
\n

scan_directory

\n \n
\n

relative_scan_directory

\n \n
\n

acquisition_type

\n # type of acquisition for this scan\n
emdia_teto6s_122021-11-120/mnt/cup/braininit/RigData/mesoscope/imaging/teto6s_12/20211112/teto6s_12/20211112mesoscope
\n \n

Total: 1

\n " - }, - "metadata": {}, - "execution_count": 10 + "output_type": "stream", + "name": "stdout", + "text": [ + "Deleting 3 rows from `u19_imaging`.`__sync_imaging_behavior`\nDeleting 87 rows from `u19_imaging`.`_field_of_view__file`\n" + ] + }, + { + "output_type": "error", + "ename": "DataJointError", + "evalue": "Cannot declare new tables inside a transaction, e.g. from inside a populate/make call", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mDataJointError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_62368/2673820732.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0mimaging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mScan\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mkeyo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36mdelete\u001b[0;34m(self, transaction, safemode)\u001b[0m\n\u001b[1;32m 417\u001b[0m \u001b[0;31m# Cascading delete\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 418\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 419\u001b[0;31m \u001b[0mdelete_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delete_cascade\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 420\u001b[0m \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 421\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtransaction\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36m_delete_cascade\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 383\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 384\u001b[0m \u001b[0mchild\u001b[0m \u001b[0;34m&=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 385\u001b[0;31m \u001b[0mchild\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delete_cascade\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 386\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 387\u001b[0m print(\"Deleting {count} rows from {table}\".format(\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36m_delete_cascade\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 383\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 384\u001b[0m \u001b[0mchild\u001b[0m \u001b[0;34m&=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 385\u001b[0;31m \u001b[0mchild\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_delete_cascade\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 386\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 387\u001b[0m print(\"Deleting {count} rows from {table}\".format(\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36m_delete_cascade\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_attempts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 355\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m \u001b[0mdelete_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdelete_quick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_count\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 357\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mIntegrityError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforeign_key_error_regexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36mdelete_quick\u001b[0;34m(self, get_count)\u001b[0m\n\u001b[1;32m 346\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 347\u001b[0m \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"SELECT ROW_COUNT()\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetchone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mget_count\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m255\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 349\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 350\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36m_log\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 217\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 219\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLog\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatabase\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatabase\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mskip_logging\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtable_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'~'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 220\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 221\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, conn, database, skip_logging)\u001b[0m\n\u001b[1;32m 799\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 800\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_declared\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 801\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdeclare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 802\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdependencies\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 803\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_user\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_user\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.conda/envs/u19_pipeline_python_env/lib/python3.7/site-packages/datajoint/table.py\u001b[0m in \u001b[0;36mdeclare\u001b[0;34m(self, context)\u001b[0m\n\u001b[1;32m 75\u001b[0m \"\"\"\n\u001b[1;32m 76\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_transaction\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 77\u001b[0;31m raise DataJointError('Cannot declare new tables inside a transaction, '\n\u001b[0m\u001b[1;32m 78\u001b[0m 'e.g. from inside a populate/make call')\n\u001b[1;32m 79\u001b[0m \u001b[0msql\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexternal_stores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeclare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfull_table_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefinition\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDataJointError\u001b[0m: Cannot declare new tables inside a transaction, e.g. from inside a populate/make call" + ] } ], "source": [ - "(imaging.Scan & keyo)" + "(imaging.Scan & keyo).delete()" ] }, { @@ -671,10 +1087,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Nos salvamos\nTraceback (most recent call last):\n File \"/tmp/ipykernel_6488/1491040166.py\", line 4, in \n raise Exception('spam', 'eggs')\nException: ('spam', 'eggs')\n\n(, Exception('spam', 'eggs'), )\n" + ] + } + ], + "source": [ + "import traceback\n", + "import sys\n", + "try:\n", + " raise Exception('spam', 'eggs')\n", + "except BaseException as e:\n", + " print('Nos salvamos')\n", + " print(traceback.format_exc())\n" + ] } ], "metadata": {