Merge remote-tracking branch 'aces/main' into PetHrrtInsertion_rebase

.gitignore

@@ -0,0 +1,3 @@
+**/__pycache__/*
+*.pyc
+.loris_mri/

INSTALL.Mac.md

@@ -7,7 +7,7 @@ For best results, we recommend installing LORIS on Ubuntu or CentOS.
 This file provides guidance on how to install the imaging pipeline on your Mac computer. LORIS must already be installed.
 
 It has been tested for Mac OS X 10.13. 
-Updates and contributions welcome (also see [Contributing Guide](https://github.com/aces/Loris/blob/master/CONTRIBUTING.md)) 
+Updates and contributions welcome (also see [Contributing Guide](https://github.com/aces/Loris/blob/main/CONTRIBUTING.md)) 
 
 ## Get the code
 
@@ -96,6 +96,26 @@ pip install nilearn
 brew install md5sha1sum
 ```
 
+## Install GNU `grep`
+
+One of the command run in the insertion pipeline uses a functionality of `grep` that
+can only be found in the GNU version of `grep` for Mac. The default installed version
+of `grep` in a Mac install does not include the possibility to search based on a 
+perl regular expression (option `-P`).
+```
+brew install grep
+```
+
+And add the following to the .bash_profile of the user that will be running the 
+pipeline so that the Linux `grep` is used when running the pipeline instead of 
+the Mac `grep`.
+
+```
+# Add path to linux grep instead of the Mac grep as the Mac one does 
+# not have all the functionatities needed by the LORIS imaging pipeline
+PATH="/usr/local/opt/grep/libexec/gnubin:$PATH"
+```
+
 ## Run install script for Mac: imaging_install_MacOSX.sh 
 
 ```

README.md

@@ -26,9 +26,12 @@ See [aces/Loris](https://github.com/aces/loris) README.md for further informatio
    sudo mkdir -p /data/$projectname/bin/mri
    sudo chown -R lorisadmin:lorisadmin /data/$projectname
    cd /data/$projectname/bin
-   git clone -b master https://github.com/aces/Loris-MRI.git mri
    ```
 
+Get the code: Download the latest release from the 
+[releases page](https://github.com/aces/Loris-MRI/releases) 
+and extract it to `/data/$projectname/bin/mri`
+
 #### 2. Install Python 3 with `pip` and `virtualenv`
 
 ```bash
@@ -81,7 +84,7 @@ For the defacing scripts, you will also need to download the pre-compiled `bic-m
   If the imaging install script reports errors in creating directories 
   (due to `/data/` mount permissions), review and manually execute 
   `mkdir/chmod/chown` commands starting at 
-  [imaging_install.sh:L97](https://github.com/aces/Loris-MRI/blob/master/imaging_install.sh#L97)
+  [imaging_install.sh:L97](https://github.com/aces/Loris-MRI/blob/main/imaging_install.sh#L97)
 
   Note: The installer will allow Apache to write to the `/data/` directories by 
   adding user `lorisadmin` to the Apache linux group.  To ensure this change takes 

batch_uploads_tarchive.pl

@@ -240,7 +240,7 @@ =head1 AUTHORS
 
     my $tarchive_path = "$tarchiveLibraryDir/$tarchive";
     my $command = sprintf(
-        "tarchiveLoader.pl -globLocation -profile %s -uploadID %s %s",
+        "tarchiveLoader.pl -profile %s -uploadID %s %s",
         $profile,
         quotemeta($upload_id),
         quotemeta($tarchive_path)

dicom-archive/DICOM/DCMSUM.pm

@@ -327,11 +327,11 @@ QUERY
         if ($invT eq '') {
             $invT = undef;
         }
-        if ($seriesName =~ /ColFA$/i) {
-            $echoT        = undef;    
-            $repT         = undef;
-            $sl_thickness = undef;
-        }
+
+        $echoT        = undef unless ($echoT =~ /\d+/);
+        $repT         = undef unless ($repT =~ /\d+/);
+        $sl_thickness = undef unless ($sl_thickness =~ /\d+/);
+
         if ($modality eq 'MR') {
             my @values = 
               (
@@ -340,7 +340,7 @@ QUERY
                $invT,       $sl_thickness, $phaseEncode, 
                $num,        $seriesUID,    $modality
               );
-            $success = $insert_series->execute(@values);
+	    $success = $insert_series->execute(@values);
         } elsif ($modality eq 'PT') {
             my @values = 
               (

dicom-archive/dicomTar.pl

@@ -199,12 +199,10 @@ =head2 Methods
 my $system          = `uname`;
 
 
-# Remove all files starting with . in the dcm_source directory
-my $cmd = "cd " . $dcm_source . "; find -type f -name '.*' | xargs rm -f";
-system($cmd);
-# Remove __MACOSX directory
-my $cmd = "cd " . $dcm_source . "; find -name '__MACOSX' | xargs rm -rf";
-system($cmd);
+# Remove all files starting with . and __MACOSX in the dcm_source directory
+my @args = ($dcm_source);
+push(@args, qw/-type f -name .* -delete -o -type d -name __MACOSX -exec rm -rf {} +/);
+system('find', @args);
 
 # create new summary object
 my $summary = DICOM::DCMSUM->new($dcm_source,$targetlocation);
@@ -329,7 +327,7 @@ =head2 Methods
 # call the updateMRI_upload script###
 if ($mri_upload_update) {
     my $script =  "updateMRI_Upload.pl"
-                 . " -profile $profile -globLocation -tarchivePath $finalTarget"
+                 . " -profile $profile -tarchivePath $finalTarget"
                  . " -sourceLocation $dcm_source";
     my $output = system($script);
     if ($output!=0)  {

dicom-archive/updateMRI_Upload.pl

@@ -26,10 +26,6 @@ =head1 SYNOPSIS
 =item *
 B<-source_location source_location> : (mandatory) value to set column 
     C<DecompressedLocation> for the newly created record in table C<mri_upload> (see below)
-    
-=item *
-B<-globLocation> : loosen the validity check of the DICOM archive allowing for the 
-     possibility that it was moved to a different directory.
 
 =item *
 B<-verbose> : be verbose
@@ -120,9 +116,6 @@ =head1 AUTHORS
     =~ /: (\d+)\.(\d+)/;
 
 
-my $globArchiveLocation = 0;             # whether to use strict ArchiveLocation strings
-                                         # or to glob them (like '%Loc')
-
 my $Help = <<HELP;
 *******************************************************************************
 
@@ -142,7 +135,7 @@ =head1 AUTHORS
 my $Usage = "------------------------------------------
 $0 updates the mri_upload table to populate the fields
 
-Usage:\n\t $0 -profile <profile> -sourceLocation src -tarchivePath path [-verbose] [-globLocation] [-timeZone tz] 
+Usage:\n\t $0 -profile <profile> -sourceLocation src -tarchivePath path [-verbose] [-timeZone tz]
 \n\n See $0 -help for more info\n\n";
 
 my @arg_table =
@@ -151,9 +144,6 @@ =head1 AUTHORS
 	  ["-profile","string",1, \$profile, "Specify the name of the config file
           which resides in .loris_mri in the current directory."],
 	  ["-verbose", "boolean", 1, \$verbose, "Be verbose."],
-      ["-globLocation", "boolean", 1, \$globArchiveLocation, "Loosen the".
-       " validity check of the tarchive allowing for the possibility that".
-       " the tarchive was moved to a different directory."],
       ["-sourceLocation", "string", 1, \$source_location, "The location".
        " where the uploaded file exists."],
       ["-tarchivePath","string",1, \$tarchive, "The absolute path to".
@@ -242,9 +232,7 @@ =head1 AUTHORS
 
 # Check if there is already an mri upload record for the tarchive
 my $mriUploadOB = NeuroDB::objectBroker::MriUploadOB->new(db => $db);
-my $resultRef = $mriUploadOB->getWithTarchive(
-    GET_COUNT, $tarchive_path, $globArchiveLocation
-);
+my $resultRef = $mriUploadOB->getWithTarchive(GET_COUNT, $tarchive_path);
 
 if($resultRef->[0]->{'COUNT(*)'} > 0) {
    print "\n\tERROR: the tarchive is already uploaded \n\n";
@@ -256,9 +244,7 @@ =head1 AUTHORS
 ################################################################
 
 my $tarchiveOB = NeuroDB::objectBroker::TarchiveOB->new(db => $db);
-$resultRef = $tarchiveOB->getByTarchiveLocation(
-    ['TarchiveID'], $tarchive_path, $globArchiveLocation
-);
+$resultRef = $tarchiveOB->getByTarchiveLocation($tarchive_path);
 
 if(@$resultRef != 1) {
     die sprintf(

docs/02-Install.md

@@ -162,7 +162,7 @@ uploaded through the Imaging Uploader GUI or not, pipeline insertion progress
 can be consulted through a live 'Log Viewer' panel.
 Some settings need to be configured properly (`php.ini` variables, 
 `MRI-Upload Directory` and `ImagingUploader Auto Launch`), and are documented in 
-the [LORIS repository: Imaging Uploader Specification](https://github.com/aces/Loris/blob/master/modules/imaging_uploader/README.md).
+the [LORIS repository: Imaging Uploader Specification](https://github.com/aces/Loris/blob/main/modules/imaging_uploader/README.md).
 
 
 2. **DICOM Archive**
@@ -175,7 +175,7 @@ Patient Name/Patient ID header values are displayed in full, or show up as
 **INVALID-HIDDEN**.
 
 More detailed specifications can be consulted in the 
-[LORIS repository: DICOM Archive Specification](https://github.com/aces/Loris/blob/master/modules/dicom_archive/README.md).
+[LORIS repository: DICOM Archive Specification](https://github.com/aces/Loris/blob/main/modules/dicom_archive/README.md).
 
 
 3. **Imaging Browser**
@@ -189,7 +189,7 @@ some files. Ensure that:
   `Images` settings are set (typically: `/data/$PROJECT/data/`). 
 
 More detailed specifications can be consulted in the 
-[LORIS repository: Imaging Browser Specification](https://github.com/aces/Loris/blob/master/modules/imaging_browser/README.md).
+[LORIS repository: Imaging Browser Specification](https://github.com/aces/Loris/blob/main/modules/imaging_browser/README.md).
 
 4. **Brainbrowser**
 
@@ -203,7 +203,7 @@ images directly from the filesystem. Ensure that:
       proper MINC toolkit path in the `<MINCToolsPath>` tagset.
 
 More detailed specifications can be consulted in the 
-[LORIS repository: Brainbrowser Specification](https://github.com/aces/Loris/blob/master/modules/brainbrowser/README.md).
+[LORIS repository: Brainbrowser Specification](https://github.com/aces/Loris/blob/main/modules/brainbrowser/README.md).
 
 
 5. **MRI Violated Scans**
@@ -217,7 +217,7 @@ Imaging Browser module. Violated scans can be viewed and the type of error
 (scan identification, protocol violation) can be reviewed from the front-end.
 
 More detailed specifications can be consulted in the 
-[LORIS repository: MRI Violated Scans Specification](https://github.com/aces/Loris/blob/master/modules/mri_violations/README.md).
+[LORIS repository: MRI Violated Scans Specification](https://github.com/aces/Loris/blob/main/modules/mri_violations/README.md).
 
 
 6. **Electrophysiology Browser**
@@ -419,6 +419,9 @@ The pipeline to insert BIDS datasets into LORIS currently support the
 following BIDS modalities/entities:
   - Electroencephalography ('eeg')
   - Magnetic Resonance Imaging ('anat', 'func', 'fmap', 'dwi') 
+  - Intracranial electroencephalography ('ieeg')
+      - **Note:** optional [electrical stimulation](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/04-intracranial-electroencephalography.html#electrical-stimulation) files for ieeg modality are currently not supported in LORIS-MRI (or LORIS). 
+      If a BIDS-IEEG directory includes these files, they will be ignored during the import process. 
 
 With slight modifications and further customization, it could handle other 
 types of electrophysiology or imaging modalities.

docs/04-Scripts.md

@@ -105,7 +105,7 @@ encountered cases.
 For cases when a scan has triggered a protocol violation, the MINC volume can be
   **force-loaded** into LORIS by running:
 ```
-uploadNeuroDB/minc_insertion.pl -acquisition_protocol t2w -bypass_extra_file_checks -create_minc_pics -profile prod -globLocation -force  -tarchivePath /data/project/dataTransfer/library/2009/DCM_2009-09-25_project_20110214_185904581.tar -mincPath /data/project/data/trashbin/TarLoad-3-34-pVzGC5/xxx0067_703739_v12_20090925_222403_18e1_mri.mnc
+uploadNeuroDB/minc_insertion.pl -acquisition_protocol t2w -bypass_extra_file_checks -create_minc_pics -profile prod -force  -tarchivePath /data/project/dataTransfer/library/2009/DCM_2009-09-25_project_20110214_185904581.tar -mincPath /data/project/data/trashbin/TarLoad-3-34-pVzGC5/xxx0067_703739_v12_20090925_222403_18e1_mri.mnc
 ```
 
 Note carefully the following arguments:

docs/AppendixA-Troubleshooting_guideline.md

@@ -92,7 +92,7 @@ Error and output messages from the imaging insertion scripts are logged in files
 ***Caveat:*** When the imaging insertion pipeline is auto-launched by the
   Imaging Uploader module, the pipeline scripts' log files are output to
   `/tmp/` and deleted. To avoid deletion, edit the Server Processes Manager
-  function [deleteProcessFiles()](https://github.com/aces/Loris/blob/master/modules/server_processes_manager/php/AbstractServerProcess.class.inc#L521)
+  function [deleteProcessFiles()](https://github.com/aces/Loris/blob/main/modules/server_processes_manager/php/AbstractServerProcess.class.inc#L521)
   to return false instead of true.
 
 ### A.4 Insertion script troubleshooting notes

docs/scripts_md/ImagingUpload.md

@@ -8,6 +8,7 @@ NeuroDB::ImagingUpload -- Provides an interface to the uploaded imaging file
 
     my $imaging_upload = &NeuroDB::ImagingUpload->new(
                            \$dbh,
+                           \$db,
                            $TmpDir_decompressed_folder,
                            $upload_id,
                            $patient_name,
@@ -33,14 +34,15 @@ and updates of the `mri_upload` table according to the upload status.
 
 ## Methods
 
-### new($dbhr, $uploaded\_temp\_folder, $upload\_id, ...) >> (constructor)
+### new($dbhr, $db, $uploaded\_temp\_folder, $upload\_id, ...) >> (constructor)
 
 Creates a new instance of this class. This constructor needs the location of
 the uploaded file. Once the uploaded file has been validated, it will be
 moved to a final destination directory.
 
 INPUTS:
   - $dbhr                : database handler
+  - $db                  : MOOSE database handle
   - $uploaded\_temp\_folder: temporary directory of the upload
   - $upload\_id           : `uploadID` from the `mri_upload` table
   - $pname               : patient name
@@ -147,4 +149,4 @@ License: GPLv3
 
 # AUTHORS
 
-LORIS community <loris.info@mcin.ca> and McGill Centre for Integrative Neuroscience
+LORIS community <loris.info@mcin.ca> and McGill Centre for Integrative Neuroscience

docs/scripts_md/MRI.md

@@ -180,7 +180,7 @@ referenced by `$file_ref`.
 
 INPUT: file hash ref
 
-### findScannerID($manufacturer, $model, $serialNumber, $softwareVersion, $centerID, $dbhr, $register\_new, $db)
+### findScannerID($manufacturer, $model, $serialNumber, $softwareVersion, $centerID, $dbhr, $db)
 
 Finds the scanner ID for the scanner as defined by `$manufacturer`, `$model`,
 `$serialNumber`, `$softwareVersion`, using the database attached to the DBI
@@ -194,7 +194,6 @@ INPUTS:
   - $softwareVersion: scanner's software version
   - $centerID       : scanner's center ID
   - $dbhr           : database handle reference
-  - $register\_new   : if set, will call the function `&registerScanner`
   - $db             : database object
 
 RETURNS: (int) scanner ID

docs/scripts_md/MRIProcessingUtility.md

@@ -12,16 +12,11 @@ utilities
                           $logfile, $LogDir, $verbose
                         );
 
-    %tarchiveInfo     = $utility->createTarchiveArray(
-                          $ArchiveLocation, $globArchiveLocation
-                        );
+    %tarchiveInfo     = $utility->createTarchiveArray($ArchiveLocation);
 
     my ($center_name, $centerID) = $utility->determinePSC(\%tarchiveInfo,0);
 
-    my $scannerID     = $utility->determineScannerID(
-                          \%tarchiveInfo, 0,
-                          $centerID,      $NewScanner
-                        );
+    my $scannerID     = $utility->determineScannerID(\%tarchiveInfo, 0, $centerID);
 
     my $subjectIDsref = $utility->determineSubjectID(
                           $scannerID,
@@ -138,16 +133,16 @@ INPUTS:
 RETURNS: subject's ID hash ref containing `CandID`, `PSCID`, Visit Label 
 and `CandMismatchError` information
 
-### createTarchiveArray($tarchive, $globArchiveLocation)
+### createTarchiveArray($tarchive)
 
-Creates the DICOM archive information hash ref.
+Creates the DICOM archive information hash ref for the tarchive that has the same
+basename as the file path passed as argument.
 
 INPUTS:
-  - $tarchive           : tarchive's path
-  - $globArchiveLocation: globArchiveLocation argument specified when running
-                           the insertion scripts
+  - $tarchive           : tarchive's path (absolute or relative).
 
-RETURNS: DICOM archive information hash ref
+RETURNS: DICOM archive information hash ref if exactly one archive was found. Exits
+         when either no match or multiple matches are found.
 
 ### determinePSC($tarchiveInfo, $to\_log, $upload\_id)
 
@@ -160,16 +155,16 @@ INPUTS:
 
 RETURNS: array of two elements: center name and center ID
 
-### determineScannerID($tarchiveInfo, $to\_log, $centerID, $NewScanner, $upload\_id)
+### determineScannerID($tarchiveInfo, $to\_log, $centerID, $upload\_id)
 
-Determines which scanner ID was used for DICOM acquisitions.
+Determines which scanner ID was used for DICOM acquisitions. Note, if 
+a scanner ID is not already associated to the scanner information found
+in the DICOM headers, then a new scanner will automatically be created.
 
 INPUTS:
   - $tarchiveInfo: archive information hash ref
   - $to\_log      : whether this step should be logged
   - $centerID    : center ID
-  - $NewScanner  : whether a new scanner entry should be created if the scanner
-                   used is a new scanner for the study
   - $upload\_id   : upload ID of the study
 
 RETURNS: scanner ID