Merge branch 'main' into 2024_02_01_eeg_revised_filters

jeffersoncasimir · Feb 27, 2024 · 791c819 · 791c819
2 parents 909aa3b + 1112e1a
commit 791c819
Show file tree

Hide file tree

Showing 102 changed files with 2,924 additions and 2,147 deletions.
diff --git a/.github/workflows/loristest.yml b/.github/workflows/loristest.yml
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        php: ['8.1', '8.2']
+        php: ['8.1', '8.2', '8.3']
     steps:
       - uses: actions/checkout@v2
 
@@ -104,7 +104,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        php: ['8.1', '8.2']
+        php: ['8.1', '8.2', '8.3']
         apiversion: ['v0.0.3', 'v0.0.4-dev']
     steps:
     - uses: actions/checkout@v2
@@ -167,9 +167,8 @@ jobs:
         mysql ${{ env.DB_DATABASE}} -uroot -proot < SQL/0000-00-03-ConfigTables.sql
         mysql ${{ env.DB_DATABASE}} -uroot -proot < SQL/0000-00-04-Help.sql
         mysql ${{ env.DB_DATABASE}} -uroot -proot < SQL/0000-00-05-ElectrophysiologyTables.sql
-        find raisinbread/instruments/instrument_sql -name *.sql -exec sh -c "echo Sourcing {}; mysql ${{ env.DB_DATABASE}} -uroot -proot < {}" \;
+        find raisinbread/instruments/instrument_sql -name *.sql -not -name 9999-99-99-drop_instrument_tables.sql -exec sh -c "echo Sourcing {}; mysql ${{ env.DB_DATABASE}} -uroot -proot < {}" \;
         find raisinbread/RB_files/ -name *.sql -exec sh -c "echo Sourcing {}; mysql ${{ env.DB_DATABASE}} -uroot -proot < {}" \;
-
     - name: Source instrument schemas
       run: |
         find raisinbread/instruments/instrument_sql -name 0000-*.sql -exec sh -c "echo Sourcing {}; mysql ${{ env.DB_DATABASE}} -uroot -proot < {}" \;
@@ -210,7 +209,7 @@ jobs:
         fail-fast: false
         matrix:
             testsuite: ['integration']
-            php: ['8.1','8.2']
+            php: ['8.1','8.2', '8.3']
             ci_node_index: [0,1,2,3]
 
             include:
@@ -221,10 +220,14 @@ jobs:
               php: '8.1'
             - testsuite: 'static'
               php: '8.2'
+            - testsuite: 'static'
+              php: '8.3'
             - testsuite: 'unit'
               php: '8.1'
             - testsuite: 'unit'
               php: '8.2'
+            - testsuite: 'unit'
+              php: '8.3'
 
     steps:
     - uses: actions/checkout@v2

diff --git a/.phan/config.php b/.phan/config.php
@@ -38,7 +38,7 @@
         "php",
         "htdocs",
         "modules",
-        "src",
+	"src",
         "vendor",
         "test"
     ],

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,9 @@ changes in the following format: PR #1234***
 #### Features
 - Add OpenID Connect authorization support to LORIS (PR #8255)
 
+#### Updates and Improvements
+- Create new `sex` table to hold candidate sex options, and change Sex and ProbandSex columns of `candidate` table to a varchar(255) datatype that is restricted by the `sex` table (PR #9025)
+
 #### Bug Fixes
 - Fix examiner site display (PR #8967)
 - bvl_feedback updates in real-time (PR #8966)

diff --git a/Dockerfile.test.db b/Dockerfile.test.db
@@ -1,4 +1,4 @@
-FROM mysql:5.7
+FROM mariadb:10.5
 
 ARG BASE_DIR
 

diff --git a/SQL/0000-00-00-schema.sql b/SQL/0000-00-00-schema.sql
@@ -72,6 +72,13 @@ CREATE TABLE `language` (
 INSERT INTO language (language_code, language_label) VALUES
     ('en-CA', 'English');
 
+CREATE TABLE `sex` (
+  `Name` varchar(255) NOT NULL,
+  PRIMARY KEY `Name` (`Name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Stores sex options available for candidates in LORIS';
+
+INSERT INTO sex (Name) VALUES ('Male'), ('Female'), ('Other');
+
 CREATE TABLE `users` (
   `ID` int(10) unsigned NOT NULL auto_increment,
   `UserID` varchar(255) NOT NULL default '',
@@ -151,7 +158,7 @@ CREATE TABLE `candidate` (
   `DoB` date DEFAULT NULL,
   `DoD` date DEFAULT NULL,
   `EDC` date DEFAULT NULL,
-  `Sex` enum('Male','Female','Other') DEFAULT NULL,
+  `Sex` varchar(255) DEFAULT NULL,
   `RegistrationCenterID` integer unsigned NOT NULL DEFAULT '0',
   `RegistrationProjectID` int(10) unsigned NOT NULL,
   `Ethnicity` varchar(255) DEFAULT NULL,
@@ -166,7 +173,7 @@ CREATE TABLE `candidate` (
   `flagged_other_status` enum('not_answered') DEFAULT NULL,
   `Testdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   `Entity_type` enum('Human','Scanner') NOT NULL DEFAULT 'Human',
-  `ProbandSex` enum('Male','Female','Other') DEFAULT NULL,
+  `ProbandSex` varchar(255) DEFAULT NULL,
   `ProbandDoB` date DEFAULT NULL,
   PRIMARY KEY (`CandID`),
   UNIQUE KEY `ID` (`ID`),
@@ -175,9 +182,13 @@ CREATE TABLE `candidate` (
   KEY `CandidateActive` (`Active`),
   KEY `FK_candidate_2_idx` (`flagged_reason`),
   KEY `PSCID` (`PSCID`),
+  KEY `FK_candidate_sex_1` (`Sex`),
+  KEY `FK_candidate_sex_2` (`ProbandSex`),
   CONSTRAINT `FK_candidate_1` FOREIGN KEY (`RegistrationCenterID`) REFERENCES `psc` (`CenterID`),
   CONSTRAINT `FK_candidate_2` FOREIGN KEY (`flagged_reason`) REFERENCES `caveat_options` (`ID`) ON DELETE RESTRICT ON UPDATE CASCADE,
-  CONSTRAINT `FK_candidate_RegistrationProjectID` FOREIGN KEY (`RegistrationProjectID`) REFERENCES `Project` (`ProjectID`) ON UPDATE CASCADE
+  CONSTRAINT `FK_candidate_RegistrationProjectID` FOREIGN KEY (`RegistrationProjectID`) REFERENCES `Project` (`ProjectID`) ON UPDATE CASCADE,
+  CONSTRAINT `FK_candidate_sex_1` FOREIGN KEY (`Sex`) REFERENCES `sex` (`Name`) ON DELETE RESTRICT ON UPDATE CASCADE,
+  CONSTRAINT `FK_candidate_sex_2` FOREIGN KEY (`ProbandSex`) REFERENCES `sex` (`Name`) ON DELETE RESTRICT ON UPDATE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 CREATE TABLE `session` (

diff --git a/SQL/0000-00-05-ElectrophysiologyTables.sql b/SQL/0000-00-05-ElectrophysiologyTables.sql
@@ -382,105 +382,6 @@ CREATE TABLE `physiological_archive` (
     ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
--- SQL tables for BIDS derivative file structure
--- Create physiological_annotation_file_type table
-CREATE TABLE `physiological_annotation_file_type` (
-    `FileType`        VARCHAR(20)   NOT NULL UNIQUE,
-    `Description`     VARCHAR(255),
-    PRIMARY KEY (`FileType`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create physiological_annotation_file table
-CREATE TABLE `physiological_annotation_file` (
-    `AnnotationFileID`    INT(10)      UNSIGNED NOT NULL AUTO_INCREMENT,
-    `PhysiologicalFileID` INT(10)      UNSIGNED NOT NULL,
-    `FileType`            VARCHAR(20)  NOT NULL,
-    `FilePath`            VARCHAR(255),
-    `LastUpdate`          TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
-    `LastWritten`         TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    PRIMARY KEY (`AnnotationFileID`),
-    CONSTRAINT `FK_phys_file_ID`
-        FOREIGN KEY (`PhysiologicalFileID`)
-        REFERENCES `physiological_file` (`PhysiologicalFileID`),
-    CONSTRAINT `FK_annotation_file_type`
-        FOREIGN KEY (`FileType`)
-        REFERENCES `physiological_annotation_file_type` (`FileType`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create annotation_archive which will store archives of all the annotation files for
--- Front-end download
-CREATE TABLE `physiological_annotation_archive` (
-  `AnnotationArchiveID` INT(10) UNSIGNED NOT NULL   AUTO_INCREMENT,
-  `PhysiologicalFileID` INT(10) UNSIGNED NOT NULL,
-  `Blake2bHash`         VARCHAR(128)     NOT NULL,
-  `FilePath`            VARCHAR(255)     NOT NULL,
-  PRIMARY KEY (`AnnotationArchiveID`),
-  CONSTRAINT `FK_physiological_file_ID`
-    FOREIGN KEY (`PhysiologicalFileID`)
-    REFERENCES `physiological_file` (`PhysiologicalFileID`)
-    ON DELETE CASCADE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create annotation_parameter table
--- Note: This corresponds with the JSON annotation files
-CREATE TABLE `physiological_annotation_parameter` (
-    `AnnotationParameterID` INT(10)      UNSIGNED NOT NULL AUTO_INCREMENT,
-    `AnnotationFileID`      INT(10)      UNSIGNED NOT NULL,
-    `Description`           TEXT         DEFAULT NULL,
-    `Sources`               VARCHAR(255),
-    `Author`                VARCHAR(255),
-    PRIMARY KEY (`AnnotationParameterID`),
-    CONSTRAINT `FK_annotation_file_ID`
-        FOREIGN KEY (`AnnotationFileID`)
-        REFERENCES `physiological_annotation_file` (`AnnotationFileID`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create an annotation_label_type table
-CREATE TABLE `physiological_annotation_label` (
-  `AnnotationLabelID`    INT(5)       UNSIGNED NOT NULL AUTO_INCREMENT,
-  `AnnotationFileID`     INT(10)      UNSIGNED DEFAULT NULL,
-  `LabelName`            VARCHAR(255) NOT NULL,
-  `LabelDescription`     TEXT         DEFAULT NULL,
-  PRIMARY KEY (`AnnotationLabelID`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create annotation_tsv table
--- Note: This corresponds with the .tsv annotation files
-CREATE TABLE `physiological_annotation_instance` (
-    `AnnotationInstanceID`  INT(10)         UNSIGNED NOT NULL AUTO_INCREMENT,
-    `AnnotationFileID`      INT(10)         UNSIGNED NOT NULL,
-    `AnnotationParameterID` INT(10)         UNSIGNED NOT NULL,
-    `Onset`                 DECIMAL(10, 4),
-    `Duration`              DECIMAL(10, 4)  DEFAULT 0,
-    `AnnotationLabelID`     INT(5)          UNSIGNED NOT NULL,
-    `Channels`              TEXT,
-    `AbsoluteTime`          TIMESTAMP,
-    `Description`           VARCHAR(255),
-    PRIMARY KEY (`AnnotationInstanceID`),
-    CONSTRAINT `FK_annotation_parameter_ID`
-        FOREIGN KEY (`AnnotationParameterID`)
-        REFERENCES `physiological_annotation_parameter` (`AnnotationParameterID`),
-    CONSTRAINT `FK_annotation_file`
-        FOREIGN KEY (`AnnotationFileID`)
-        REFERENCES `physiological_annotation_file` (`AnnotationFileID`),
-    CONSTRAINT `FK_annotation_label_ID`
-        FOREIGN KEY (`AnnotationLabelID`)
-        REFERENCES `physiological_annotation_label` (`AnnotationLabelID`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
--- Create physiological_annotation_rel table
-CREATE TABLE `physiological_annotation_rel` (
-    `AnnotationTSV`  INT(10)    UNSIGNED NOT NULL,
-    `AnnotationJSON` INT(10)    UNSIGNED NOT NULL,
-    PRIMARY KEY (`AnnotationTSV`, `AnnotationJSON`),
-    CONSTRAINT `FK_AnnotationTSV`
-        FOREIGN KEY (`AnnotationTSV`)
-        REFERENCES `physiological_annotation_file` (`AnnotationFileID`),
-    CONSTRAINT `FK_AnnotationJSON`
-        FOREIGN KEY (`AnnotationJSON`)
-        REFERENCES `physiological_annotation_file` (`AnnotationFileID`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-
 -- Create EEG upload table
 CREATE TABLE `electrophysiology_uploader` (
     `UploadID` int(10) unsigned NOT NULL AUTO_INCREMENT,
@@ -634,39 +535,3 @@ INSERT INTO ImagingFileTypes
   ('edf',  'European data format (EEG)'),
   ('cnt',  'Neuroscan CNT data format (EEG)'),
   ('archive', 'Archive file');
-
--- Insert into annotation_file_type
-INSERT INTO physiological_annotation_file_type
-    (FileType, Description)
-    VALUES
-    ('tsv',  'TSV File Type, contains information about each annotation'),
-    ('json', 'JSON File Type, metadata for annotations');
-
--- Insert into annotation_label_type
-INSERT INTO physiological_annotation_label
-    (AnnotationLabelID, LabelName, LabelDescription)
-    VALUES
-    (1,  'artifact',            'artifactual data'),
-    (2,  'motion',              'motion related artifact'),
-    (3,  'flux_jump',           'artifactual data due to flux jump'),
-    (4,  'line_noise',          'artifactual data due to line noise (e.g., 50Hz)'),
-    (5,  'muscle',              'artifactual data due to muscle activity'),
-    (6,  'epilepsy_interictal', 'period deemed interictal'),
-    (7,  'epilepsy_preictal',   'onset of preictal state prior to onset of epilepsy'),
-    (8,  'epilepsy_seizure',    'onset of epilepsy'),
-    (9,  'epilepsy_postictal',  'postictal seizure period'),
-    (10, 'epileptiform',        'unspecified epileptiform activity'),
-    (11, 'epileptiform_single', 'a single epileptiform graphoelement (including possible slow wave)'),
-    (12, 'epileptiform_run',    'a run of one or more epileptiform graphoelements'),
-    (13, 'eye_blink',           'Eye blink'),
-    (14, 'eye_movement',        'Smooth Pursuit / Saccadic eye movement'),
-    (15, 'eye_fixation',        'Fixation onset'),
-    (16, 'sleep_N1',            'sleep stage N1'),
-    (17, 'sleep_N2',            'sleep stage N2'),
-    (18, 'sleep_N3',            'sleep stage N3'),
-    (19, 'sleep_REM',           'REM sleep'),
-    (20, 'sleep_wake',          'sleep stage awake'),
-    (21, 'sleep_spindle',       'sleep spindle'),
-    (22, 'sleep_k-complex',     'sleep K-complex'),
-    (23, 'scorelabeled',        'a global label indicating that the EEG has been annotated with SCORE.');
-
diff --git a/SQL/New_patches/2024-01-29-Physiological-Events-Replace-Annotations.sql b/SQL/New_patches/2024-01-29-Physiological-Events-Replace-Annotations.sql
@@ -0,0 +1,17 @@
+-- Dropping all tables regarding annotations
+DROP TABLE physiological_annotation_archive;
+DROP TABLE physiological_annotation_rel;
+DROP TABLE physiological_annotation_instance;
+DROP TABLE physiological_annotation_parameter;
+DROP TABLE physiological_annotation_label;
+DROP TABLE physiological_annotation_file;
+DROP TABLE physiological_annotation_file_type;
+
+-- Event files are always associated to Projects, sometimes exclusively (dataset-scope events.json files)
+-- Add ProjectID and make PhysiologicalFileID DEFAULT NULL (ProjectID should ideally not be NULLable)
+ALTER TABLE `physiological_event_file`
+  CHANGE `PhysiologicalFileID` `PhysiologicalFileID` int(10) unsigned DEFAULT NULL,
+  ADD COLUMN `ProjectID` int(10) unsigned DEFAULT NULL AFTER `PhysiologicalFileID`,
+  ADD KEY `FK_physiological_event_file_project_id` (`ProjectID`),
+  ADD CONSTRAINT `FK_physiological_event_file_project_id`
+    FOREIGN KEY (`ProjectID`) REFERENCES `Project` (`ProjectID`);
diff --git a/SQL/New_patches/2024-01-29-create-sex-table.sql b/SQL/New_patches/2024-01-29-create-sex-table.sql
@@ -0,0 +1,14 @@
+CREATE TABLE `sex` (
+  `Name` varchar(255) NOT NULL,
+  PRIMARY KEY `Name` (`Name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Stores sex options available for candidates in LORIS';
+
+INSERT INTO sex (Name) VALUES ('Male'), ('Female'), ('Other');
+
+ALTER TABLE candidate
+  MODIFY COLUMN sex varchar(255) DEFAULT NULL,
+  MODIFY COLUMN ProbandSex varchar(255) DEFAULT NULL,
+  ADD KEY `FK_candidate_sex_1` (`Sex`),
+  ADD KEY `FK_candidate_sex_2` (`ProbandSex`),
+  ADD CONSTRAINT `FK_candidate_sex_1` FOREIGN KEY (`Sex`) REFERENCES `sex` (`Name`) ON DELETE RESTRICT ON UPDATE CASCADE,
+  ADD CONSTRAINT `FK_candidate_sex_2` FOREIGN KEY (`ProbandSex`) REFERENCES `sex` (`Name`) ON DELETE RESTRICT ON UPDATE CASCADE;
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,7 +38,7 @@ @@
             "php",
             "htdocs",
             "modules",
-            "src",
+    	"src",
             "vendor",
             "test"
         ],
@@ Expand Down @@