[EEG] Database Architecture for HED Tags (#8036)

This contains the necessary changes for adding HED Tag support to the database. There is a single use script that should be run to archive the events files.
aces · Jun 28, 2022 · e781689 · e781689
1 parent 7e46419
commit e781689
Show file tree

Hide file tree

Showing 6 changed files with 214 additions and 14 deletions.
diff --git a/SQL/0000-00-00-schema.sql b/SQL/0000-00-00-schema.sql
@@ -1715,7 +1715,8 @@ INSERT INTO `parameter_type` (Name, Alias, Type, Description, SourceFrom) VALUES
   ('fov_dimensions','FieldOfViewDimensions','text','Dimensions of the field of view, in mm. If Field of View Shape (0018,1147) is: RECTANGLE: row dimension followed by column. ROUND: diameter. HEXAGONAL: diameter of a circumscribed circle. DICOM:0018_1149','parameter_file'),
   ('laterality','Laterality','text','Laterality of (paired) body part examined. Required if the body part examined is a paired structure and Image Laterality (0020,0062) or Frame Laterality (0020,9072) are not sent. DICOM:0020_0060','parameter_file'),
   ('position_reference_indicator','PositionReferenceIndicator','text','Part of the imaging target used as a reference. DICOM:0020_1040','parameter_file'),
-  ('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file');
+  ('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file'),
+  ('HEDVersion', 'HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file');
 
 CREATE TABLE `parameter_type_category` (
   `ParameterTypeCategoryID` int(11) unsigned NOT NULL auto_increment,

diff --git a/SQL/0000-00-05-ElectrophysiologyTables.sql b/SQL/0000-00-05-ElectrophysiologyTables.sql
@@ -180,13 +180,28 @@ CREATE TABLE `physiological_electrode` (
   REFERENCES `physiological_electrode_material` (`PhysiologicalElectrodeMaterialID`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-
+-- Create `physiological_event_file` table
+CREATE TABLE `physiological_event_file` (
+    `EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `PhysiologicalFileID` int(10) unsigned NOT NULL,
+    `FileType` varchar(20) NOT NULL,
+    `FilePath` varchar(255) DEFAULT NULL,
+    `LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    `LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`EventFileID`),
+    KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
+    KEY `FK_event_file_type` (`FileType`),
+    CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `ImagingFileTypes` (`type`),
+    CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
 
 -- Create physiological_task_event table that will store all information
 -- regarding the task executed during the physiological recording
 CREATE TABLE `physiological_task_event` (
   `PhysiologicalTaskEventID` INT(10) UNSIGNED NOT NULL      AUTO_INCREMENT,
   `PhysiologicalFileID`      INT(10) UNSIGNED NOT NULL,
+  `EventFileID`              INT(10) unsigned NOT NULL,
   `InsertTime`               TIMESTAMP        NOT NULL      DEFAULT CURRENT_TIMESTAMP,
   `Onset`                    DECIMAL(11,6)    NOT NULL,
   `Duration`                 DECIMAL(11,6)    NOT NULL,
@@ -196,15 +211,60 @@ CREATE TABLE `physiological_task_event` (
   `EventType`                VARCHAR(50)      DEFAULT NULL,
   `TrialType`                VARCHAR(255)     DEFAULT NULL,
   `ResponseTime`             TIME             DEFAULT NULL,
-  `FilePath`                 VARCHAR(255)     DEFAULT NULL,
+  `AssembledHED`	     TEXT             DEFAULT NULL,
   PRIMARY KEY (`PhysiologicalTaskEventID`),
+  KEY `FK_event_file` (`EventFileID`),
   CONSTRAINT `FK_phys_file_FileID_4`
     FOREIGN KEY (`PhysiologicalFileID`)
     REFERENCES `physiological_file` (`PhysiologicalFileID`)
-    ON DELETE CASCADE
+    ON DELETE CASCADE,
+  CONSTRAINT `FK_event_file` 
+    FOREIGN KEY (`EventFileID`) 
+    REFERENCES `physiological_event_file` (`EventFileID`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
-
+-- Create `physiological_event_archive` to store event archive info
+CREATE TABLE `physiological_event_archive` (
+    `EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `PhysiologicalFileID` int(10) unsigned NOT NULL,
+    `Blake2bHash` varchar(128) NOT NULL,
+    `FilePath` varchar(255) NOT NULL,
+    PRIMARY KEY (`EventArchiveID`),
+    KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
+    CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
+
+-- Create `physiological_event_parameter` to capture all event parameters
+-- from events.json
+CREATE TABLE `physiological_event_parameter` (
+    `EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `EventFileID` int(10) unsigned NOT NULL,
+    `ParameterName` varchar(255) NOT NULL,
+    `Description` text DEFAULT NULL,
+    `LongName` varchar(255) DEFAULT NULL,
+    `Units` varchar(50) DEFAULT NULL,
+    `isCategorical` enum('Y', 'N') DEFAULT NULL,
+    `HED` text DEFAULT NULL,
+    PRIMARY KEY (`EventParameterID`),
+    KEY `FK_event_file_ID` (`EventFileID`),
+    CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
+
+-- Create `physiological_event_parameter_category_level` to capture 
+-- category levels from events.json
+CREATE TABLE `physiological_event_parameter_category_level` (
+    `CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `EventParameterID` int(10) unsigned NOT NULL,
+    `LevelName` varchar(255) NOT NULL,
+    `Description` text DEFAULT NULL,
+    `HED` text DEFAULT NULL,
+    PRIMARY KEY (`CategoricalLevelID`),
+    KEY `FK_event_param_ID` (`EventParameterID`),
+    CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
 
 -- Create physiological_archive which will store archives of all the files for
 -- Front-end download

diff --git a/SQL/New_patches/2022-03-03-AddHEDTags.sql b/SQL/New_patches/2022-03-03-AddHEDTags.sql
@@ -0,0 +1,93 @@
+-- ############################## CAPTURE HEDVersion ########################## --
+-- HEDVersion from dataset_description.json to be added to parameter_type
+-- Entry in physiological_parameter_file will be added on dataset import**
+INSERT INTO parameter_type (Name, Type, Description, SourceFrom) VALUES
+    ('HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file')
+;
+
+-- ############################## HANDLE EVENT FILES ########################## --
+-- Create `physiological_event_file` table
+CREATE TABLE `physiological_event_file` (
+    `EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `PhysiologicalFileID` int(10) unsigned NOT NULL,
+    `FileType` varchar(20) NOT NULL,
+    `FilePath` varchar(255) DEFAULT NULL,
+    `LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+    `LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY (`EventFileID`),
+    KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
+    KEY `FK_event_file_type` (`FileType`),
+    CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `ImagingFileTypes` (`type`),
+    CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
+
+-- Create reference to EventFileID in `physiological_task_event` table
+SET FOREIGN_KEY_CHECKS= 0;
+ALTER TABLE physiological_task_event
+    ADD COLUMN `EventFileID` int(10) unsigned NOT NULL AFTER PhysiologicalFileID,
+    ADD KEY `FK_event_file` (`EventFileID`),
+    ADD CONSTRAINT `FK_event_file` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
+;
+
+-- Create column for assembled HEd tags in `physiological_task_event` table
+ALTER TABLE physiological_task_event
+    ADD COLUMN `AssembledHED` text DEFAULT NULL
+;
+
+-- Insert files into `physiological_event_file` table
+INSERT INTO physiological_event_file (PhysiologicalFileID, FilePath, FileType)
+    SELECT DISTINCT PhysiologicalFileID, FilePath, 'tsv' FROM physiological_task_event
+;
+
+-- Update EventFileID reference in `physiological_task_event` table
+UPDATE physiological_task_event te
+    SET EventFileID=(SELECT EventFileID FROM physiological_event_file WHERE PhysiologicalFileID=te.PhysiologicalFileID)
+;
+
+-- Delete FilePath column in `physiological_task_event` table
+ALTER TABLE physiological_task_event
+    DROP COLUMN FilePath
+;
+
+
+-- ############################## EVENT FILES ARCHIVE ########################## --
+CREATE TABLE `physiological_event_archive` (
+    `EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `PhysiologicalFileID` int(10) unsigned NOT NULL,
+    `Blake2bHash` varchar(128) NOT NULL,
+    `FilePath` varchar(255) NOT NULL,
+    PRIMARY KEY (`EventArchiveID`),
+    KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
+    CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
+
+
+-- ############################## CAPTURE EVENT PARAMETERS ########################## --
+CREATE TABLE `physiological_event_parameter` (
+    `EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `EventFileID` int(10) unsigned NOT NULL,
+    `ParameterName` varchar(255) NOT NULL,
+    `Description` text DEFAULT NULL,
+    `LongName` varchar(255) DEFAULT NULL,
+    `Units` varchar(50) DEFAULT NULL,
+    `isCategorical` enum('Y', 'N') DEFAULT NULL,
+    `HED` text DEFAULT NULL,
+    PRIMARY KEY (`EventParameterID`),
+    KEY `FK_event_file_ID` (`EventFileID`),
+    CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
+
+CREATE TABLE `physiological_event_parameter_category_level` (
+    `CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `EventParameterID` int(10) unsigned NOT NULL,
+    `LevelName` varchar(255) NOT NULL,
+    `Description` text DEFAULT NULL,
+    `HED` text DEFAULT NULL,
+    PRIMARY KEY (`CategoricalLevelID`),
+    KEY `FK_event_param_ID` (`EventParameterID`),
+    CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8
+;
diff --git a/modules/electrophysiology_browser/jsx/electrophysiologySessionView.js b/modules/electrophysiology_browser/jsx/electrophysiologySessionView.js
@@ -206,13 +206,10 @@ class ElectrophysiologySessionView extends Component {
                 ),
             epochsURL:
                 dbEntry
-                && dbEntry.file.downloads.map(
-                  (group) =>
-                    group.links[3]?.file
-                    && loris.BaseURL
-                      + '/electrophysiology_browser/file_reader/?file='
-                      + group.links[3].file
-                ),
+                && dbEntry.file?.epochsURL
+                && [loris.BaseURL
+                  + '/electrophysiology_browser/file_reader/?file='
+                  + dbEntry.file.epochsURL],
             electrodesURL:
                 dbEntry
                 && dbEntry.file.downloads.map(

diff --git a/modules/electrophysiology_browser/php/sessions.class.inc b/modules/electrophysiology_browser/php/sessions.class.inc
@@ -460,6 +460,13 @@ class Sessions extends \NDB_Page
 
         $fileSummary['downloads']   = $this->getDownloadLinks($physioFileObj);
         $fileSummary['chunks_urls'] = $physioFileObj->getChunksURLs();
+        $fileSummary['epochsURL']   = $db->pselectOne(
+            "SELECT FilePath 
+            FROM physiological_event_file
+            WHERE PhysiologicalFileID=:physioFileID 
+            AND FileType='tsv'",
+            ['physioFileID' => $physioFileID]
+        );
 
         $fileOutput = $db->pselectone(
             'SELECT pot.OutputTypeName
@@ -548,15 +555,15 @@ class Sessions extends \NDB_Page
         $queries = [
             'physiological_electrode'          => 'physiological_electrode_file',
             'physiological_channel'            => 'physiological_channel_file',
-            'physiological_task_event'         => 'physiological_task_event_file',
+            'physiological_event_archive'      => 'physiological_event_files',
             'physiological_annotation_archive' => 'physiological_annotation_files',
             'physiological_archive'            => 'all_files',
         ];
 
         $labels = [
             'physiological_electrode_file'   => 'Electrodes',
             'physiological_channel_file'     => 'Channels',
-            'physiological_task_event_file'  => 'Events',
+            'physiological_event_files'      => 'Events',
             'physiological_annotation_files' => 'Annotations',
             'all_files'                      => 'All Files',
         ];

diff --git a/tools/single_use/archive_eeg_event_files.php b/tools/single_use/archive_eeg_event_files.php
@@ -0,0 +1,42 @@
+<?php declare(strict_types=1);
+require_once __DIR__ . "/../generic_includes.php";
+
+$db = \NDB_Factory::singleton()->database();
+
+$dataDir = $db->pselectOne(
+    'SELECT Value
+    FROM Config AS config
+    INNER JOIN ConfigSettings AS c
+    ON c.Name=:name AND config.ConfigID=c.ID',
+    ['name' => 'dataDirBasepath']
+);
+
+$filepaths = $db->pselect(
+    "SELECT
+    DISTINCT(FilePath), PhysiologicalFileID
+    FROM physiological_event_file
+    WHERE FileType='tsv'",
+    []
+);
+
+foreach ($filepaths as $record) {
+    $tgz_name = str_replace(".tsv", '.tgz', $record['FilePath']);
+    $tgz_path = $dataDir . $tgz_name;
+    print_r($tgz_path . "\n");
+    $eventPath = $dataDir . $record['FilePath'];
+    $arch_file = new \PharData($tgz_path);
+    $arch_file->addFile($eventPath, basename($eventPath));
+
+    $f    = file_get_contents($tgz_path);
+    $hash = sodium_crypto_generichash($f);
+
+    //Update database with hash
+    $db->insert(
+        'physiological_event_archive',
+        [
+            'PhysiologicalFileID' => $record['PhysiologicalFileID'],
+            'FilePath'            => str_replace($dataDir, '', $tgz_path),
+            'Blake2bHash'         => bin2hex($hash)
+        ]
+    );
+}