Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[EEG] Database Architecture for HED Tags #8036

Merged
merged 7 commits into from
Jun 28, 2022
Merged

[EEG] Database Architecture for HED Tags #8036

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8cbc947
DB tables for HED Tags
Mar 3, 2022
82112f4
viz bugfixes events file access
Mar 7, 2022
0d6cef9
make param description nullable
jesscall Mar 9, 2022
aa07a6d
move AssembledHED to physiological_task_event
May 17, 2022
19d70cb
phpcs
Jun 14, 2022
cc07a82
use ImagingFileTypes table
jesscall Jun 21, 2022
1d83aca
use ImagingFileTypes for physio event files
jesscall Jun 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion SQL/0000-00-00-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1715,7 +1715,8 @@ INSERT INTO `parameter_type` (Name, Alias, Type, Description, SourceFrom) VALUES
('fov_dimensions','FieldOfViewDimensions','text','Dimensions of the field of view, in mm. If Field of View Shape (0018,1147) is: RECTANGLE: row dimension followed by column. ROUND: diameter. HEXAGONAL: diameter of a circumscribed circle. DICOM:0018_1149','parameter_file'),
('laterality','Laterality','text','Laterality of (paired) body part examined. Required if the body part examined is a paired structure and Image Laterality (0020,0062) or Frame Laterality (0020,9072) are not sent. DICOM:0020_0060','parameter_file'),
('position_reference_indicator','PositionReferenceIndicator','text','Part of the imaging target used as a reference. DICOM:0020_1040','parameter_file'),
('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file');
('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file'),
('HEDVersion', 'HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file');

CREATE TABLE `parameter_type_category` (
`ParameterTypeCategoryID` int(11) unsigned NOT NULL auto_increment,
Expand Down
93 changes: 89 additions & 4 deletions SQL/0000-00-05-ElectrophysiologyTables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,37 @@ CREATE TABLE `physiological_electrode` (
REFERENCES `physiological_electrode_material` (`PhysiologicalElectrodeMaterialID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


-- Create `physiological_event_file_type` table
CREATE TABLE `physiological_event_file_type` (
`FileType` varchar(20) NOT NULL,
`Description` varchar(255) DEFAULT NULL,
PRIMARY KEY (`FileType`),
UNIQUE KEY `FileType` (`FileType`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_file` table
CREATE TABLE `physiological_event_file` (
`EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`FileType` varchar(20) NOT NULL,
`FilePath` varchar(255) DEFAULT NULL,
`LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`EventFileID`),
KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
KEY `FK_event_file_type` (`FileType`),
CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `physiological_event_file_type` (`FileType`),
CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create physiological_task_event table that will store all information
-- regarding the task executed during the physiological recording
CREATE TABLE `physiological_task_event` (
`PhysiologicalTaskEventID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` INT(10) UNSIGNED NOT NULL,
`EventFileID` INT(10) unsigned NOT NULL,
`InsertTime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`Onset` DECIMAL(11,6) NOT NULL,
`Duration` DECIMAL(11,6) NOT NULL,
Expand All @@ -196,15 +220,70 @@ CREATE TABLE `physiological_task_event` (
`EventType` VARCHAR(50) DEFAULT NULL,
`TrialType` VARCHAR(255) DEFAULT NULL,
`ResponseTime` TIME DEFAULT NULL,
`FilePath` VARCHAR(255) DEFAULT NULL,
PRIMARY KEY (`PhysiologicalTaskEventID`),
KEY `FK_event_file` (`EventFileID`),
CONSTRAINT `FK_phys_file_FileID_4`
FOREIGN KEY (`PhysiologicalFileID`)
REFERENCES `physiological_file` (`PhysiologicalFileID`)
ON DELETE CASCADE
ON DELETE CASCADE,
CONSTRAINT `FK_event_file`
FOREIGN KEY (`EventFileID`)
REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


-- Create `physiological_event_archive` to store event archive info
CREATE TABLE `physiological_event_archive` (
`EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`Blake2bHash` varchar(128) NOT NULL,
`FilePath` varchar(255) NOT NULL,
PRIMARY KEY (`EventArchiveID`),
KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_parameter` to capture all event parameters
-- from events.json
CREATE TABLE `physiological_event_parameter` (
`EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventFileID` int(10) unsigned NOT NULL,
`ParameterName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`LongName` varchar(255) DEFAULT NULL,
`Units` varchar(50) DEFAULT NULL,
`isCategorical` enum('Y', 'N') DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`EventParameterID`),
KEY `FK_event_file_ID` (`EventFileID`),
CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_parameter_category_level` to capture
-- category levels from events.json
CREATE TABLE `physiological_event_parameter_category_level` (
`CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventParameterID` int(10) unsigned NOT NULL,
`LevelName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`CategoricalLevelID`),
KEY `FK_event_param_ID` (`EventParameterID`),
CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_assembled_hed_tag` to store assembled HED Tags
CREATE TABLE `physiological_event_assembled_hed_tag` (
`TaskEventID` int(10) unsigned NOT NULL,
`EventParameterID` int(10) unsigned NOT NULL,
`AssembledHED` text NOT NULL,
PRIMARY KEY (`TaskEventID`, `EventParameterID`),
CONSTRAINT `FK_task_event_ID` FOREIGN KEY (`TaskEventID`) REFERENCES `physiological_task_event` (`PhysiologicalTaskEventID`),
CONSTRAINT `FK_event_parameter_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create physiological_archive which will store archives of all the files for
-- Front-end download
Expand Down Expand Up @@ -426,3 +505,9 @@ INSERT INTO physiological_annotation_label
(22, 'sleep_k-complex', 'sleep K-complex'),
(23, 'scorelabeled', 'a global label indicating that the EEG has been annotated with SCORE.');

-- Insert file types (json, tsv)
INSERT INTO physiological_event_file_type (FileType, Description) VALUES
('json', 'JSON File Type, sidecar for event metadata'),
('tsv', 'TSV File Type, contains information about each event')
;

116 changes: 116 additions & 0 deletions SQL/New_patches/2022-03-03-AddHEDTags.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
-- ############################## CAPTURE HEDVersion ########################## --
-- HEDVersion from dataset_description.json to be added to parameter_type
-- Entry in physiological_parameter_file will be added on dataset import**
INSERT INTO parameter_type (Name, Type, Description, SourceFrom) VALUES
('HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file')
;

-- ############################## HANDLE EVENT FILES ########################## --
-- Create `physiological_event_file_type` table
CREATE TABLE `physiological_event_file_type` (
cmadjar marked this conversation as resolved.
Show resolved Hide resolved
`FileType` varchar(20) NOT NULL,
`Description` varchar(255) DEFAULT NULL,
PRIMARY KEY (`FileType`),
UNIQUE KEY `FileType` (`FileType`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Insert file types (json, tsv)
INSERT INTO physiological_event_file_type (FileType, Description) VALUES
('json', 'JSON File Type, sidecar for event metadata'),
('tsv', 'TSV File Type, contains information about each event')
;

-- Create `physiological_event_file` table
CREATE TABLE `physiological_event_file` (
`EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`FileType` varchar(20) NOT NULL,
`FilePath` varchar(255) DEFAULT NULL,
`LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`EventFileID`),
KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
KEY `FK_event_file_type` (`FileType`),
CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `physiological_event_file_type` (`FileType`),
CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create reference to EventFileID in `physiological_task_event` table
SET FOREIGN_KEY_CHECKS= 0;
ALTER TABLE physiological_task_event
ADD COLUMN `EventFileID` int(10) unsigned NOT NULL AFTER PhysiologicalFileID,
ADD KEY `FK_event_file` (`EventFileID`),
ADD CONSTRAINT `FK_event_file` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
;

-- Insert files into `physiological_event_file` table
INSERT INTO physiological_event_file (PhysiologicalFileID, FilePath, FileType)
SELECT DISTINCT PhysiologicalFileID, FilePath, 'tsv' FROM physiological_task_event
;

-- Update EventFileID reference in `physiological_task_event` table
UPDATE physiological_task_event te
SET EventFileID=(SELECT EventFileID FROM physiological_event_file WHERE PhysiologicalFileID=te.PhysiologicalFileID)
;

-- Delete FilePath column in `physiological_task_event` table
ALTER TABLE physiological_task_event
DROP COLUMN FilePath
;


-- ############################## EVENT FILES ARCHIVE ########################## --
CREATE TABLE `physiological_event_archive` (
`EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`Blake2bHash` varchar(128) NOT NULL,
`FilePath` varchar(255) NOT NULL,
PRIMARY KEY (`EventArchiveID`),
KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;


-- ############################## CAPTURE EVENT PARAMETERS ########################## --
CREATE TABLE `physiological_event_parameter` (
`EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventFileID` int(10) unsigned NOT NULL,
`ParameterName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`LongName` varchar(255) DEFAULT NULL,
`Units` varchar(50) DEFAULT NULL,
`isCategorical` enum('Y', 'N') DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`EventParameterID`),
KEY `FK_event_file_ID` (`EventFileID`),
CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

CREATE TABLE `physiological_event_parameter_category_level` (
`CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventParameterID` int(10) unsigned NOT NULL,
`LevelName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`CategoricalLevelID`),
KEY `FK_event_param_ID` (`EventParameterID`),
CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;


-- ############################## STORE ASSEMBLED HED TAGS ########################## --
CREATE TABLE `physiological_event_assembled_hed_tag` (
jesscall marked this conversation as resolved.
Show resolved Hide resolved
`TaskEventID` int(10) unsigned NOT NULL,
`EventParameterID` int(10) unsigned NOT NULL,
jesscall marked this conversation as resolved.
Show resolved Hide resolved
`AssembledHED` text NOT NULL,
PRIMARY KEY (`TaskEventID`, `EventParameterID`),
CONSTRAINT `FK_task_event_ID` FOREIGN KEY (`TaskEventID`) REFERENCES `physiological_task_event` (`PhysiologicalTaskEventID`),
CONSTRAINT `FK_event_parameter_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
jesscall marked this conversation as resolved.
Show resolved Hide resolved
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

11 changes: 4 additions & 7 deletions modules/electrophysiology_browser/jsx/electrophysiologySessionView.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,13 +206,10 @@ class ElectrophysiologySessionView extends Component {
),
epochsURL:
dbEntry
&& dbEntry.file.downloads.map(
(group) =>
group.links[3]?.file
&& loris.BaseURL
+ '/electrophysiology_browser/file_reader/?file='
+ group.links[3].file
),
&& dbEntry.file?.epochsURL
&& [loris.BaseURL
+ '/electrophysiology_browser/file_reader/?file='
+ dbEntry.file.epochsURL],
electrodesURL:
dbEntry
&& dbEntry.file.downloads.map(
Expand Down
13 changes: 10 additions & 3 deletions modules/electrophysiology_browser/php/sessions.class.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,7 +459,14 @@ class Sessions extends \NDB_Page
// get the links to the files for downloads

$fileSummary['downloads'] = $this->getDownloadLinks($physioFileObj);
$fileSummary['chunks_urls'] = $physioFileObj->getChunksURLs();
$fileSummary['chunks_urls'] = $physioFileObj->getChunksURLs();
$fileSummary['epochsURL'] = $db->pselectOne(
"SELECT FilePath
FROM physiological_event_file
WHERE PhysiologicalFileID=:physioFileID
AND FileType='tsv'",
['physioFileID' => $physioFileID]
);

$fileOutput = $db->pselectone(
'SELECT pot.OutputTypeName
Expand Down Expand Up @@ -548,15 +555,15 @@ class Sessions extends \NDB_Page
$queries = [
'physiological_electrode' => 'physiological_electrode_file',
'physiological_channel' => 'physiological_channel_file',
'physiological_task_event' => 'physiological_task_event_file',
'physiological_event_archive' => 'physiological_event_files',
jesscall marked this conversation as resolved.
Show resolved Hide resolved
'physiological_annotation_archive' => 'physiological_annotation_files',
'physiological_archive' => 'all_files',
];

$labels = [
'physiological_electrode_file' => 'Electrodes',
'physiological_channel_file' => 'Channels',
'physiological_task_event_file' => 'Events',
'physiological_event_files' => 'Events',
'physiological_annotation_files' => 'Annotations',
'all_files' => 'All Files',
];
Expand Down
42 changes: 42 additions & 0 deletions tools/single_use/archive_eeg_event_files.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<?php declare(strict_types=1);
cmadjar marked this conversation as resolved.
Show resolved Hide resolved
require_once __DIR__ . "/../generic_includes.php";

$db = \NDB_Factory::singleton()->database();

$dataDir = $db->pselectOne(
'SELECT Value
FROM Config AS config
INNER JOIN ConfigSettings AS c
ON c.Name=:name AND config.ConfigID=c.ID',
['name' => 'dataDirBasepath']
);

$filepaths = $db->pselect(
"SELECT
DISTINCT(FilePath), PhysiologicalFileID
FROM physiological_event_file
WHERE FileType='tsv'",
[]
);

foreach ($filepaths as $record) {
$tgz_name = str_replace(".tsv", '.tgz', $record['FilePath']);
$tgz_path = $dataDir . $tgz_name;
print_r($tgz_path . "\n");
$eventPath = $dataDir . $record['FilePath'];
$arch_file = new \PharData($tgz_path);
$arch_file->addFile($eventPath, basename($eventPath));

$f = file_get_contents($tgz_path);
$hash = sodium_crypto_generichash($f);

//Update database with hash
$db->insert(
'physiological_event_archive',
[
'PhysiologicalFileID' => $record['PhysiologicalFileID'],
'FilePath' => str_replace($dataDir, '', $tgz_path),
'Blake2bHash' => bin2hex($hash)
]
);
}