From 7e756398db297c07942b296c9f053d6691878b5f Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Tue, 24 Dec 2019 10:36:35 -0500 Subject: [PATCH 1/3] [Dashboard] Fix study progression performance For long running studies the "study progression" widget on the dashboard was doing an excessive number of queries on the database (one per site per month of the study having been running) which was causing the dashboard to take an excessive amount of time to load. This replaces it with a single query using a GROUP BY, and then massages the data in PHP to the same format. --- modules/dashboard/ajax/get_scan_line_data.php | 101 +++++++----------- 1 file changed, 40 insertions(+), 61 deletions(-) diff --git a/modules/dashboard/ajax/get_scan_line_data.php b/modules/dashboard/ajax/get_scan_line_data.php index 90e1f8ed0ca..010dadb3fbb 100644 --- a/modules/dashboard/ajax/get_scan_line_data.php +++ b/modules/dashboard/ajax/get_scan_line_data.php @@ -40,79 +40,58 @@ WHERE pt.Name='acquisition_date'", array() ); -$scanData['labels'] - = createLineChartLabels($scanStartDate, $scanEndDate); + +// Run a query to get all the data. Order matters to ensure that the labels are calculated +// in the correct order. +$data= $DB->pselect("SELECT s.CenterID, CONCAT(MONTH(pf.Value), '-', YEAR(pf.Value)) as datelabel, COUNT(distinct s.ID) as count + FROM files f + LEFT JOIN parameter_file pf USING (FileID) + LEFT JOIN session s ON (s.ID=f.SessionID) + JOIN parameter_type pt USING (ParameterTypeID) + WHERE pt.Name='acquisition_date' + GROUP BY MONTH(pf.Value), YEAR(pf.Value), s.CenterID + ORDER BY YEAR(pf.Value), MONTH(pf.Value), s.CenterID", + array() + ); + +$scanData['labels'] = createLineChartLabels($data); + $list_of_sites = Utility::getSiteList(true, false); foreach ($list_of_sites as $siteID => $siteName) { $scanData['datasets'][] = array( "name" => $siteName, - "data" => getScanData($siteID, $scanData['labels']), + "data" => getScanData2($data, $siteID, $scanData['labels']) ); } -print json_encode($scanData); - -return 0; +function getScanData(array $data, int $siteID, array $labels) { + $sitedata = array_filter($data, function($row) use ($siteID) { + return $row['CenterID'] == $siteID; + }); + $mappeddata = []; + foreach ($sitedata as $row) { + $mappeddata[$row['datelabel']] = $row['count']; + } -/** - * Create chart labels (dates) - * - * @param string $startDate start date of scans - * @param string $endDate end date of scans - * - * @return array - */ -function createLineChartLabels($startDate, $endDate) -{ - $startDateYear = substr($startDate, 0, 4); - $endDateYear = substr($endDate, 0, 4); - $startDateMonth = substr($startDate, 4, 2); - $endDateMonth = substr($endDate, 4, 2); - $labels = array(); - for ($year = (int)$startDateYear; $year <= (int)$endDateYear; $year++) { - $startMonth = ($year == (int)$startDateYear) ? (int)$startDateMonth : 1; - $endMonth = ($year == (int)$endDateYear) ? (int)$endDateMonth : 12; - for ($month = $startMonth; $month <= $endMonth; $month++) { - $labels[] = $month . "-" . $year; - } + $data = []; + foreach($labels as $i => $label) { + $data[$i] = $mappeddata[$label] ?? 0; } - return $labels; + return $data; } -/** - * Get scan data for each month in the label array - * - * @param string $siteID ID of a site - * @param array $labels chart labels (months to query) - * - * @return array - */ -function getScanData($siteID, $labels) -{ - $DB = Database::singleton(); - $data = array(); - foreach ($labels as $label) { - $month = (strlen($label) == 6) - ? substr($label, 0, 1) : substr($label, 0, 2); - $year = substr($label, -4, 4); - $data[] = $DB->pselectOne( - "SELECT COUNT(distinct s.ID) - FROM files f - LEFT JOIN parameter_file pf USING (FileID) - LEFT JOIN session s ON (s.ID=f.SessionID) - JOIN parameter_type pt USING (ParameterTypeID) - WHERE s.CenterID=:Site - AND pt.Name='acquisition_date' - AND MONTH(pf.Value)=:Month - AND YEAR(pf.Value)=:Year", - array( - 'Site' => $siteID, - 'Month' => $month, - 'Year' => $year, - ) - ); +function createLineChartLabels(array $data) { + // Order was determined by the SQL statement. This should + // ensure that duplicates (for different sites) are stripped + // out. + $labels = []; + foreach ($data as $row) { + $labels[$row['datelabel']] = true; } - return $data; + return array_keys($labels); } +print json_encode($scanData); + +return 0; From b66ab1914a169b304653918758b0c041733fde14 Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Tue, 24 Dec 2019 11:44:58 -0500 Subject: [PATCH 2/3] PHPCS --- modules/dashboard/ajax/get_scan_line_data.php | 69 +++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/modules/dashboard/ajax/get_scan_line_data.php b/modules/dashboard/ajax/get_scan_line_data.php index 010dadb3fbb..13d3506f2ff 100644 --- a/modules/dashboard/ajax/get_scan_line_data.php +++ b/modules/dashboard/ajax/get_scan_line_data.php @@ -25,15 +25,15 @@ $DB = Database::singleton(); -$scanData = array(); -$scanStartDate = $DB->pselectOne( +$scanData = array(); +$scanStartDate = $DB->pselectOne( "SELECT MIN(pf.Value) FROM parameter_file pf JOIN parameter_type pt USING (ParameterTypeID) WHERE pt.Name='acquisition_date'", array() ); -$scanEndDate = $DB->pselectOne( +$scanEndDate = $DB->pselectOne( "SELECT MAX(pf.Value) FROM parameter_file pf JOIN parameter_type pt USING (ParameterTypeID) @@ -41,46 +41,75 @@ array() ); -// Run a query to get all the data. Order matters to ensure that the labels are calculated -// in the correct order. -$data= $DB->pselect("SELECT s.CenterID, CONCAT(MONTH(pf.Value), '-', YEAR(pf.Value)) as datelabel, COUNT(distinct s.ID) as count - FROM files f +// Run a query to get all the data. Order matters to ensure that the labels +// are calculated in the correct order. +$data = $DB->pselect( + "SELECT s.CenterID, + CONCAT(MONTH(pf.Value), '-', YEAR(pf.Value)) as datelabel, + COUNT(distinct s.ID) as count + FROM files f LEFT JOIN parameter_file pf USING (FileID) LEFT JOIN session s ON (s.ID=f.SessionID) JOIN parameter_type pt USING (ParameterTypeID) - WHERE pt.Name='acquisition_date' - GROUP BY MONTH(pf.Value), YEAR(pf.Value), s.CenterID - ORDER BY YEAR(pf.Value), MONTH(pf.Value), s.CenterID", - array() - ); + WHERE pt.Name='acquisition_date' + GROUP BY MONTH(pf.Value), YEAR(pf.Value), s.CenterID + ORDER BY YEAR(pf.Value), MONTH(pf.Value), s.CenterID", + array() +); $scanData['labels'] = createLineChartLabels($data); -$list_of_sites = Utility::getSiteList(true, false); +$list_of_sites = Utility::getSiteList(true, false); foreach ($list_of_sites as $siteID => $siteName) { $scanData['datasets'][] = array( "name" => $siteName, - "data" => getScanData2($data, $siteID, $scanData['labels']) + "data" => getScanData($data, $siteID, $scanData['labels']) ); } -function getScanData(array $data, int $siteID, array $labels) { - $sitedata = array_filter($data, function($row) use ($siteID) { - return $row['CenterID'] == $siteID; - }); +/** + * Massages the data passed into the format expected by C3. + * Extracts the data for $siteID from $data, and ensures + * that each label has a value in the correct index of the + * returned array. Labels not in $data for site are given + * a value of 0. + * + * @param array $data The data to be extracted from + * @param int $siteID The siteID to be extracted + * @param array $labels A list of labels that should appear + * in the result in the correct order. + * + * @return array The data for $siteID massaged into the correct form. + */ +function getScanData(array $data, int $siteID, array $labels) +{ + $sitedata = array_filter( + $data, + function ($row) use ($siteID) { + return $row['CenterID'] == $siteID; + } + ); $mappeddata = []; foreach ($sitedata as $row) { $mappeddata[$row['datelabel']] = $row['count']; } $data = []; - foreach($labels as $i => $label) { + foreach ($labels as $i => $label) { $data[$i] = $mappeddata[$label] ?? 0; } return $data; } -function createLineChartLabels(array $data) { +/** + * Extract all month labels that are in $data. + * + * @param array $data The data to extract labels from + * + * @return array An array of date labels that appear in the data. + */ +function createLineChartLabels(array $data) +{ // Order was determined by the SQL statement. This should // ensure that duplicates (for different sites) are stripped // out. From 67a698758d51a9a7d36b78e13d8d4f8396f20dd9 Mon Sep 17 00:00:00 2001 From: Dave MacFarlane Date: Mon, 6 Jan 2020 10:48:13 -0500 Subject: [PATCH 3/3] update comment to be more explicit --- modules/dashboard/ajax/get_scan_line_data.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/dashboard/ajax/get_scan_line_data.php b/modules/dashboard/ajax/get_scan_line_data.php index 13d3506f2ff..e93aa4a8a7c 100644 --- a/modules/dashboard/ajax/get_scan_line_data.php +++ b/modules/dashboard/ajax/get_scan_line_data.php @@ -68,14 +68,14 @@ } /** - * Massages the data passed into the format expected by C3. - * Extracts the data for $siteID from $data, and ensures - * that each label has a value in the correct index of the - * returned array. Labels not in $data for site are given - * a value of 0. + * Massages the data passed into the format expected by the + * C3 library. Extracts the data for $siteID from $data, and + * ensures that each label has a value in the correct index + * of the returned array. Labels not in $data for site are + * given a value of 0. * - * @param array $data The data to be extracted from - * @param int $siteID The siteID to be extracted + * @param array $data The data to be extracted from. + * @param int $siteID The siteID to be extracted. * @param array $labels A list of labels that should appear * in the result in the correct order. *