Skip to content

[statistics] Recruitment graph optimization #9196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 16, 2024
+144 −117
Merged

[statistics] Recruitment graph optimization #9196

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5553a82
[statistics] Optimize recruitment graph queries
driusan Apr 5, 2024
83001ae
Optimize statistics/php/widgets.class.inc as well
driusan Apr 5, 2024
b42d985
Fix static tests
driusan Apr 5, 2024
2e35cf9
get rid of getTotalRecruitment
driusan Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 80 additions & 43 deletions modules/statistics/php/charts.class.inc
View file
Original file line number Diff line number Diff line change
@@ -111,20 +111,22 @@ class Charts extends \NDB_Page
$user = \NDB_Factory::singleton()->user();
$list_of_sites = $user->getStudySites();

foreach ($list_of_sites as $siteID => $siteName) {
$totalRecruitment = $DB->pselectOne(
"SELECT COUNT(c.CandID)
$data = $DB->pselectColWithIndexKey(
"
SELECT COUNT(c.CandID), c.RegistrationCenterID as CenterID
FROM candidate c
WHERE
c.RegistrationCenterID=:Site AND
c.Active='Y' AND
c.Entity_type='Human'",
['Site' => $siteID]
);
WHERE c.Active='Y' AND c.Entity_type='Human'
GROUP BY c.RegistrationCenterID
",
[],
"CenterID"
);

foreach ($list_of_sites as $siteID => $siteName) {

$recruitmentBySiteData[] = [
"label" => $siteName,
"total" => $totalRecruitment,
"total" => intval($data[$siteID]) ?? 0,
];
}
return new \LORIS\Http\Response\JsonResponse($recruitmentBySiteData);
@@ -142,23 +144,38 @@ class Charts extends \NDB_Page
$user = \NDB_Factory::singleton()->user();
$list_of_sites = $user->getStudySites();

$data = $DB->pselect(
"
SELECT COUNT(c.CandID) as Count,
c.RegistrationCenterID as SiteID,
c.Sex as Sex
FROM candidate c
WHERE c.Active='Y' AND c.Entity_type='Human'
GROUP BY c.RegistrationCenterID, c.Sex",
[]
);
$processed_data = [];
foreach ($data as $row) {
$siteid = $row['SiteID'];
$count = intval($row['Count']);
$sex = strtolower($row['Sex']);
if (!isset($processed_data[$siteid])) {
$processed_data[$siteid] = [
strtolower($sex) => $count
];
} else {
assert(!isset($processed_data[$siteid][$sex]));
$processed_data[$siteid][$sex] = $count;
}
}
foreach ($list_of_sites as $siteID => $siteName) {
$sexData['labels'][] = $siteName;
$sexData['datasets']['female'][] = $DB->pselectOne(
"SELECT COUNT(c.CandID)
FROM candidate c
WHERE c.RegistrationCenterID=:Site
AND c.Sex='female' AND c.Active='Y'
AND c.Entity_type='Human'",
['Site' => $siteID]
);
$sexData['datasets']['male'][] = $DB->pselectOne(
"SELECT COUNT(c.CandID)
FROM candidate c
WHERE c.RegistrationCenterID=:Site AND c.Sex='male' AND c.Active='Y'
AND c.Entity_type='Human'",
['Site' => $siteID]
);

$sexData['datasets']['female'][]
= $processed_data[$siteID]['female'] ?? 0;

$sexData['datasets']['male'][]
= $processed_data[$siteID]['male'] ?? 0;
}
return (new \LORIS\Http\Response\JsonResponse($sexData));
}
@@ -287,6 +304,36 @@ class Charts extends \NDB_Page
$user = \NDB_Factory::singleton()->user();
$list_of_sites = $user->getStudySites();

$recruitment_summary = $DB->pselect(
"SELECT COUNT(c.CandID) as Count,
MONTH(c.Date_registered) as Month,
YEAR(c.Date_registered) as Year,
c.RegistrationCenterID as SiteID
FROM candidate c
WHERE c.Entity_type='Human'
GROUP BY MONTH(c.Date_registered),
YEAR(c.Date_registered),
c.RegistrationCenterID",
[]
);

$recruitmentdata = [];
foreach ($recruitment_summary as $row) {
$siteId = $row['SiteID'];
$year = $row['Year'];
$month = $row['Month'];
if (!isset($recruitmentdata[$siteId])) {
$recruitmentdata[$siteId] = [
$year => [$month => $row['Count']]
];
} else if (!isset($recruitmentdata[$siteId][$year])) {
$recruitmentdata[$siteId][$year] = [$month => $row['Count']];
} else {
assert(!isset($recruitmentdata[$siteId][$year][$month]));
$recruitmentdata[$siteId][$year][$month] = $row['Count'];
}

}
foreach ($list_of_sites as $siteID => $siteName) {
if (!isset($recruitmentData['labels'])) {
continue;
@@ -296,6 +343,7 @@ class Charts extends \NDB_Page
"data" => $this->_getSiteLineRecruitmentData(
$siteID,
$recruitmentData['labels'],
$recruitmentdata,
),
];
}
@@ -333,33 +381,22 @@ class Charts extends \NDB_Page
/**
* Helper to generate the data for the site recruitment line for $siteID.
*
* @param int $siteID The centerID to get data for.
* @param array $labels The list of labels on the chart to fill the data for.
* @param int $siteID The centerID to get data for.
* @param array $labels The list of labels on the chart to fill
* the data for.
* @param array $recruitmentdata The raw recruitment data to split according to
the labels
*
* @return array
*/
private function _getSiteLineRecruitmentData($siteID, $labels)
private function _getSiteLineRecruitmentData($siteID, $labels, $recruitmentdata)
{
$DB = \NDB_Factory::singleton()->database();
$data = [];

foreach ($labels as $label) {
$month = (strlen($label) == 6)
? substr($label, 0, 1) : substr($label, 0, 2);
$year = substr($label, -4, 4);
$data[] = $DB->pselectOne(
"SELECT COUNT(c.CandID)
FROM candidate c
WHERE c.RegistrationCenterID=:Site
AND MONTH(c.Date_registered)=:Month
AND YEAR(c.Date_registered)=:Year
AND c.Entity_type='Human'",
[
'Site' => $siteID,
'Month' => $month,
'Year' => $year,
]
);
$data[] = intval($recruitmentdata[$siteID][$year][$month] ?? "0");
}
return $data;
}
138 changes: 64 additions & 74 deletions modules/statistics/php/widgets.class.inc
View file
Original file line number Diff line number Diff line change
@@ -65,14 +65,25 @@ class Widgets extends \NDB_Page implements ETagCalculator

$recruitmentTarget = $config->getSetting('recruitmentTarget');

$totalScans = $this->_getTotalRecruitment($db);
$recruitment = [
$recruitmentRaw = $db->pselect(
"SELECT
COUNT(*) as Count,
c.Sex as Sex,
c.RegistrationProjectID as ProjectID
FROM candidate c
WHERE c.Active='Y' AND c.Entity_type='Human'
AND c.RegistrationCenterID <> 1
GROUP BY c.Sex, c.RegistrationProjectID",
[]
);
$totalScans = array_sum(array_column($recruitmentRaw, 'Count'));
$recruitment = [
'overall' => $this->_createProjectProgressBar(
'overall',
"Overall Recruitment",
$recruitmentTarget,
$totalScans,
$db,
$recruitmentRaw,
)
];

@@ -89,8 +100,8 @@ class Widgets extends \NDB_Page implements ETagCalculator
$projectID,
$projectInfo['Name'],
$projectInfo['recruitmentTarget'],
$this->getTotalRecruitmentByProject($db, $projectID),
$db
$this->getTotalRecruitmentByProject($recruitmentRaw, $projectID),
$recruitmentRaw
);
}

@@ -108,34 +119,17 @@ class Widgets extends \NDB_Page implements ETagCalculator
return $this->_cache;
}

/**
* Gets the total count of candidates associated with a specific project
*
* @param \Database $DB The database connection to get the count from.
*
* @return int
*/
private function _getTotalRecruitment(\Database $DB): int
{
return $DB->pselectOneInt(
"SELECT COUNT(*) FROM candidate c
WHERE c.Active='Y' AND c.Entity_type='Human'
AND c.RegistrationCenterID <> 1",
[]
) ?? 0;
}

/**
* Generates the template data for a progress bar.
*
* @param string $ID The name of the progress bar being
* created.
* @param string $title The title to add to the template
* variables.
* @param int $recruitmentTarget The target for this recruitment type.
* @param int $totalRecruitment The total number of candidates of all
* types.
* @param \Database $db The database connection to get data from.
* @param string $ID The name of the progress bar being
* created.
* @param string $title The title to add to the template
* variables.
* @param int $recruitmentTarget The target for this recruitment type.
* @param int $totalRecruitment The total number of candidates of all
* types.
* @param array $rawData The raw data from the database
*
* @return array Smarty template data
*/
@@ -144,7 +138,7 @@ class Widgets extends \NDB_Page implements ETagCalculator
$title,
$recruitmentTarget,
$totalRecruitment,
\Database $db
$rawData
): array {
$rv = [
'total_recruitment' => $totalRecruitment,
@@ -156,17 +150,21 @@ class Widgets extends \NDB_Page implements ETagCalculator

$rv['recruitment_target'] = $recruitmentTarget;
if ($ID == 'overall') {
$totalFemales = $this->_getTotalSex($db, "Female");
$totalFemales = $this->_getTotalSex($rawData, "Female");
} else {
$totalFemales = $this->getTotalSexByProject($db, "Female", intval($ID));
$totalFemales = $this->getTotalSexByProject(
$rawData,
"Female",
intval($ID)
);
}
$rv['female_total'] = $totalFemales;
$rv['female_percent']
= round($totalFemales / $recruitmentTarget * 100);
if ($ID == 'overall') {
$totalMales = $this->_getTotalSex($db, "Male");
$totalMales = $this->_getTotalSex($rawData, "Male");
} else {
$totalMales = $this->getTotalSexByProject($db, "Male", intval($ID));
$totalMales = $this->getTotalSexByProject($rawData, "Male", intval($ID));
}
$rv['male_total'] = $totalMales;
$rv['male_percent']
@@ -186,68 +184,60 @@ class Widgets extends \NDB_Page implements ETagCalculator
/**
* Gets the total count of candidates of a specific sex
*
* @param \Database $db A database connection to retrieve information
* from.
* @param string $sex Biological sex (male or female)
* @param array $raw The raw data returned from the SQL query.
* @param string $sex Biological sex (male or female)
*
* @return int
*/
private function _getTotalSex(\Database $db, string $sex) : int
private function _getTotalSex(array $raw, string $sex) : int
{
return $db->pselectOneInt(
"SELECT COUNT(c.CandID)
FROM candidate c
WHERE c.Sex=:sex AND c.Active='Y' AND c.Entity_type='Human'
AND c.RegistrationCenterID <> 1",
['sex' => $sex]
) ?? 0;
$sum = 0;
foreach ($raw as $row) {
if ($row['Sex'] == $sex) {
$sum += intval($row['Count']);
}
}
return $sum;
}

/**
* Gets the total count of candidates of a specific sex,
* associated with a specific project
*
* @param \Database $DB A database connection to retrieve information
* from.
* @param string $sex A biological sex (male or female)
* @param int $projectID Project ID
* @param array $raw The raw data returned from the SQL query.
* @param string $sex A biological sex (male or female)
* @param int $projectID Project ID
*
* @return int
*/
function getTotalSexByProject(\Database $DB, string $sex, int $projectID) : int
function getTotalSexByProject(array $raw, string $sex, int $projectID) : int
{
return $DB->pselectOneInt(
"SELECT COUNT(c.CandID)
FROM candidate c
WHERE c.Sex=:sex AND c.Active='Y' AND c.RegistrationProjectID=:PID
AND c.Entity_type='Human' AND c.RegistrationCenterID <> 1",
[
'sex' => $sex,
'PID' => "$projectID",
]
) ?? 0;
$sum = 0;
foreach ($raw as $row) {
if ($row['Sex'] == $sex && $row['ProjectID'] == $projectID) {
$sum += intval($row['Count']);
}
}
return $sum;
}

/**
* Gets the total count of candidates associated with a specific project.
*
* @param \Database $db A database connection to retrieve information
* from.
* @param int $projectID The Project ID to get recruitment for.
* @param array $data The raw data returned from the SQL query.
* @param int $projectID The Project ID to get recruitment for.
*
* @return int
*/
function getTotalRecruitmentByProject(\Database $db, int $projectID): int
function getTotalRecruitmentByProject(array $data, int $projectID): int
{
return $db->pselectOneInt(
"SELECT COUNT(*)
FROM candidate c
WHERE c.Active='Y'
AND c.RegistrationProjectID=:PID
AND c.Entity_type='Human'
AND c.RegistrationCenterID <> 1",
['PID' => "$projectID"]
) ?? 0;
$sum = 0;
foreach ($data as $row) {
if (intval($row['ProjectID']) == $projectID) {
$sum += intval($row['Count']);
}
}
return $sum;
}

/**