Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[HGCAL] Fix the implementation of scaling test in HGCalRecHitProducers #47269

Merged
merged 1 commit into from
Feb 5, 2025
Merged

[HGCAL] Fix the implementation of scaling test in HGCalRecHitProducers #47269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitCalibrationAlgorithms.dev.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,6 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
HGCalCalibParamDevice const& device_calib,
HGCalConfigParamDevice const& device_config) const {
LogDebug("HGCalRecHitCalibrationAlgorithms") << "\n\nINFO -- Start of calibrate\n\n" << std::endl;
LogDebug("HGCalRecHitCalibrationAlgorithms")
<< "N blocks: " << n_blocks_ << "\tN threads: " << n_threads_ << std::endl;
auto grid = make_workdiv<Acc1D>(n_blocks_, n_threads_);

LogDebug("HGCalRecHitCalibrationAlgorithms") << "\n\nINFO -- Copying the digis to the device\n\n" << std::endl;
HGCalDigiDevice device_digis(host_digis.view().metadata().size(), queue);
Expand All @@ -134,6 +131,16 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
<< "\n\nINFO -- Allocating rechits buffer and initiating values" << std::endl;
HGCalRecHitDevice device_recHits(device_digis.view().metadata().size(), queue);

// number of items per group
uint32_t items = n_threads_;
// use as many groups as needed to cover the whole problem
uint32_t groups = divide_up_by(device_digis.view().metadata().size(), items);
// map items to
// - threads with a single element per thread on a GPU backend
// - elements within a single thread on a CPU backend
auto grid = make_workdiv<Acc1D>(groups, items);
LogDebug("HGCalRecHitCalibrationAlgorithms") << "N groups: " << groups << "\tN items: " << items << std::endl;

alpaka::exec<Acc1D>(queue,
grid,
HGCalRecHitCalibrationKernel_flagRecHits{},
Expand Down
73 changes: 59 additions & 14 deletions RecoLocalCalo/HGCalRecAlgos/plugins/alpaka/HGCalRecHitProducers.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
#include "CondFormats/HGCalObjects/interface/alpaka/HGCalCalibrationParameterDevice.h"
#include "RecoLocalCalo/HGCalRecAlgos/interface/alpaka/HGCalRecHitCalibrationAlgorithms.h"

// flag to assist the computational performance test
// #define HGCAL_PERF_TEST

namespace ALPAKA_ACCELERATOR_NAMESPACE {

using namespace cms::alpakatools;
Expand All @@ -47,11 +50,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
edm::ESWatcher<HGCalElectronicsMappingRcd> calibWatcher_;
edm::ESWatcher<HGCalModuleConfigurationRcd> configWatcher_;
const edm::EDGetTokenT<hgcaldigi::HGCalDigiHost> digisToken_;
device::ESGetToken<hgcalrechit::HGCalCalibParamDevice, HGCalModuleConfigurationRcd> calibToken_;
edm::ESGetToken<hgcalrechit::HGCalCalibParamHost, HGCalModuleConfigurationRcd> calibToken_;
device::ESGetToken<hgcalrechit::HGCalConfigParamDevice, HGCalModuleConfigurationRcd> configToken_;
const device::EDPutToken<hgcalrechit::HGCalRecHitDevice> recHitsToken_;
const HGCalRecHitCalibrationAlgorithms calibrator_;
int n_hits_scale;
const int n_hits_scale;
};

HGCalRecHitsProducer::HGCalRecHitsProducer(const edm::ParameterSet& iConfig)
Expand All @@ -61,7 +64,13 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
configToken_{esConsumes(iConfig.getParameter<edm::ESInputTag>("configSource"))},
recHitsToken_{produces()},
calibrator_{iConfig.getParameter<int>("n_blocks"), iConfig.getParameter<int>("n_threads")},
n_hits_scale{iConfig.getParameter<int>("n_hits_scale")} {}
n_hits_scale{iConfig.getParameter<int>("n_hits_scale")} {
#ifndef HGCAL_PERF_TEST
if (n_hits_scale > 1) {
throw cms::Exception("RuntimeError") << "Build with `HGCAL_PERF_TEST` flag to activate `n_hits_scale`.";
}
#endif
}

void HGCalRecHitsProducer::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
edm::ParameterSetDescription desc;
Expand All @@ -78,7 +87,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
auto& queue = iEvent.queue();

// Read digis
auto const& deviceCalibParamProvider = iSetup.getData(calibToken_);
auto const& hostCalibParamProvider = iSetup.getData(calibToken_);
auto const& deviceConfigParamProvider = iSetup.getData(configToken_);
auto const& hostDigisIn = iEvent.get(digisToken_);

Expand All @@ -87,28 +96,53 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
if (calibWatcher_.check(iSetup)) {
for (int i = 0; i < deviceConfigParamProvider.view().metadata().size(); i++) {
log << "idx = " << i << ", "
<< "gain = " << deviceConfigParamProvider.view()[i].gain() << ","
<< "ADC_ped = " << deviceCalibParamProvider.view()[i].ADC_ped() << ", "
<< "CM_slope = " << deviceCalibParamProvider.view()[i].CM_slope() << ", "
<< "CM_ped = " << deviceCalibParamProvider.view()[i].CM_ped() << ", "
<< "BXm1_slope = " << deviceCalibParamProvider.view()[i].BXm1_slope() << ", ";
<< "gain = " << deviceConfigParamProvider.view()[i].gain() << ", ";
}
for (int i = 0; i < hostCalibParamProvider.view().metadata().size(); i++) {
log << "idx = " << i << ", "
<< "ADC_ped = " << hostCalibParamProvider.view()[i].ADC_ped() << ", "
<< "CM_slope = " << hostCalibParamProvider.view()[i].CM_slope() << ", "
<< "CM_ped = " << hostCalibParamProvider.view()[i].CM_ped() << ", "
<< "BXm1_slope = " << hostCalibParamProvider.view()[i].BXm1_slope() << ", ";
}
}
});

int oldSize = hostDigisIn.view().metadata().size();
int newSize = oldSize * n_hits_scale;
#ifdef HGCAL_PERF_TEST
uint32_t oldSize = hostDigisIn.view().metadata().size();
uint32_t newSize = oldSize * (n_hits_scale > 0 ? (unsigned)n_hits_scale : 1);
auto hostDigis = HGCalDigiHost(newSize, queue);
auto hostCalibParam = HGCalCalibParamHost(newSize, queue);
// TODO: replace with memcp ?
for (int i = 0; i < newSize; i++) {
for (uint32_t i = 0; i < newSize; i++) {
hostDigis.view()[i].tctp() = hostDigisIn.view()[i % oldSize].tctp();
hostDigis.view()[i].adcm1() = hostDigisIn.view()[i % oldSize].adcm1();
hostDigis.view()[i].adc() = hostDigisIn.view()[i % oldSize].adc();
hostDigis.view()[i].tot() = hostDigisIn.view()[i % oldSize].tot();
hostDigis.view()[i].toa() = hostDigisIn.view()[i % oldSize].toa();
hostDigis.view()[i].cm() = hostDigisIn.view()[i % oldSize].cm();
hostDigis.view()[i].flags() = hostDigisIn.view()[i % oldSize].flags();

hostCalibParam.view()[i].ADC_ped() = hostCalibParamProvider.view()[i % oldSize].ADC_ped();
hostCalibParam.view()[i].Noise() = hostCalibParamProvider.view()[i % oldSize].Noise();
hostCalibParam.view()[i].CM_slope() = hostCalibParamProvider.view()[i % oldSize].CM_slope();
hostCalibParam.view()[i].CM_ped() = hostCalibParamProvider.view()[i % oldSize].CM_ped();
hostCalibParam.view()[i].BXm1_slope() = hostCalibParamProvider.view()[i % oldSize].BXm1_slope();
hostCalibParam.view()[i].TOTtoADC() = hostCalibParamProvider.view()[i % oldSize].TOTtoADC();
hostCalibParam.view()[i].TOT_ped() = hostCalibParamProvider.view()[i % oldSize].TOT_ped();
hostCalibParam.view()[i].TOT_lin() = hostCalibParamProvider.view()[i % oldSize].TOT_lin();
hostCalibParam.view()[i].TOT_P0() = hostCalibParamProvider.view()[i % oldSize].TOT_P0();
hostCalibParam.view()[i].TOT_P1() = hostCalibParamProvider.view()[i % oldSize].TOT_P1();
hostCalibParam.view()[i].TOT_P2() = hostCalibParamProvider.view()[i % oldSize].TOT_P2();
hostCalibParam.view()[i].TOAtops() = hostCalibParamProvider.view()[i % oldSize].TOAtops();
hostCalibParam.view()[i].MIPS_scale() = hostCalibParamProvider.view()[i % oldSize].MIPS_scale();
hostCalibParam.view()[i].valid() = hostCalibParamProvider.view()[i % oldSize].valid();
}
#else
const auto& hostDigis = hostDigisIn;
const auto& hostCalibParam = hostCalibParamProvider;
#endif

LogDebug("HGCalRecHitsProducer") << "Loaded host digis: " << hostDigis.view().metadata().size(); //<< std::endl;

LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- calling calibrate method"; //<< std::endl;
Expand All @@ -118,13 +152,24 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE {
auto start = std::chrono::steady_clock::now();
#endif

auto recHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParamProvider, deviceConfigParamProvider);
LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- Copying the calib to the device\n\n" << std::endl;
HGCalCalibParamDevice deviceCalibParam(hostCalibParam.view().metadata().size(), queue);
alpaka::memcpy(queue, deviceCalibParam.buffer(), hostCalibParam.const_buffer());

#ifdef HGCAL_PERF_TEST
auto tmpRecHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParam, deviceConfigParamProvider);
HGCalRecHitDevice recHits(oldSize, queue);
alpaka::memcpy(queue, recHits.buffer(), tmpRecHits.const_buffer(), oldSize);
#else
auto recHits = calibrator_.calibrate(queue, hostDigis, deviceCalibParam, deviceConfigParamProvider);
#endif

#ifdef EDM_ML_DEBUG
alpaka::wait(queue);
auto stop = std::chrono::steady_clock::now();
std::chrono::duration<float> elapsed = stop - start;
LogDebug("HGCalRecHitsProducer") << "Time spent calibrating: " << elapsed.count(); //<< std::endl;
LogDebug("HGCalRecHitsProducer") << "Time spent calibrating " << hostDigis.view().metadata().size()
<< " digis: " << elapsed.count(); //<< std::endl;
#endif

LogDebug("HGCalRecHitsProducer") << "\n\nINFO -- storing rec hits in the event"; //<< std::endl;
Expand Down