-
Notifications
You must be signed in to change notification settings - Fork 4.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #40564 from makortel/alpakaModuleTypeResolver
Enable "alternative module loading" approach for Alpaka modules
- Loading branch information
Showing
17 changed files
with
342 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
10 changes: 10 additions & 0 deletions
10
HeterogeneousCore/AlpakaCore/interface/module_backend_config.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #ifndef HeterogeneousCore_AlpakaCore_interface_module_backend_config_h | ||
| #define HeterogeneousCore_AlpakaCore_interface_module_backend_config_h | ||
|
|
||
| #include "FWCore/Framework/interface/FrameworkfwdMostUsed.h" | ||
|
|
||
| namespace cms::alpakatools { | ||
| void module_backend_config(edm::ConfigurationDescriptions& iDesc); | ||
| } | ||
|
|
||
| #endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| <library file="*.cc" name="HeterogeneousCoreAlpakaCorePlugins"> | ||
| <use name="FWCore/Framework"/> | ||
| <use name="FWCore/MessageLogger"/> | ||
| <use name="FWCore/ParameterSet"/> | ||
| <use name="FWCore/Utilities"/> | ||
| <flags EDM_PLUGIN="1"/> | ||
| </library> |
66 changes: 66 additions & 0 deletions
66
HeterogeneousCore/AlpakaCore/plugins/ModuleTypeResolverAlpaka.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| #include "FWCore/Framework/interface/ModuleTypeResolverMaker.h" | ||
| #include "FWCore/MessageLogger/interface/MessageLogger.h" | ||
| #include "FWCore/ParameterSet/interface/ParameterSet.h" | ||
| #include "FWCore/Utilities/interface/EDMException.h" | ||
| #include "FWCore/Utilities/interface/thread_safety_macros.h" | ||
|
|
||
| #include <cassert> | ||
| #include <cstring> | ||
| #include <string> | ||
| #include <unordered_map> | ||
|
|
||
| class ModuleTypeResolverAlpaka : public edm::ModuleTypeResolverBase { | ||
| public: | ||
| ModuleTypeResolverAlpaka(std::string backendPrefix) : backendPrefix_(std::move(backendPrefix)) {} | ||
|
|
||
| std::pair<std::string, int> resolveType(std::string basename, int index) const final { | ||
| assert(index == kInitialIndex); | ||
| constexpr auto kAlpaka = "@alpaka"; | ||
| auto found = basename.find(kAlpaka); | ||
| if (found != std::string::npos) { | ||
| if (backendPrefix_.empty()) { | ||
| throw edm::Exception(edm::errors::LogicError) | ||
| << "AlpakaModuleTypeResolver encountered a module with type name " << basename | ||
| << " but the backend prefix was empty. This should not happen. Please contact framework developers."; | ||
| } | ||
| basename.replace(found, std::strlen(kAlpaka), ""); | ||
| basename = backendPrefix_ + basename; | ||
| } | ||
| return {basename, kLastIndex}; | ||
| } | ||
|
|
||
| private: | ||
| std::string const backendPrefix_; | ||
| }; | ||
|
|
||
| class ModuleTypeResolverMakerAlpaka : public edm::ModuleTypeResolverMaker { | ||
| public: | ||
| ModuleTypeResolverMakerAlpaka() {} | ||
|
|
||
| std::shared_ptr<edm::ModuleTypeResolverBase const> makeResolver(edm::ParameterSet const& modulePSet) const final { | ||
| std::string prefix = ""; | ||
| if (modulePSet.existsAs<edm::ParameterSet>("alpaka", false)) { | ||
| auto const& backend = | ||
| modulePSet.getUntrackedParameter<edm::ParameterSet>("alpaka").getUntrackedParameter<std::string>("backend"); | ||
| prefix = fmt::format("alpaka_{}::", backend); | ||
|
|
||
| LogDebug("AlpakaModuleTypeResolver") | ||
| .format("AlpakaModuleTypeResolver: module {} backend prefix {}", | ||
| modulePSet.getParameter<std::string>("@module_label"), | ||
| prefix); | ||
| } | ||
| auto found = cache_.find(prefix); | ||
| if (found == cache_.end()) { | ||
| bool inserted; | ||
| std::tie(found, inserted) = cache_.emplace(prefix, std::make_shared<ModuleTypeResolverAlpaka>(prefix)); | ||
| } | ||
| return found->second; | ||
| } | ||
|
|
||
| private: | ||
| // no protection needed because this object is used only in single-thread context | ||
| CMS_SA_ALLOW mutable std::unordered_map<std::string, std::shared_ptr<ModuleTypeResolverAlpaka const>> cache_; | ||
| }; | ||
|
|
||
| #include "FWCore/Framework/interface/ModuleTypeResolverMakerFactory.h" | ||
| DEFINE_EDM_PLUGIN(edm::ModuleTypeResolverMakerFactory, ModuleTypeResolverMakerAlpaka, "ModuleTypeResolverAlpaka"); |
70 changes: 70 additions & 0 deletions
70
HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import FWCore.ParameterSet.Config as cms | ||
|
|
||
| class ModuleTypeResolverAlpaka: | ||
| def __init__(self, accelerators, backend): | ||
| # first element is used as the default is nothing is set | ||
| self._valid_backends = [] | ||
| if "gpu-nvidia" in accelerators: | ||
| self._valid_backends.append("cuda_async") | ||
| if "cpu" in accelerators: | ||
| self._valid_backends.append("serial_sync") | ||
| if len(self._valid_backends) == 0: | ||
| raise cms.EDMException(cms.edm.errors.UnavailableAccelerator, "ModuleTypeResolverAlpaka had no backends available because of the combination of the job configuration and accelerator availability of on the machine. The job sees {} accelerators.".format(", ".join(accelerators))) | ||
| if backend is not None: | ||
| if not backend in self._valid_backends: | ||
| raise cms.EDMException(cms.edm.errors.UnavailableAccelerator, "The ProcessAcceleratorAlpaka was configured to use {} backend, but that backend is not available because of the combination of the job configuration and accelerator availability on the machine. The job was configured to use {} accelerators, which translates to {} Alpaka backends.".format( | ||
| backend, ", ".join(accelerators), ", ".join(self._valid_backends))) | ||
| if backend != self._valid_backends[0]: | ||
| self._valid_backends.remove(backend) | ||
| self._valid_backends.insert(0, backend) | ||
|
|
||
| def plugin(self): | ||
| return "ModuleTypeResolverAlpaka" | ||
|
|
||
| def setModuleVariant(self, module): | ||
| if module.type_().endswith("@alpaka"): | ||
| defaultBackend = self._valid_backends[0] | ||
| if hasattr(module, "alpaka"): | ||
| if hasattr(module.alpaka, "backend"): | ||
| if module.alpaka.backend == "": | ||
| module.alpaka.backend = defaultBackend | ||
| elif module.alpaka.backend.value() not in self._valid_backends: | ||
| raise cms.EDMException(cms.edm.errors.UnavailableAccelerator, "Module {} has the Alpaka backend set explicitly, but its accelerator is not available for the job because of the combination of the job configuration and accelerator availability on the machine. The following Alpaka backends are available for the job {}.".format(module.label_(), ", ".join(self._valid_backends))) | ||
| else: | ||
| module.alpaka.backend = cms.untracked.string(defaultBackend) | ||
| else: | ||
| module.alpaka = cms.untracked.PSet( | ||
| backend = cms.untracked.string(defaultBackend) | ||
| ) | ||
|
|
||
| class ProcessAcceleratorAlpaka(cms.ProcessAccelerator): | ||
| """ProcessAcceleratorAlpaka itself does not define or inspect | ||
| availability of any accelerator devices. It merely sets up | ||
| necessary Alpaka infrastructure based on the availability of | ||
| accelerators that the concrete ProcessAccelerators (like | ||
| ProcessAcceleratorCUDA) define. | ||
| """ | ||
| def __init__(self): | ||
| super(ProcessAcceleratorAlpaka,self).__init__() | ||
| self._backend = None | ||
| # User-facing interface | ||
| def setBackend(self, backend): | ||
| self._backend = backend | ||
| # Framework-facing interface | ||
| def moduleTypeResolver(self, accelerators): | ||
| return ModuleTypeResolverAlpaka(accelerators, self._backend) | ||
| def apply(self, process, accelerators): | ||
| if not hasattr(process, "AlpakaServiceSerialSync"): | ||
| from HeterogeneousCore.AlpakaServices.AlpakaServiceSerialSync_cfi import AlpakaServiceSerialSync | ||
| process.add_(AlpakaServiceSerialSync) | ||
| if not hasattr(process, "AlpakaServiceCudaAsync"): | ||
| from HeterogeneousCore.AlpakaServices.AlpakaServiceCudaAsync_cfi import AlpakaServiceCudaAsync | ||
| process.add_(AlpakaServiceCudaAsync) | ||
|
|
||
| if not hasattr(process.MessageLogger, "AlpakaService"): | ||
| process.MessageLogger.AlpakaService = cms.untracked.PSet() | ||
|
|
||
| process.AlpakaServiceSerialSync.enabled = "cpu" in accelerators | ||
| process.AlpakaServiceCudaAsync.enabled = "gpu-nvidia" in accelerators | ||
|
|
||
| cms.specialImportRegistry.registerSpecialImportForType(ProcessAcceleratorAlpaka, "from HeterogeneousCore.AlpakaCore.ProcessAcceleratorAlpaka import ProcessAcceleratorAlpaka") |
3 changes: 3 additions & 0 deletions
3
HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka_cfi.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| from HeterogeneousCore.AlpakaCore.ProcessAcceleratorAlpaka import ProcessAcceleratorAlpaka as _ProcessAcceleratorAlpaka | ||
|
|
||
| ProcessAcceleratorAlpaka = _ProcessAcceleratorAlpaka() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" | ||
| #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" | ||
| #include "HeterogeneousCore/AlpakaCore/interface/module_backend_config.h" | ||
|
|
||
| namespace { | ||
| const std::string kPSetName("alpaka"); | ||
| const char* const kComment = | ||
| "PSet allows to override the Alpaka backend per module instance. Has an effect only when the module class name " | ||
| "has '@alpaka' suffix, i.e. has no effect when the Alpaka backend namespace is used explicitly."; | ||
| } // namespace | ||
|
|
||
| namespace cms::alpakatools { | ||
| void module_backend_config(edm::ConfigurationDescriptions& iDesc) { | ||
| // the code below leads to 'alpaka = untracked.PSet(backend = untracked.string)' to be added to the generated cfi files | ||
| // TODO: I don't know if this is a desired behavior for HLT | ||
| edm::ParameterSetDescription descAlpaka; | ||
| descAlpaka.addUntracked<std::string>("backend", "") | ||
| ->setComment( | ||
| "Alpaka backend for this module. Can be empty string (for the global default), 'serial_sync', or " | ||
| "'cuda_async'"); | ||
|
|
||
| if (iDesc.defaultDescription()) { | ||
| if (iDesc.defaultDescription()->isLabelUnused(kPSetName)) { | ||
| iDesc.defaultDescription() | ||
| ->addUntracked<edm::ParameterSetDescription>(kPSetName, descAlpaka) | ||
| ->setComment(kComment); | ||
| } | ||
| } | ||
| for (auto& v : iDesc) { | ||
| if (v.second.isLabelUnused(kPSetName)) { | ||
| v.second.addUntracked<edm::ParameterSetDescription>(kPSetName, descAlpaka)->setComment(kComment); | ||
| } | ||
| } | ||
| } | ||
| } // namespace cms::alpakatools |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.