From 7f35f00b3dc47daf18e3eacebd21e381487ba1ec Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Sun, 24 Nov 2024 23:17:50 +0100 Subject: [PATCH 1/5] WIP: Plan mode for transformations --- loki/batch/scheduler.py | 69 +++--------- loki/batch/transformation.py | 101 ++++++++++++------ .../build_system/file_write.py | 28 +++-- loki/transformations/build_system/plan.py | 84 +++++++++++++++ .../build_system/tests/test_file_write.py | 16 ++- 5 files changed, 199 insertions(+), 99 deletions(-) create mode 100644 loki/transformations/build_system/plan.py diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 54915135a..5be0ad12f 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -382,7 +382,7 @@ def rekey_item_cache(self): if item.name not in deleted_keys ) - def process(self, transformation): + def process(self, transformation, plan=False): """ Process all :attr:`items` in the scheduler's graph with either a :any:`Pipeline` or a single :any:`Transformation`. @@ -398,16 +398,16 @@ def process(self, transformation): The transformation or transformation pipeline to apply """ if isinstance(transformation, Transformation): - self.process_transformation(transformation=transformation) + self.process_transformation(transformation=transformation, plan=plan) elif isinstance(transformation, Pipeline): - self.process_pipeline(pipeline=transformation) + self.process_pipeline(pipeline=transformation, plan=plan) else: error('[Loki::Scheduler] Batch processing requires Transformation or Pipeline object') raise RuntimeError('[Loki] Could not batch process {transformation_or_pipeline}') - def process_pipeline(self, pipeline): + def process_pipeline(self, pipeline, plan=False): """ Process a given :any:`Pipeline` by applying its assocaited transformations in turn. @@ -418,9 +418,9 @@ def process_pipeline(self, pipeline): The transformation pipeline to apply """ for transformation in pipeline.transformations: - self.process_transformation(transformation) + self.process_transformation(transformation, plan=plan) - def process_transformation(self, transformation): + def process_transformation(self, transformation, plan=False): """ Process all :attr:`items` in the scheduler's graph @@ -498,7 +498,8 @@ def _get_definition_items(_item, sgraph_items): _item.scope_ir, role=_item.role, mode=_item.mode, item=_item, targets=_item.targets, items=_get_definition_items(_item, sgraph_items), successors=graph.successors(_item, item_filter=item_filter), - depths=graph.depths, build_args=self.build_args + depths=graph.depths, build_args=self.build_args, + plan=plan ) if transformation.renames_items: @@ -628,53 +629,7 @@ def write_cmake_plan(self, filepath, mode, buildpath, rootpath): """ info(f'[Loki] Scheduler writing CMake plan: {filepath}') - rootpath = None if rootpath is None else Path(rootpath).resolve() - buildpath = None if buildpath is None else Path(buildpath) - sources_to_append = [] - sources_to_remove = [] - sources_to_transform = [] - - # Filter the SGraph to get a pure call-tree - item_filter = ProcedureItem - if self.config.enable_imports: - item_filter = as_tuple(item_filter) + (ModuleItem,) - graph = self.sgraph.as_filegraph( - self.item_factory, self.config, item_filter=item_filter, - exclude_ignored=True - ) - traversal = SFilter(graph, reverse=False, include_external=False) - for item in traversal: - if item.is_ignored: - continue - - sourcepath = item.path.resolve() - newsource = sourcepath.with_suffix(f'.{mode.lower()}.F90') - if buildpath: - newsource = buildpath/newsource.name - - # Make new CMake paths relative to source again - if rootpath is not None: - sourcepath = sourcepath.relative_to(rootpath) - - debug(f'Planning:: {item.name} (role={item.role}, mode={mode})') - - # Inject new object into the final binary libs - if newsource not in sources_to_append: - sources_to_transform += [sourcepath] - if item.replicate: - # Add new source file next to the old one - sources_to_append += [newsource] - else: - # Replace old source file to avoid ghosting - sources_to_append += [newsource] - sources_to_remove += [sourcepath] - - with Path(filepath).open('w') as f: - s_transform = '\n'.join(f' {s}' for s in sources_to_transform) - f.write(f'set( LOKI_SOURCES_TO_TRANSFORM \n{s_transform}\n )\n') - - s_append = '\n'.join(f' {s}' for s in sources_to_append) - f.write(f'set( LOKI_SOURCES_TO_APPEND \n{s_append}\n )\n') - - s_remove = '\n'.join(f' {s}' for s in sources_to_remove) - f.write(f'set( LOKI_SOURCES_TO_REMOVE \n{s_remove}\n )\n') + from loki.transformations.build_system.plan import CMakePlanTransformation + planner = CMakePlanTransformation(rootpath=rootpath) + self.process(planner, plan=True) + planner.write_plan(filepath) diff --git a/loki/batch/transformation.py b/loki/batch/transformation.py index 7b4454c8c..b7fcd1e93 100644 --- a/loki/batch/transformation.py +++ b/loki/batch/transformation.py @@ -128,6 +128,11 @@ def transform_subroutine(self, routine, **kwargs): Keyword arguments for the transformation. """ + def plan_subroutine(self, routine, **kwargs): + """ + ... + """ + def transform_module(self, module, **kwargs): """ Defines the transformation to apply to :any:`Module` items. @@ -144,6 +149,11 @@ def transform_module(self, module, **kwargs): Keyword arguments for the transformation. """ + def plan_module(self, module, **kwargs): + """ + ... + """ + def transform_file(self, sourcefile, **kwargs): """ Defines the transformation to apply to :any:`Sourcefile` items. @@ -160,7 +170,12 @@ def transform_file(self, sourcefile, **kwargs): Keyword arguments for the transformation. """ - def apply(self, source, post_apply_rescope_symbols=False, **kwargs): + def plan_file(self, sourcefile, **kwargs): + """ + ... + """ + + def apply(self, source, post_apply_rescope_symbols=False, plan=False, **kwargs): """ Dispatch method to apply transformation to :data:`source`. @@ -179,17 +194,18 @@ def apply(self, source, post_apply_rescope_symbols=False, **kwargs): actual transformation. """ if isinstance(source, Sourcefile): - self.apply_file(source, **kwargs) + self.apply_file(source, plan=plan, **kwargs) if isinstance(source, Subroutine): - self.apply_subroutine(source, **kwargs) + self.apply_subroutine(source, plan=plan, **kwargs) if isinstance(source, Module): - self.apply_module(source, **kwargs) + self.apply_module(source, plan=plan, **kwargs) - self.post_apply(source, rescope_symbols=post_apply_rescope_symbols) + if not plan: + self.post_apply(source, rescope_symbols=post_apply_rescope_symbols) - def apply_file(self, sourcefile, **kwargs): + def apply_file(self, sourcefile, plan=False, **kwargs): """ Apply transformation to all items in :data:`sourcefile`. @@ -212,16 +228,19 @@ def apply_file(self, sourcefile, **kwargs): if not isinstance(sourcefile, Sourcefile): raise TypeError('Transformation.apply_file can only be applied to Sourcefile object') - if sourcefile._incomplete: - raise RuntimeError('Transformation.apply_file requires Sourcefile to be complete') - item = kwargs.pop('item', None) items = kwargs.pop('items', None) role = kwargs.pop('role', None) targets = kwargs.pop('targets', None) # Apply file-level transformations - self.transform_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) + if plan: + self.plan_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) + else: + if not plan and sourcefile._incomplete: + raise RuntimeError('Transformation.apply_file requires Sourcefile to be complete') + + self.transform_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) # Recurse to modules, if configured if self.recurse_to_modules: @@ -243,12 +262,20 @@ def apply_file(self, sourcefile, **kwargs): # Provide the list of items that belong to this module item_items = tuple(_it for _it in items if _it.scope is item.ir) - self.transform_module( - item.ir, item=item, role=item_role, targets=item.targets, items=item_items, **kwargs - ) + if plan: + self.plan_module( + item.ir, item=item, role=item_role, targets=item.targets, items=item_items, **kwargs + ) + else: + self.transform_module( + item.ir, item=item, role=item_role, targets=item.targets, items=item_items, **kwargs + ) else: for module in sourcefile.modules: - self.transform_module(module, item=item, role=role, targets=targets, items=items, **kwargs) + if plan: + self.plan_module(module, item=item, role=role, targets=targets, items=items, **kwargs) + else: + self.transform_module(module, item=item, role=role, targets=targets, items=items, **kwargs) # Recurse into procedures, if configured if self.recurse_to_procedures: @@ -256,14 +283,22 @@ def apply_file(self, sourcefile, **kwargs): # Recursion into all subroutine items in the current file for item in items: if isinstance(item, ProcedureItem): - self.transform_subroutine( - item.ir, item=item, role=item.role, targets=item.targets, **kwargs - ) + if plan: + self.plan_subroutine( + item.ir, item=item, role=item.role, targets=item.targets, **kwargs + ) + else: + self.transform_subroutine( + item.ir, item=item, role=item.role, targets=item.targets, **kwargs + ) else: for routine in sourcefile.all_subroutines: - self.transform_subroutine(routine, item=item, role=role, targets=targets, **kwargs) + if plan: + self.plan_subroutine(routine, item=item, role=role, targets=targets, **kwargs) + else: + self.transform_subroutine(routine, item=item, role=role, targets=targets, **kwargs) - def apply_subroutine(self, subroutine, **kwargs): + def apply_subroutine(self, subroutine, plan=False, **kwargs): """ Apply transformation to a given :any:`Subroutine` object and its members. @@ -284,18 +319,21 @@ def apply_subroutine(self, subroutine, **kwargs): if not isinstance(subroutine, Subroutine): raise TypeError('Transformation.apply_subroutine can only be applied to Subroutine object') - if subroutine._incomplete: - raise RuntimeError('Transformation.apply_subroutine requires Subroutine to be complete') - # Apply the actual transformation for subroutines - self.transform_subroutine(subroutine, **kwargs) + if plan: + self.plan_subroutine(subroutine, **kwargs) + else: + if subroutine._incomplete: + raise RuntimeError('Transformation.apply_subroutine requires Subroutine to be complete') + + self.transform_subroutine(subroutine, **kwargs) # Recurse to internal procedures if self.recurse_to_internal_procedures: for routine in subroutine.subroutines: - self.apply_subroutine(routine, **kwargs) + self.apply_subroutine(routine, plan=plan, **kwargs) - def apply_module(self, module, **kwargs): + def apply_module(self, module, plan=False, **kwargs): """ Apply transformation to a given :any:`Module` object and its members. @@ -315,16 +353,19 @@ def apply_module(self, module, **kwargs): if not isinstance(module, Module): raise TypeError('Transformation.apply_module can only be applied to Module object') - if module._incomplete: - raise RuntimeError('Transformation.apply_module requires Module to be complete') - # Apply the actual transformation for modules - self.transform_module(module, **kwargs) + if plan: + self.plan_module(module, **kwargs) + else: + if module._incomplete: + raise RuntimeError('Transformation.apply_module requires Module to be complete') + + self.transform_module(module, **kwargs) # Recurse to procedures contained in this module if self.recurse_to_procedures: for routine in module.subroutines: - self.apply_subroutine(routine, **kwargs) + self.apply_subroutine(routine, plan=plan, **kwargs) def post_apply(self, source, rescope_symbols=False): """ diff --git a/loki/transformations/build_system/file_write.py b/loki/transformations/build_system/file_write.py index edfa84a86..af2e99aae 100644 --- a/loki/transformations/build_system/file_write.py +++ b/loki/transformations/build_system/file_write.py @@ -54,12 +54,7 @@ def item_filter(self): return (ProcedureItem, ModuleItem) return ProcedureItem - def transform_file(self, sourcefile, **kwargs): - item = kwargs.get('item', None) - if not item and 'items' in kwargs: - if kwargs['items']: - item = kwargs['items'][0] - + def _get_file_path(self, item, build_args): if not item: raise ValueError('No Item provided; required to determine file write path') @@ -69,7 +64,26 @@ def transform_file(self, sourcefile, **kwargs): path = Path(item.path) suffix = self.suffix if self.suffix else path.suffix sourcepath = Path(item.path).with_suffix(f'.{_mode}{suffix}') - build_args = kwargs.get('build_args', {}) if build_args and (output_dir := build_args.get('output_dir', None)) is not None: sourcepath = Path(output_dir)/sourcepath.name + return sourcepath + + def transform_file(self, sourcefile, **kwargs): + item = kwargs.get('item') + if not item and 'items' in kwargs: + if kwargs['items']: + item = kwargs['items'][0] + + build_args = kwargs.get('build_args', {}) + sourcepath = self._get_file_path(item, build_args) sourcefile.write(path=sourcepath, cuf=self.cuf) + + def plan_file(self, sourcefile, **kwargs): # pylint: disable=unused-argument + item = kwargs.get('item') + if not item and 'items' in kwargs: + if kwargs['items']: + item = kwargs['items'][0] + + build_args = kwargs.get('build_args', {}) + sourcepath = self._get_file_path(item, build_args) + item.trafo_data['FileWriteTransformation'] = {'path': sourcepath} diff --git a/loki/transformations/build_system/plan.py b/loki/transformations/build_system/plan.py new file mode 100644 index 000000000..38880d455 --- /dev/null +++ b/loki/transformations/build_system/plan.py @@ -0,0 +1,84 @@ +# (C) Copyright 2018- ECMWF. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +""" +Transformations to be used for exposing planned changes to +the build system +""" + +from pathlib import Path + +from loki.batch import Transformation +from loki.logging import debug + +class CMakePlanTransformation(Transformation): + """ + Generate the "plan file" for CMake + + The plan file is a CMake file defining three lists: + + * ``LOKI_SOURCES_TO_TRANSFORM``: The list of files that are + processed in the dependency graph + * ``LOKI_SOURCES_TO_APPEND``: The list of files that are created + and have to be added to the build target as part of the processing + * ``LOKI_SOURCES_TO_REMOVE``: The list of files that are no longer + required (because they have been replaced by transformed files) and + should be removed from the build target. + + Parameters + ---------- + rootpath : str (optional) + + """ + + # This transformation is applied over the file graph + traverse_file_graph = True + + item_filter = None + + def __init__(self, rootpath=None): + self.rootpath = None if rootpath is None else Path(rootpath).resolve() + self.sources_to_append = [] + self.sources_to_remove = [] + self.sources_to_transform = [] + + def plan_file(self, sourcefile, **kwargs): + item = kwargs.get('item') + if not item: + raise ValueError('No Item provided; required to determine CMake plan') + + if not 'FileWriteTransformation' in item.trafo_data: + return + + sourcepath = item.path.resolve() + if self.rootpath is not None: + sourcepath = sourcepath.relative_to(self.rootpath) + + newsource = item.trafo_data['FileWriteTransformation']['path'] + + debug(f'Planning:: {item.name} (role={item.role}, mode={item.mode})') + + if newsource not in self.sources_to_append: + self.sources_to_transform += [sourcepath] + if item.replicate: + # Add new source file next to the old one + self.sources_to_append += [newsource] + else: + # Replace old source file to avoid ghosting + self.sources_to_append += [newsource] + self.sources_to_remove += [sourcepath] + + def write_plan(self, filepath): + with Path(filepath).open('w') as f: + s_transform = '\n'.join(f' {s}' for s in self.sources_to_transform) + f.write(f'set( LOKI_SOURCES_TO_TRANSFORM \n{s_transform}\n )\n') + + s_append = '\n'.join(f' {s}' for s in self.sources_to_append) + f.write(f'set( LOKI_SOURCES_TO_APPEND \n{s_append}\n )\n') + + s_remove = '\n'.join(f' {s}' for s in self.sources_to_remove) + f.write(f'set( LOKI_SOURCES_TO_REMOVE \n{s_remove}\n )\n') diff --git a/loki/transformations/build_system/tests/test_file_write.py b/loki/transformations/build_system/tests/test_file_write.py index 8a0de54e4..fcc94bc27 100644 --- a/loki/transformations/build_system/tests/test_file_write.py +++ b/loki/transformations/build_system/tests/test_file_write.py @@ -153,9 +153,16 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le # Check the dependency graph assert expected_items == {item.name for item in scheduler.items} + # Set-up the file write + transformation = FileWriteTransformation( + suffix=suffix, + include_module_var_imports=enable_imports + ) + # Generate the CMake plan plan_file = tmp_path/'plan.cmake' root_path = tmp_path if use_rootpath else None + scheduler.process(transformation, plan=True) scheduler.write_cmake_plan( filepath=plan_file, mode=config.default['mode'], buildpath=out_path, rootpath=root_path @@ -192,10 +199,6 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le } # Write the outputs - transformation = FileWriteTransformation( - suffix=suffix, - include_module_var_imports=enable_imports - ) scheduler.process(transformation) # Validate the list of written files @@ -290,7 +293,11 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con # Check the dependency graph assert expected_items == {item.name for item in scheduler.items} + # Set-up the file write + transformation = FileWriteTransformation(include_module_var_imports=True) + # Generate the CMake plan + scheduler.process(transformation, plan=True) plan_file = tmp_path/'plan.cmake' caplog.clear() @@ -319,7 +326,6 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con assert plan_dict['LOKI_SOURCES_TO_APPEND'] == {'a.foobar', 'b.foobar', 'c.foobar', 'd.foobar'} # Write the outputs - transformation = FileWriteTransformation(include_module_var_imports=True) scheduler.process(transformation) # Validate the list of written files From ad75311a8905dbe90f6d2340b7bdc98a99886a41 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Mon, 25 Nov 2024 12:15:27 +0100 Subject: [PATCH 2/5] CMake integration of pipeline-based plan writer --- loki/batch/scheduler.py | 2 +- loki/batch/tests/test_scheduler.py | 28 +++++----- loki/tests/test_cmake.py | 46 ++++++++++------ .../build_system/tests/test_file_write.py | 10 +--- scripts/loki_transform.py | 53 +++++++++++-------- 5 files changed, 77 insertions(+), 62 deletions(-) diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 5be0ad12f..80c5037bf 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -610,7 +610,7 @@ def callgraph(self, path, with_file_graph=False, with_legend=False): warning(f'[Loki] Failed to render filegraph due to graphviz error:\n {e}') @Timer(logger=perf, text='[Loki::Scheduler] Wrote CMake plan file in {:.2f}s') - def write_cmake_plan(self, filepath, mode, buildpath, rootpath): + def write_cmake_plan(self, filepath, rootpath=None): """ Generate the "plan file" for CMake diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 40977b85a..5508ade26 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -74,7 +74,7 @@ nodes as ir, FindNodes, FindInlineCalls, FindVariables ) from loki.transformations import ( - DependencyTransformation, ModuleWrapTransformation + DependencyTransformation, ModuleWrapTransformation, FileWriteTransformation ) @@ -1156,35 +1156,35 @@ def test_scheduler_cmake_planner(tmp_path, testdir, frontend): proj_b = sourcedir/'projB' config = SchedulerConfig.from_dict({ - 'default': {'role': 'kernel', 'expand': True, 'strict': True, 'ignore': ('header_mod',)}, + 'default': { + 'role': 'kernel', + 'expand': True, + 'strict': True, + 'ignore': ('header_mod',), + 'mode': 'foobar' + }, 'routines': { 'driverB': {'role': 'driver'}, 'kernelB': {'ignore': ['ext_driver']}, } }) + builddir = tmp_path/'scheduler_cmake_planner_dummy_dir' + builddir.mkdir(exist_ok=True) # Populate the scheduler # (this is the same as SchedulerA in test_scheduler_dependencies_ignore, so no need to # check scheduler set-up itself) scheduler = Scheduler( paths=[proj_a, proj_b], includes=proj_a/'include', - config=config, frontend=frontend, xmods=[tmp_path] + config=config, frontend=frontend, xmods=[tmp_path], + output_dir=builddir ) # Apply the transformation - builddir = tmp_path/'scheduler_cmake_planner_dummy_dir' - builddir.mkdir(exist_ok=True) planfile = builddir/'loki_plan.cmake' - scheduler.write_cmake_plan( - filepath=planfile, mode='foobar', buildpath=builddir, rootpath=sourcedir - ) - - # Validate the generated lists - expected_files = { - proj_a/'module/driverB_mod.f90', proj_a/'module/kernelB_mod.F90', - proj_a/'module/compute_l1_mod.f90', proj_a/'module/compute_l2_mod.f90' - } + scheduler.process(FileWriteTransformation(), plan=True) + scheduler.write_cmake_plan(filepath=planfile, rootpath=sourcedir) # Validate the plan file content plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL) diff --git a/loki/tests/test_cmake.py b/loki/tests/test_cmake.py index 2a9f01bfa..f313d6617 100644 --- a/loki/tests/test_cmake.py +++ b/loki/tests/test_cmake.py @@ -71,9 +71,32 @@ def fixture_config(tmp_dir): the file path """ default_config = { - 'default': {'role': 'kernel', 'expand': True, 'strict': True, 'enable_imports': True}, + 'default': { + 'role': 'kernel', + 'expand': True, + 'strict': True, + 'enable_imports': True + }, 'routines': { 'driverB': {'role': 'driver'}, + }, + 'transformations': { + 'IdemTrafo': { + 'classname': 'IdemTransformation', + 'module': 'loki.transformations.idempotence', + }, + 'FileWriteTransformation': { + 'classname': 'FileWriteTransformation', + 'module': 'loki.transformations.build_system', + 'options': { + 'include_module_var_imports': True + } + } + }, + 'pipelines': { + 'idem': { + 'transformations': ['IdemTrafo'] + } } } filepath = tmp_dir/'test_cmake_loki.config' @@ -101,30 +124,25 @@ def fixture_loki_install(here, tmp_dir, ecbuild, silent, request): Install Loki using CMake into an install directory """ builddir = tmp_dir/'loki_bootstrap' - if builddir.exists(): - shutil.rmtree(builddir) - builddir.mkdir() - cmd = ['cmake', '-DENABLE_CLAW=OFF', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake', str(here.parent.parent)] + cmd = [ + 'cmake', f'-DCMAKE_MODULE_PATH={ecbuild}/cmake', + '-S', str(here.parent.parent), + '-B', str(builddir) + ] if request.param: cmd += ['-DENABLE_EDITABLE=ON'] else: cmd += ['-DENABLE_EDITABLE=OFF'] - execute(cmd, silent=silent, cwd=builddir) + execute(cmd, silent=silent, cwd=tmp_dir) lokidir = tmp_dir/'loki' - if lokidir.exists(): - shutil.rmtree(lokidir) execute( ['cmake', '--install', '.', '--prefix', str(lokidir)], silent=True, cwd=builddir ) yield builddir, lokidir - if builddir.exists(): - shutil.rmtree(builddir) - if lokidir.exists(): - shutil.rmtree(lokidir) @contextmanager @@ -137,7 +155,6 @@ def clean_builddir(builddir): shutil.rmtree(builddir) builddir.mkdir() yield builddir - shutil.rmtree(builddir) @pytest.fixture(scope='module', name='cmake_project') @@ -158,7 +175,6 @@ def fixture_cmake_project(here, config, srcdir): loki_transform_plan( MODE idem - FRONTEND fp CONFIG {config} SOURCEDIR ${{CMAKE_CURRENT_SOURCE_DIR}} CALLGRAPH ${{CMAKE_CURRENT_BINARY_DIR}}/loki_callgraph @@ -166,8 +182,6 @@ def fixture_cmake_project(here, config, srcdir): SOURCES {proj_a} {proj_b} - HEADERS - {proj_a}/module/header_mod.f90 ) """ filepath = srcdir/'CMakeLists.txt' diff --git a/loki/transformations/build_system/tests/test_file_write.py b/loki/transformations/build_system/tests/test_file_write.py index fcc94bc27..d137b4a9a 100644 --- a/loki/transformations/build_system/tests/test_file_write.py +++ b/loki/transformations/build_system/tests/test_file_write.py @@ -163,10 +163,7 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le plan_file = tmp_path/'plan.cmake' root_path = tmp_path if use_rootpath else None scheduler.process(transformation, plan=True) - scheduler.write_cmake_plan( - filepath=plan_file, mode=config.default['mode'], buildpath=out_path, - rootpath=root_path - ) + scheduler.write_cmake_plan(filepath=plan_file, rootpath=root_path) # Validate the plan file content plan_pattern = re.compile(r'set\(\s*(\w+)\s*(.*?)\s*\)', re.DOTALL) @@ -302,10 +299,7 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con caplog.clear() with caplog.at_level(log_levels['WARNING']): - scheduler.write_cmake_plan( - filepath=plan_file, mode=config.default['mode'], buildpath=out_path, - rootpath=tmp_path - ) + scheduler.write_cmake_plan(filepath=plan_file, rootpath=tmp_path) if have_non_replicate_conflict: assert len(caplog.records) == 1 assert 'c.f90' in caplog.records[0].message diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 931cce88b..690e88a04 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -13,6 +13,7 @@ """ from pathlib import Path +import sys import click from loki import ( @@ -110,6 +111,12 @@ def cli(debug): help="Recursively derive explicit shape dimension for argument arrays") @click.option('--eliminate-dead-code/--no-eliminate-dead-code', default=True, help='Perform dead code elimination, where unreachable branches are trimmed from the code.') +@click.option('--plan-file', type=click.Path(), default=None, + help='Process pipeline in planning mode and generate CMake "plan" file.') +@click.option('--callgraph', '-g', type=click.Path(), default=None, + help='Generate and display the subroutine callgraph.') +@click.option('--root', type=click.Path(), default=None, + help='Root path to which all paths are relative to.') @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING', type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']), help='Log level to output during batch processing') @@ -118,7 +125,7 @@ def convert( data_offload, remove_openmp, assume_deviceptr, frontend, trim_vector_sections, global_var_offload, remove_derived_args, inline_members, inline_marked, resolve_sequence_association, resolve_sequence_association_inlined_calls, - derive_argument_array_shape, eliminate_dead_code, log_level + derive_argument_array_shape, eliminate_dead_code, plan_file, callgraph, root, log_level ): """ Batch-processing mode for Fortran-to-Fortran transformations that @@ -133,7 +140,12 @@ def convert( loki_config['log-level'] = log_level - info(f'[Loki] Batch-processing source files using config: {config} ') + plan = plan_file is not None + + if plan: + info(f'[Loki] Creating CMake plan file from config: {config}') + else: + info(f'[Loki] Batch-processing source files using config: {config} ') config = SchedulerConfig.from_file(config) @@ -179,7 +191,7 @@ def convert( info(f'[Loki-transform] Applying custom pipeline {mode} from config:') info(str(config.pipelines[mode])) - scheduler.process( config.pipelines[mode] ) + scheduler.process(config.pipelines[mode], plan=plan) mode = mode.replace('-', '_') # Sanitize mode string @@ -187,10 +199,21 @@ def convert( file_write_trafo = scheduler.config.transformations.get('FileWriteTransformation', None) if not file_write_trafo: file_write_trafo = FileWriteTransformation(cuf='cuf' in mode) - scheduler.process(transformation=file_write_trafo) + scheduler.process(transformation=file_write_trafo, plan=plan) + + if plan: + scheduler.write_cmake_plan(plan_file, rootpath=root) + + if callgraph: + scheduler.callgraph(callgraph) return + if plan: + msg = '[Loki] ERROR: Plan mode requires a pipeline definition in the config file.\n' + msg += '[Loki] Please provide a config file with configured transformation or pipelines instead.\n' + sys.exit(msg) + # If we do not use a custom pipeline, it should be one of the internally supported ones assert mode in [ 'idem', 'c', 'idem-stack', 'sca', 'claw', 'scc', 'scc-hoist', 'scc-stack', @@ -442,32 +465,16 @@ def convert( @click.option('--log-level', '-l', default='info', envvar='LOKI_LOGGING', type=click.Choice(['debug', 'detail', 'perf', 'info', 'warning', 'error']), help='Log level to output during batch processing') +@click.pass_context def plan( - mode, config, header, source, build, root, cpp, directive, + ctx, mode, config, header, source, build, root, cpp, directive, frontend, callgraph, plan_file, log_level ): """ Create a "plan", a schedule of files to inject and transform for a given configuration. """ - - loki_config['log-level'] = log_level - - info(f'[Loki] Creating CMake plan file from config: {config}') - config = SchedulerConfig.from_file(config) - - paths = [Path(s).resolve() for s in source] - paths += [Path(h).resolve().parent for h in header] - scheduler = Scheduler(paths=paths, config=config, frontend=frontend, full_parse=False, preprocess=cpp) - - mode = mode.replace('-', '_') # Sanitize mode string - - # Construct the transformation plan as a set of CMake lists of source files - scheduler.write_cmake_plan(filepath=plan_file, mode=mode, buildpath=build, rootpath=root) - - # Output the resulting callgraph - if callgraph: - scheduler.callgraph(callgraph) + return ctx.forward(convert) if __name__ == "__main__": From 23597c30e85a67a532003b99cd68f11d7ad116a3 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 10 Dec 2024 15:41:36 +0100 Subject: [PATCH 3/5] Generalize plan mode to processing strategy --- loki/batch/scheduler.py | 86 ++++++++++++----- loki/batch/tests/test_scheduler.py | 4 +- loki/batch/transformation.py | 92 ++++++++++++++----- loki/transformations/build_system/plan.py | 34 ++++++- .../build_system/tests/test_file_write.py | 6 +- scripts/loki_transform.py | 16 ++-- 6 files changed, 175 insertions(+), 63 deletions(-) diff --git a/loki/batch/scheduler.py b/loki/batch/scheduler.py index 80c5037bf..13d910d04 100644 --- a/loki/batch/scheduler.py +++ b/loki/batch/scheduler.py @@ -5,6 +5,7 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. +from enum import Enum, auto from os.path import commonpath from pathlib import Path from codetiming import Timer @@ -21,10 +22,40 @@ from loki.frontend import FP, REGEX, RegexParserClass from loki.tools import as_tuple, CaseInsensitiveDict, flatten -from loki.logging import info, perf, warning, debug, error +from loki.logging import info, perf, warning, error -__all__ = ['Scheduler'] +__all__ = ['ProcessingStrategy', 'Scheduler'] + + +class ProcessingStrategy(Enum): + """ + List of available processing types for :any:`Scheduler.process` + + Multiple options exist how the :any:`Scheduler.process` method can + apply a provided :any:`Transformation` or :any:`Pipeline` object to the + items in a :any:`Scheduler` graph. The permissible options and default + values are provided by this class. + """ + + SEQUENCE = auto() + """Sequential processing of transformations + + For each transformation in a pipeline, the :any:`Transformation.apply` + method is called for every item in the graph, following the graph traversal + mode specified in the transformation's manifest, before repeating the + same for the next transformation in the pipeline. + """ + + PLAN = auto() + """Planning mode using :any:`ProcessingStrategy.SEQUENCE` strategy. + + This calls :any:`Transformation.plan` (instead of :any:`Transformation.apply`) + for each transformation. + """ + + DEFAULT = SEQUENCE + """Default processing strategy, currently :any:`ProcessingStrategy.SEQUENCE`""" class Scheduler: @@ -382,7 +413,7 @@ def rekey_item_cache(self): if item.name not in deleted_keys ) - def process(self, transformation, plan=False): + def process(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT): """ Process all :attr:`items` in the scheduler's graph with either a :any:`Pipeline` or a single :any:`Transformation`. @@ -396,18 +427,21 @@ def process(self, transformation, plan=False): ---------- transformation : :any:`Transformation` or :any:`Pipeline` The transformation or transformation pipeline to apply + proc_strategy : :any:`ProcessingStrategy` + The processing strategy to use when applying the given + :data:`transformation` to the scheduler's graph. """ if isinstance(transformation, Transformation): - self.process_transformation(transformation=transformation, plan=plan) + self.process_transformation(transformation=transformation, proc_strategy=proc_strategy) elif isinstance(transformation, Pipeline): - self.process_pipeline(pipeline=transformation, plan=plan) + self.process_pipeline(pipeline=transformation, proc_strategy=proc_strategy) else: error('[Loki::Scheduler] Batch processing requires Transformation or Pipeline object') - raise RuntimeError('[Loki] Could not batch process {transformation_or_pipeline}') + raise RuntimeError(f'Could not batch process {transformation}') - def process_pipeline(self, pipeline, plan=False): + def process_pipeline(self, pipeline, proc_strategy=ProcessingStrategy.DEFAULT): """ Process a given :any:`Pipeline` by applying its assocaited transformations in turn. @@ -416,11 +450,14 @@ def process_pipeline(self, pipeline, plan=False): ---------- transformation : :any:`Pipeline` The transformation pipeline to apply + proc_strategy : :any:`ProcessingStrategy` + The processing strategy to use when applying the given + :data:`pipeline` to the scheduler's graph. """ for transformation in pipeline.transformations: - self.process_transformation(transformation, plan=plan) + self.process_transformation(transformation, proc_strategy=proc_strategy) - def process_transformation(self, transformation, plan=False): + def process_transformation(self, transformation, proc_strategy=ProcessingStrategy.DEFAULT): """ Process all :attr:`items` in the scheduler's graph @@ -445,6 +482,9 @@ def process_transformation(self, transformation, plan=False): ---------- transformation : :any:`Transformation` The transformation to apply over the dependency tree + proc_strategy : :any:`ProcessingStrategy` + The processing strategy to use when applying the given + :data:`transformation` to the scheduler's graph. """ def _get_definition_items(_item, sgraph_items): # For backward-compatibility with the DependencyTransform and LinterTransformation @@ -464,6 +504,10 @@ def _get_definition_items(_item, sgraph_items): items += (item,) + child_items return items + if proc_strategy not in (ProcessingStrategy.SEQUENCE, ProcessingStrategy.PLAN): + error(f'[Loki::Scheduler] Processing {proc_strategy} is not implemented!') + raise RuntimeError(f'Could not batch process {transformation}') + trafo_name = transformation.__class__.__name__ log = f'[Loki::Scheduler] Applied transformation <{trafo_name}>' + ' in {:.2f}s' with Timer(logger=info, text=log): @@ -499,7 +543,7 @@ def _get_definition_items(_item, sgraph_items): item=_item, targets=_item.targets, items=_get_definition_items(_item, sgraph_items), successors=graph.successors(_item, item_filter=item_filter), depths=graph.depths, build_args=self.build_args, - plan=plan + plan_mode=proc_strategy == ProcessingStrategy.PLAN ) if transformation.renames_items: @@ -614,22 +658,18 @@ def write_cmake_plan(self, filepath, rootpath=None): """ Generate the "plan file" for CMake - The plan file is a CMake file defining three lists: - - * ``LOKI_SOURCES_TO_TRANSFORM``: The list of files that are - processed in the dependency graph - * ``LOKI_SOURCES_TO_APPEND``: The list of files that are created - and have to be added to the build target as part of the processing - * ``LOKI_SOURCES_TO_REMOVE``: The list of files that are no longer - required (because they have been replaced by transformed files) and - should be removed from the build target. + See :any:`CMakePlanTransformation` for the specification of that file. - These lists are used by the CMake wrappers to schedule the source - updates and update the source lists of the CMake target object accordingly. + Parameters + ---------- + filepath : str or Path + The path of the CMake file to write. + rootpath : str or Path (optional) + If given, all paths in the CMake file will be made relative to this root directory """ info(f'[Loki] Scheduler writing CMake plan: {filepath}') - from loki.transformations.build_system.plan import CMakePlanTransformation + from loki.transformations.build_system.plan import CMakePlanTransformation # pylint: disable=import-outside-toplevel planner = CMakePlanTransformation(rootpath=rootpath) - self.process(planner, plan=True) + self.process(planner, proc_strategy=ProcessingStrategy.PLAN) planner.write_plan(filepath) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index 5508ade26..b52c622be 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -64,7 +64,7 @@ from loki.batch import ( Scheduler, SchedulerConfig, Item, ProcedureItem, ProcedureBindingItem, InterfaceItem, TypeDefItem, SFilter, - ExternalItem, Transformation, Pipeline + ExternalItem, Transformation, Pipeline, ProcessingStrategy ) from loki.expression import Scalar, Array, Literal, ProcedureSymbol from loki.frontend import ( @@ -1183,7 +1183,7 @@ def test_scheduler_cmake_planner(tmp_path, testdir, frontend): # Apply the transformation planfile = builddir/'loki_plan.cmake' - scheduler.process(FileWriteTransformation(), plan=True) + scheduler.process(FileWriteTransformation(), proc_strategy=ProcessingStrategy.PLAN) scheduler.write_cmake_plan(filepath=planfile, rootpath=sourcedir) # Validate the plan file content diff --git a/loki/batch/transformation.py b/loki/batch/transformation.py index b7fcd1e93..b427779ff 100644 --- a/loki/batch/transformation.py +++ b/loki/batch/transformation.py @@ -130,7 +130,19 @@ def transform_subroutine(self, routine, **kwargs): def plan_subroutine(self, routine, **kwargs): """ - ... + Define the planning steps to apply for :any:`Subroutine` items. + + For transformations that modify the dependencies of :data:`routine` + (e.g., adding new procedure calls, inlining calls, renaming the interface) + this should be implemented. It gets called via the dispatch method :meth:`apply` + if the optional ``plan_mode`` argument is set to `True`. + + Parameters + ---------- + routine : :any:`Subroutine` + The subroutine to be transformed. + **kwargs : optional + Keyword arguments for the transformation. """ def transform_module(self, module, **kwargs): @@ -151,7 +163,19 @@ def transform_module(self, module, **kwargs): def plan_module(self, module, **kwargs): """ - ... + Define the planning steps to apply for :any:`Module` items. + + For transformations that modify the dependencies or definitions of :data:`module` + (e.g., renaming the module, adding new subroutines, adding or removing imports) + this should be implemented. It gets called via the dispatch method :meth:`apply` + if the optional ``plan_mode`` argument is set to `True`. + + Parameters + ---------- + module : :any:`Module` + The module to be transformed. + **kwargs : optional + Keyword arguments for the transformation. """ def transform_file(self, sourcefile, **kwargs): @@ -172,10 +196,21 @@ def transform_file(self, sourcefile, **kwargs): def plan_file(self, sourcefile, **kwargs): """ - ... + Define the planning steps to apply for :any:`Sourcefile` items. + + For transformations that modify the definitions or dependencies of :data:`sourcefile` + this should be implemented. It gets called via the dispatch method :meth:`apply` + if the optional ``plan_mode`` argument is set to `True`. + + Parameters + ---------- + sourcefile : :any:`Sourcefile` + The sourcefile to be transformed. + **kwargs : optional + Keyword arguments for the transformation. """ - def apply(self, source, post_apply_rescope_symbols=False, plan=False, **kwargs): + def apply(self, source, post_apply_rescope_symbols=False, plan_mode=False, **kwargs): """ Dispatch method to apply transformation to :data:`source`. @@ -194,22 +229,23 @@ def apply(self, source, post_apply_rescope_symbols=False, plan=False, **kwargs): actual transformation. """ if isinstance(source, Sourcefile): - self.apply_file(source, plan=plan, **kwargs) + self.apply_file(source, plan_mode=plan_mode, **kwargs) if isinstance(source, Subroutine): - self.apply_subroutine(source, plan=plan, **kwargs) + self.apply_subroutine(source, plan_mode=plan_mode, **kwargs) if isinstance(source, Module): - self.apply_module(source, plan=plan, **kwargs) + self.apply_module(source, plan_mode=plan_mode, **kwargs) - if not plan: + if not plan_mode: self.post_apply(source, rescope_symbols=post_apply_rescope_symbols) - def apply_file(self, sourcefile, plan=False, **kwargs): + def apply_file(self, sourcefile, plan_mode=False, **kwargs): """ Apply transformation to all items in :data:`sourcefile`. - This calls :meth:`transform_file`. + This calls :meth:`transform_file` or, if :data:`plan_mode` is enabled, + calls :meth:`plan_file`. If the :attr:`recurse_to_modules` class property is set, it will also invoke :meth:`apply` on all :any:`Module` objects in @@ -222,6 +258,8 @@ def apply_file(self, sourcefile, plan=False, **kwargs): ---------- sourcefile : :any:`Sourcefile` The file to transform. + plan_mode : bool, optional + If enabled, apply planning mode. **kwargs : optional Keyword arguments that are passed on to transformation methods. """ @@ -234,10 +272,10 @@ def apply_file(self, sourcefile, plan=False, **kwargs): targets = kwargs.pop('targets', None) # Apply file-level transformations - if plan: + if plan_mode: self.plan_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) else: - if not plan and sourcefile._incomplete: + if not plan_mode and sourcefile._incomplete: raise RuntimeError('Transformation.apply_file requires Sourcefile to be complete') self.transform_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) @@ -262,7 +300,7 @@ def apply_file(self, sourcefile, plan=False, **kwargs): # Provide the list of items that belong to this module item_items = tuple(_it for _it in items if _it.scope is item.ir) - if plan: + if plan_mode: self.plan_module( item.ir, item=item, role=item_role, targets=item.targets, items=item_items, **kwargs ) @@ -272,7 +310,7 @@ def apply_file(self, sourcefile, plan=False, **kwargs): ) else: for module in sourcefile.modules: - if plan: + if plan_mode: self.plan_module(module, item=item, role=role, targets=targets, items=items, **kwargs) else: self.transform_module(module, item=item, role=role, targets=targets, items=items, **kwargs) @@ -283,7 +321,7 @@ def apply_file(self, sourcefile, plan=False, **kwargs): # Recursion into all subroutine items in the current file for item in items: if isinstance(item, ProcedureItem): - if plan: + if plan_mode: self.plan_subroutine( item.ir, item=item, role=item.role, targets=item.targets, **kwargs ) @@ -293,16 +331,17 @@ def apply_file(self, sourcefile, plan=False, **kwargs): ) else: for routine in sourcefile.all_subroutines: - if plan: + if plan_mode: self.plan_subroutine(routine, item=item, role=role, targets=targets, **kwargs) else: self.transform_subroutine(routine, item=item, role=role, targets=targets, **kwargs) - def apply_subroutine(self, subroutine, plan=False, **kwargs): + def apply_subroutine(self, subroutine, plan_mode=False, **kwargs): """ Apply transformation to a given :any:`Subroutine` object and its members. - This calls :meth:`transform_subroutine`. + This calls :meth:`transform_subroutine` or, if :data:`plan_mode` is enabled, + calls :meth:`plan_subroutine`. If the :attr:`recurse_to_member_procedures` class property is set, it will also invoke :meth:`apply` on all @@ -313,6 +352,8 @@ def apply_subroutine(self, subroutine, plan=False, **kwargs): ---------- subroutine : :any:`Subroutine` The subroutine to transform. + plan_mode : bool, optional + If enabled, apply planning mode. **kwargs : optional Keyword arguments that are passed on to transformation methods. """ @@ -320,7 +361,7 @@ def apply_subroutine(self, subroutine, plan=False, **kwargs): raise TypeError('Transformation.apply_subroutine can only be applied to Subroutine object') # Apply the actual transformation for subroutines - if plan: + if plan_mode: self.plan_subroutine(subroutine, **kwargs) else: if subroutine._incomplete: @@ -331,13 +372,14 @@ def apply_subroutine(self, subroutine, plan=False, **kwargs): # Recurse to internal procedures if self.recurse_to_internal_procedures: for routine in subroutine.subroutines: - self.apply_subroutine(routine, plan=plan, **kwargs) + self.apply_subroutine(routine, plan_mode=plan_mode, **kwargs) - def apply_module(self, module, plan=False, **kwargs): + def apply_module(self, module, plan_mode=False, **kwargs): """ Apply transformation to a given :any:`Module` object and its members. - This calls :meth:`transform_module`. + This calls :meth:`transform_module` or, if :data:`plan_mode` is enabled, + calls :meth:`plan_module`. If the :attr:`recurse_to_procedures` class property is set, it will also invoke :meth:`apply` on all :any:`Subroutine` @@ -347,6 +389,8 @@ def apply_module(self, module, plan=False, **kwargs): ---------- module : :any:`Module` The module to transform. + plan_mode : bool, optional + If enabled, apply planning mode. **kwargs : optional Keyword arguments that are passed on to transformation methods. """ @@ -354,7 +398,7 @@ def apply_module(self, module, plan=False, **kwargs): raise TypeError('Transformation.apply_module can only be applied to Module object') # Apply the actual transformation for modules - if plan: + if plan_mode: self.plan_module(module, **kwargs) else: if module._incomplete: @@ -365,7 +409,7 @@ def apply_module(self, module, plan=False, **kwargs): # Recurse to procedures contained in this module if self.recurse_to_procedures: for routine in module.subroutines: - self.apply_subroutine(routine, plan=plan, **kwargs) + self.apply_subroutine(routine, plan_mode=plan_mode, **kwargs) def post_apply(self, source, rescope_symbols=False): """ diff --git a/loki/transformations/build_system/plan.py b/loki/transformations/build_system/plan.py index 38880d455..03c51638f 100644 --- a/loki/transformations/build_system/plan.py +++ b/loki/transformations/build_system/plan.py @@ -17,9 +17,30 @@ class CMakePlanTransformation(Transformation): """ - Generate the "plan file" for CMake + Gather the planning information from all :any:`Item.trafo_data` to which + this information is applied and allows writing a CMake plan file - The plan file is a CMake file defining three lists: + This requires that :any:`FileWriteTransformation` has been applied in planning + mode first. + + Applying this transformation to a :any:`Item` updates internal lists: + + * :attr:`sources_to_transform`: The path of all source files that contain + objects that are transformed by a Loki transformation in the pipeline + * :attr:`sources_to_append`: The path of any new source files that exist + as a consequence of the Loki transformation pipeline, e.g., transformed + source files that are written. + * :attr:`sources_to_remove`: The path of any existing source files that + are to be removed from the compilation target. This includes all items + that don't have the :any:`Item.replicate` property. + + The :any:`Sourcefile.path` is used to determine the file path from which a + Fortran sourcefile was read. New paths are provided in + ``item.trafo_data['FileWriteTransformation']['path']``. + + The method :meth:`write_plan` allows to write the gathered information to + a CMake file that can be included in the CMake scripts that build a library. + The plan file is a CMake file defining three lists matching the above: * ``LOKI_SOURCES_TO_TRANSFORM``: The list of files that are processed in the dependency graph @@ -29,10 +50,14 @@ class CMakePlanTransformation(Transformation): required (because they have been replaced by transformed files) and should be removed from the build target. + These lists are used by the Loki CMake wrappers (particularly + ``loki_transform_target``) to schedule the source updates and update the + source lists of the CMake target object accordingly. + Parameters ---------- rootpath : str (optional) - + If given, all paths will be resolved relative to this root directory """ # This transformation is applied over the file graph @@ -73,6 +98,9 @@ def plan_file(self, sourcefile, **kwargs): self.sources_to_remove += [sourcepath] def write_plan(self, filepath): + """ + Write the CMake plan file to :data:`filepath` + """ with Path(filepath).open('w') as f: s_transform = '\n'.join(f' {s}' for s in self.sources_to_transform) f.write(f'set( LOKI_SOURCES_TO_TRANSFORM \n{s_transform}\n )\n') diff --git a/loki/transformations/build_system/tests/test_file_write.py b/loki/transformations/build_system/tests/test_file_write.py index d137b4a9a..40f2b5b9a 100644 --- a/loki/transformations/build_system/tests/test_file_write.py +++ b/loki/transformations/build_system/tests/test_file_write.py @@ -15,7 +15,7 @@ import pytest -from loki.batch import Scheduler, SchedulerConfig +from loki.batch import Scheduler, SchedulerConfig, ProcessingStrategy from loki.frontend import available_frontends, OMNI from loki.logging import log_levels from loki.transformations.build_system import FileWriteTransformation @@ -162,7 +162,7 @@ def test_file_write_module_imports(frontend, tmp_path, enable_imports, import_le # Generate the CMake plan plan_file = tmp_path/'plan.cmake' root_path = tmp_path if use_rootpath else None - scheduler.process(transformation, plan=True) + scheduler.process(transformation, proc_strategy=ProcessingStrategy.PLAN) scheduler.write_cmake_plan(filepath=plan_file, rootpath=root_path) # Validate the plan file content @@ -294,7 +294,7 @@ def test_file_write_replicate(tmp_path, caplog, frontend, have_non_replicate_con transformation = FileWriteTransformation(include_module_var_imports=True) # Generate the CMake plan - scheduler.process(transformation, plan=True) + scheduler.process(transformation, proc_strategy=ProcessingStrategy.PLAN) plan_file = tmp_path/'plan.cmake' caplog.clear() diff --git a/scripts/loki_transform.py b/scripts/loki_transform.py index 690e88a04..8e9f747bb 100644 --- a/scripts/loki_transform.py +++ b/scripts/loki_transform.py @@ -20,7 +20,7 @@ config as loki_config, Sourcefile, Frontend, as_tuple, set_excepthook, auto_post_mortem_debugger, info, warning ) -from loki.batch import Pipeline, Scheduler, SchedulerConfig +from loki.batch import Pipeline, Scheduler, SchedulerConfig, ProcessingStrategy # Get generalized transformations provided by Loki from loki.transformations.argument_shape import ( @@ -140,11 +140,11 @@ def convert( loki_config['log-level'] = log_level - plan = plan_file is not None - - if plan: + if plan_file is not None: + processing_strategy = ProcessingStrategy.PLAN info(f'[Loki] Creating CMake plan file from config: {config}') else: + processing_strategy = ProcessingStrategy.DEFAULT info(f'[Loki] Batch-processing source files using config: {config} ') config = SchedulerConfig.from_file(config) @@ -191,7 +191,7 @@ def convert( info(f'[Loki-transform] Applying custom pipeline {mode} from config:') info(str(config.pipelines[mode])) - scheduler.process(config.pipelines[mode], plan=plan) + scheduler.process(config.pipelines[mode], proc_strategy=processing_strategy) mode = mode.replace('-', '_') # Sanitize mode string @@ -199,9 +199,9 @@ def convert( file_write_trafo = scheduler.config.transformations.get('FileWriteTransformation', None) if not file_write_trafo: file_write_trafo = FileWriteTransformation(cuf='cuf' in mode) - scheduler.process(transformation=file_write_trafo, plan=plan) + scheduler.process(transformation=file_write_trafo, proc_strategy=processing_strategy) - if plan: + if plan_file is not None: scheduler.write_cmake_plan(plan_file, rootpath=root) if callgraph: @@ -209,7 +209,7 @@ def convert( return - if plan: + if plan_file is not None: msg = '[Loki] ERROR: Plan mode requires a pipeline definition in the config file.\n' msg += '[Loki] Please provide a config file with configured transformation or pipelines instead.\n' sys.exit(msg) From d84fbb0a1665190179ad8aa7d73518c4250eda7f Mon Sep 17 00:00:00 2001 From: Balthasar Reuter Date: Tue, 10 Dec 2024 17:18:58 +0100 Subject: [PATCH 4/5] Fix linter warnings --- loki/batch/tests/test_scheduler.py | 7 +++++-- loki/transformations/build_system/plan.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/loki/batch/tests/test_scheduler.py b/loki/batch/tests/test_scheduler.py index b52c622be..f280c1d6d 100644 --- a/loki/batch/tests/test_scheduler.py +++ b/loki/batch/tests/test_scheduler.py @@ -1001,10 +1001,13 @@ def test_scheduler_missing_files(testdir, config, frontend, strict, tmp_path): # Check processing with missing items class CheckApply(Transformation): - def apply(self, source, post_apply_rescope_symbols=False, **kwargs): + def apply(self, source, post_apply_rescope_symbols=False, plan_mode=False, **kwargs): assert 'item' in kwargs assert not isinstance(kwargs['item'], ExternalItem) - super().apply(source, post_apply_rescope_symbols=post_apply_rescope_symbols, **kwargs) + super().apply( + source, post_apply_rescope_symbols=post_apply_rescope_symbols, + plan_mode=plan_mode, **kwargs + ) if strict: with pytest.raises(RuntimeError): diff --git a/loki/transformations/build_system/plan.py b/loki/transformations/build_system/plan.py index 03c51638f..cf14628de 100644 --- a/loki/transformations/build_system/plan.py +++ b/loki/transformations/build_system/plan.py @@ -12,7 +12,7 @@ from pathlib import Path -from loki.batch import Transformation +from loki.batch.transformation import Transformation from loki.logging import debug class CMakePlanTransformation(Transformation): From 633d038b61b316a5a4324dbe4e62044ce9c1a968 Mon Sep 17 00:00:00 2001 From: Balthasar Reuter <6384870+reuterbal@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:36:50 +0100 Subject: [PATCH 5/5] Remove redundant check Co-authored-by: Michael Staneker <50531288+MichaelSt98@users.noreply.github.com> --- loki/batch/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loki/batch/transformation.py b/loki/batch/transformation.py index b427779ff..6732a0c25 100644 --- a/loki/batch/transformation.py +++ b/loki/batch/transformation.py @@ -275,7 +275,7 @@ def apply_file(self, sourcefile, plan_mode=False, **kwargs): if plan_mode: self.plan_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs) else: - if not plan_mode and sourcefile._incomplete: + if sourcefile._incomplete: raise RuntimeError('Transformation.apply_file requires Sourcefile to be complete') self.transform_file(sourcefile, item=item, role=role, targets=targets, items=items, **kwargs)