Extend the C++ module scanner to handle Fortran, too.

mesonbuild/backend/ninjabackend.py

@@ -117,7 +117,7 @@ def get_rsp_threshold():
 NINJA_QUOTE_BUILD_PAT = re.compile(r"[$ :\n]")
 NINJA_QUOTE_VAR_PAT = re.compile(r"[$ \n]")
 
-def ninja_quote(text, is_build_line=False):
+def ninja_quote(text: str, is_build_line=False) -> str:
     if is_build_line:
         quote_re = NINJA_QUOTE_BUILD_PAT
     else:
@@ -872,7 +872,11 @@ def generate_target(self, target):
         self.generate_shlib_aliases(target, self.get_target_dir(target))
         self.add_build(elem)
 
-    def should_scan_target(self, target):
+    def should_use_dyndeps_for_target(self, target: 'build.BuildTarget') -> bool:
+        if mesonlib.version_compare(self.ninja_version, '<1.10.0'):
+            return False
+        if 'fortran' in target.compilers:
+            return True
         if 'cpp' not in target.compilers:
             return False
         # Currently only the preview version of Visual Studio is supported.
@@ -883,18 +887,16 @@ def should_scan_target(self, target):
             return False
         if mesonlib.version_compare(cpp.version, '<19.28.28617'):
             return False
-        if mesonlib.version_compare(self.ninja_version, '<1.10.0'):
-            return False
         return True
 
     def generate_dependency_scan_target(self, target, compiled_sources, source2object):
-        if not self.should_scan_target(target):
+        if not self.should_use_dyndeps_for_target(target):
             return
         depscan_file = self.get_dep_scan_file_for(target)
         pickle_base = target.name + '.dat'
         pickle_file = os.path.join(self.get_target_private_dir(target), pickle_base).replace('\\', '/')
         pickle_abs = os.path.join(self.get_target_private_dir_abs(target), pickle_base).replace('\\', '/')
-        rule_name = 'cppscan'
+        rule_name = 'depscan'
         scan_sources = self.select_sources_to_scan(compiled_sources)
         elem = NinjaBuildElement(self.all_outputs, depscan_file, rule_name, scan_sources)
         elem.add_item('picklefile', pickle_file)
@@ -907,10 +909,11 @@ def select_sources_to_scan(self, compiled_sources):
         # in practice pick up C++ and Fortran files. If some other language
         # requires scanning (possibly Java to deal with inner class files)
         # then add them here.
+        all_suffixes = set(compilers.lang_suffixes['cpp']) | set(compilers.lang_suffixes['fortran'])
         selected_sources = []
         for source in compiled_sources:
             ext = os.path.splitext(source)[1][1:]
-            if ext in compilers.lang_suffixes['cpp']:
+            if ext in all_suffixes:
                 selected_sources.append(source)
         return selected_sources
 
@@ -1945,7 +1948,15 @@ def generate_swift_compile_rules(self, compiler):
         description = 'Compiling Swift source $in'
         self.add_rule(NinjaRule(rule, command, [], description))
 
-    def generate_fortran_dep_hack(self, crstr):
+    def use_dyndeps_for_fortran(self) -> bool:
+        '''Use the new Ninja feature for scanning dependencies during build,
+        rather than up front. Remove this and all old scanning code once Ninja
+        minimum version is bumped to 1.10.'''
+        return mesonlib.version_compare(self.ninja_version, '>=1.10.0')
+
+    def generate_fortran_dep_hack(self, crstr: str) -> None:
+        if self.use_dyndeps_for_fortran():
+            return
         rule = 'FORTRAN_DEP_HACK{}'.format(crstr)
         if mesonlib.is_windows():
             cmd = ['cmd', '/C']
@@ -2029,22 +2040,16 @@ def generate_pch_rule_for(self, langname, compiler):
 
 
     def generate_scanner_rules(self):
-        scanner_languages = {'cpp'} # Fixme, add Fortran.
-        for for_machine in MachineChoice:
-            clist = self.environment.coredata.compilers[for_machine]
-            for langname, compiler in clist.items():
-                if langname not in scanner_languages:
-                    continue
-                rulename = '{}scan'.format(langname)
-                if rulename in self.ruledict:
-                    # Scanning command is the same for native and cross compilation.
-                    continue
-                command = self.environment.get_build_command() + \
-                    ['--internal', 'depscan']
-                args = ['$picklefile', '$out', '$in']
-                description = 'Module scanner for {}.'.format(langname)
-                rule = NinjaRule(rulename, command, args, description)
-                self.add_rule(rule)
+        rulename = 'depscan'
+        if rulename in self.ruledict:
+            # Scanning command is the same for native and cross compilation.
+            return
+        command = self.environment.get_build_command() + \
+            ['--internal', 'depscan']
+        args = ['$picklefile', '$out', '$in']
+        description = 'Module scanner.'
+        rule = NinjaRule(rulename, command, args, description)
+        self.add_rule(rule)
 
 
     def generate_compile_rules(self):
@@ -2146,6 +2151,8 @@ def scan_fortran_module_outputs(self, target):
         """
         Find all module and submodule made available in a Fortran code file.
         """
+        if self.use_dyndeps_for_fortran():
+            return
         compiler = None
         # TODO other compilers
         for lang, c in self.environment.coredata.compilers.host.items():
@@ -2198,6 +2205,8 @@ def get_fortran_deps(self, compiler: FortranCompiler, src: Path, target) -> T.Li
         """
         Find all module and submodule needed by a Fortran target
         """
+        if self.use_dyndeps_for_fortran():
+            return []
 
         dirname = Path(self.get_target_private_dir(target))
         tdeps = self.fortran_deps[target.get_basename()]
@@ -2502,16 +2511,20 @@ def generate_single_compile(self, target, src, is_generated=False, header_deps=N
             if not is_generated:
                 abs_src = Path(build_dir) / rel_src
                 extra_deps += self.get_fortran_deps(compiler, abs_src, target)
-            # Dependency hack. Remove once multiple outputs in Ninja is fixed:
-            # https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8
-            for modname, srcfile in self.fortran_deps[target.get_basename()].items():
-                modfile = os.path.join(self.get_target_private_dir(target),
-                                       compiler.module_name_to_filename(modname))
-
-                if srcfile == src:
-                    crstr = self.get_rule_suffix(target.for_machine)
-                    depelem = NinjaBuildElement(self.all_outputs, modfile, 'FORTRAN_DEP_HACK' + crstr, rel_obj)
-                    self.add_build(depelem)
+            if not self.use_dyndeps_for_fortran():
+                # Dependency hack. Remove once multiple outputs in Ninja is fixed:
+                # https://groups.google.com/forum/#!topic/ninja-build/j-2RfBIOd_8
+                for modname, srcfile in self.fortran_deps[target.get_basename()].items():
+                    modfile = os.path.join(self.get_target_private_dir(target),
+                                           compiler.module_name_to_filename(modname))
+
+                    if srcfile == src:
+                        crstr = self.get_rule_suffix(target.for_machine)
+                        depelem = NinjaBuildElement(self.all_outputs,
+                                                    modfile,
+                                                     'FORTRAN_DEP_HACK' + crstr,
+                                                      rel_obj)
+                        self.add_build(depelem)
             commands += compiler.get_module_outdir_args(self.get_target_private_dir(target))
 
         element = NinjaBuildElement(self.all_outputs, rel_obj, compiler_name, rel_src)
@@ -2537,7 +2550,7 @@ def generate_single_compile(self, target, src, is_generated=False, header_deps=N
         return (rel_obj, rel_src.replace('\\', '/'))
 
     def add_dependency_scanner_entries_to_element(self, target, compiler, element):
-        if not self.should_scan_target(target):
+        if not self.should_use_dyndeps_for_target(target):
             return
         dep_scan_file = self.get_dep_scan_file_for(target)
         element.add_item('dyndep', dep_scan_file)

mesonbuild/scripts/depscan.py

@@ -15,12 +15,24 @@
 import pathlib
 import pickle
 import re
+import os
+import sys
 import typing as T
 
-from ..backend.ninjabackend import TargetDependencyScannerInfo
+from ..backend.ninjabackend import TargetDependencyScannerInfo, ninja_quote
+from ..compilers.compilers import lang_suffixes
 
-import_re = re.compile('\w*import ([a-zA-Z0-9]+);')
-export_re = re.compile('\w*export module ([a-zA-Z0-9]+);')
+CPP_IMPORT_RE = re.compile('\w*import ([a-zA-Z0-9]+);')
+CPP_EXPORT_RE = re.compile('\w*export module ([a-zA-Z0-9]+);')
+
+FORTRAN_INCLUDE_PAT = r"^\s*include\s*['\"](\w+\.\w+)['\"]"
+FORTRAN_MODULE_PAT = r"^\s*\bmodule\b\s+(\w+)\s*(?:!+.*)*$"
+FORTRAN_SUBMOD_PAT = r"^\s*\bsubmodule\b\s*\((\w+:?\w+)\)\s*(\w+)"
+FORTRAN_USE_PAT = r"^\s*use,?\s*(?:non_intrinsic)?\s*(?:::)?\s*(\w+)"
+
+FORTRAN_MODULE_RE = re.compile(FORTRAN_MODULE_PAT, re.IGNORECASE)
+FORTRAN_SUBMOD_RE = re.compile(FORTRAN_SUBMOD_PAT, re.IGNORECASE)
+FORTRAN_USE_RE = re.compile(FORTRAN_USE_PAT, re.IGNORECASE)
 
 class DependencyScanner:
     def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]):
@@ -32,11 +44,73 @@ def __init__(self, pickle_file: str, outfile: str, sources: T.List[str]):
         self.exports = {} # type: T.Dict[str, str]
         self.needs = {} # type: T.Dict[str, T.List[str]]
         self.sources_with_exports = [] # type: T.List[str]
-    
+
     def scan_file(self, fname: str) -> None:
-        for line in pathlib.Path(fname).read_text().split('\n'):
-            import_match = import_re.match(line)
-            export_match = export_re.match(line)
+        suffix = os.path.splitext(fname)[1][1:]
+        if suffix in lang_suffixes['fortran']:
+            self.scan_fortran_file(fname)
+        elif suffix in lang_suffixes['cpp']:
+            self.scan_cpp_file(fname)
+        else:
+            sys.exit('Can not scan files with suffix .{}.'.format(suffix))
+
+    def scan_fortran_file(self, fname: str) -> None:
+        fpath = pathlib.Path(fname)
+        modules_in_this_file = set()
+        for line in fpath.read_text().split('\n'):
+            import_match = FORTRAN_USE_RE.match(line)
+            export_match = FORTRAN_MODULE_RE.match(line)
+            submodule_export_match = FORTRAN_SUBMOD_RE.match(line)
+            if import_match:
+                needed = import_match.group(1).lower()
+                # In Fortran you have an using declaration also for the module
+                # you define in the same file. Prevent circular dependencies.
+                if needed not in modules_in_this_file:
+                    if fname in self.needs:
+                        self.needs[fname].append(needed)
+                    else:
+                        self.needs[fname] = [needed]
+            if export_match:
+                exported_module = export_match.group(1).lower()
+                assert(exported_module not in modules_in_this_file)
+                modules_in_this_file.add(exported_module)
+                if exported_module in self.provided_by:
+                    raise RuntimeError('Multiple files provide module {}.'.format(exported_module))
+                self.sources_with_exports.append(fname)
+                self.provided_by[exported_module] = fname
+                self.exports[fname] = exported_module
+            if submodule_export_match:
+                # Store submodule "Foo" "Bar" as "foo:bar".
+                # A submodule declaration can be both an import and an export declaration:
+                #
+                # submodule (a1:a2) a3
+                #  - requires a1@a2.smod
+                #  - produces a1@a3.smod
+                parent_module_name_full = submodule_export_match.group(1).lower()
+                parent_module_name = parent_module_name_full.split(':')[0]
+                submodule_name = submodule_export_match.group(2).lower()
+                concat_name = '{}:{}'.format(parent_module_name, submodule_name)
+                self.sources_with_exports.append(fname)
+                self.provided_by[concat_name] = fname
+                self.exports[fname] = concat_name
+                # Fortran requires that the immediate parent module must be built
+                # before the current one. Thus:
+                #
+                # submodule (parent) parent   <- requires parent.mod (really parent.smod, but they are created at the same time)
+                # submodule (a1:a2) a3        <- requires a1@a2.smod
+                #
+                # a3 does not depend on the a1 parent module directly, only transitively.
+                if fname in self.needs:
+                    self.needs[fname].append(parent_module_name_full)
+                else:
+                    self.needs[fname] = [parent_module_name_full]
+
+
+    def scan_cpp_file(self, fname: str) -> None:
+        fpath = pathlib.Path(fname)
+        for line in fpath.read_text().split('\n'):
+            import_match = CPP_IMPORT_RE.match(line)
+            export_match = CPP_EXPORT_RE.match(line)
             if import_match:
                 needed = import_match.group(1)
                 if fname in self.needs:
@@ -56,8 +130,22 @@ def objname_for(self, src: str) -> str:
         assert(isinstance(objname, str))
         return objname
 
-    def ifcname_for(self, src: str) -> str:
-        return '{}.ifc'.format(self.exports[src])
+    def module_name_for(self, src: str) -> str:
+        suffix= os.path.splitext(src)[1][1:]
+        if suffix in lang_suffixes['fortran']:
+            exported = self.exports[src]
+            # Module foo:bar goes to a file name foo@bar.smod
+            # Module Foo goes to a file name foo.mod
+            namebase = exported.replace(':', '@')
+            if ':' in exported:
+                extension = 'smod'
+            else:
+                extension = 'mod'
+            return os.path.join(self.target_data.private_dir, '{}.{}'.format(namebase, extension))
+        elif suffix in lang_suffixes['cpp']:
+            return '{}.ifc'.format(self.exports[src])
+        else:
+            raise RuntimeError('Unreachable code.')
 
     def scan(self) -> int:
         for s in self.sources:
@@ -66,21 +154,43 @@ def scan(self) -> int:
             ofile.write('ninja_dyndep_version = 1\n')
             for src in self.sources:
                 objfilename = self.objname_for(src)
+                mods_and_submods_needed = []
+                module_files_generated = []
+                module_files_needed = []
                 if src in self.sources_with_exports:
-                    ifc_entry = '| ' + self.ifcname_for(src)
-                else:
-                    ifc_entry = ''
+                    module_files_generated.append(self.module_name_for(src))
                 if src in self.needs:
-                    # FIXME, handle all sources, not just the first one
-                    modname = self.needs[src][0]
+                    for modname in self.needs[src]:
+                        if modname not in self.provided_by:
+                            # Nothing provides this module, we assume that it
+                            # comes from a dependency library somewhere and is
+                            # already built by the time this complation starts.
+                            pass
+                        else:
+                            mods_and_submods_needed.append(modname)
+
+                for modname in mods_and_submods_needed:
                     provider_src = self.provided_by[modname]
-                    provider_ifc = self.ifcname_for(provider_src)
-                    mod_dep = '| ' + provider_ifc
+                    provider_modfile = self.module_name_for(provider_src)
+                    # Prune self-dependencies
+                    if provider_src != src:
+                        module_files_needed.append(provider_modfile)
+
+                quoted_objfilename = ninja_quote(objfilename, True)
+                quoted_module_files_generated = [ninja_quote(x, True) for x in module_files_generated]
+                quoted_module_files_needed = [ninja_quote(x, True) for x in module_files_needed]
+                if quoted_module_files_generated:
+                    mod_gen = '| ' + ' '.join(quoted_module_files_generated)
+                else:
+                    mod_gen = ''
+                if quoted_module_files_needed:
+                    mod_dep = '| '  + ' '.join(quoted_module_files_needed)
                 else:
                     mod_dep = ''
-                ofile.write('build {} {}: dyndep {}\n'.format(objfilename,
-                                                              ifc_entry,
-                                                              mod_dep))
+                build_line = 'build {} {}: dyndep {}'.format(quoted_objfilename,
+                                                             mod_gen,
+                                                             mod_dep)
+                ofile.write(build_line + '\n')
         return 0
 
 def run(args: T.List[str]) -> int: