Add a new target "ninja clippy" for Rust projects

docs/markdown/IDE-integration.md

@@ -93,13 +93,21 @@ can provide code completion for all source files.
 ```json
 {
     "language": "language ID",
+    "machine": "build" / "host",
     "compiler": ["The", "compiler", "command"],
     "parameters": ["list", "of", "compiler", "parameters"],
     "sources": ["list", "of", "all", "source", "files", "for", "this", "language"],
     "generated_sources": ["list", "of", "all", "source", "files", "that", "where", "generated", "somewhere", "else"]
 }
 ```
 
+*(New in 1.7.0)* The `machine` and `language` keys make it possible to
+to access further information about the compiler in the `compilers`
+introspection information.  `machine` can be absent if `language` is
+`unknown`.  In this case, information about the compiler is not
+available; Meson is therefore unable to know if the output relates
+to either the build of the host machine.
+
 It should be noted that the compiler parameters stored in the
 `parameters` differ from the actual parameters used to compile the
 file. This is because the parameters are optimized for the usage in an

docs/markdown/Unit-tests.md

@@ -83,13 +83,13 @@ possible.
 
 By default Meson uses as many concurrent processes as there are cores
 on the test machine. You can override this with the environment
-variable `MESON_TESTTHREADS` like this.
+variable `MESON_TESTTHREADS` or, *since 1.7.0*, `MESON_NUM_PROCESSES`:
 
 ```console
-$ MESON_TESTTHREADS=5 meson test
+$ MESON_NUM_PROCESSES=5 meson test
 ```
 
-Setting `MESON_TESTTHREADS` to 0 enables the default behavior (core
+Setting `MESON_NUM_PROCESSES` to 0 enables the default behavior (core
 count), whereas setting an invalid value results in setting the job
 count to 1.
 

docs/markdown/howtox.md

@@ -239,6 +239,26 @@ And then pass it through the variable (remember to use absolute path):
 $ SCANBUILD=$(pwd)/my-scan-build.sh ninja -C builddir scan-build
 ```
 
+## Use clippy
+
+If your project includes Rust targets, you can invoke clippy like this:
+
+```console
+$ meson setup builddir
+$ ninja -C builddir clippy
+```
+
+Clippy will also obey the `werror` [builtin option](Builtin-options.md#core-options).
+
+By default Meson uses as many concurrent processes as there are cores
+on the test machine. You can override this with the environment
+variable `MESON_NUM_PROCESSES`.
+
+Meson will look for `clippy-driver` in the same directory as `rustc`,
+or try to invoke it using `rustup` if `rustc` points to a `rustup`
+binary.  If `clippy-driver` is not detected properly, you can add it to
+a [machine file](Machine-files.md).
+
 ## Use profile guided optimization
 
 Using profile guided optimization with GCC is a two phase

docs/markdown/snippets/clippy.md

@@ -0,0 +1,8 @@
+## Meson can run "clippy" on Rust projects
+
+Meson now defines a `clippy` target if the project uses the Rust programming
+language.  The target runs clippy on all Rust sources, using the `clippy-driver`
+program from the same Rust toolchain as the `rustc` compiler.
+
+Using `clippy-driver` as the Rust compiler will now emit a warning, as it
+is not meant to be a general-purpose compiler front-end.

docs/markdown/snippets/introspect_machine.md

@@ -0,0 +1,5 @@
+## "machine" entry in target introspection data
+
+The JSON data returned by `meson introspect --targets` now has a `machine`
+entry in each `target_sources` block.  The new entry can be one of `build`
+or `host` for compiler-built targets, or absent for `custom_target` targets.

docs/markdown/snippets/num-processes.md

@@ -0,0 +1,8 @@
+## Control the number of child processes with an environment variable
+
+Previously, `meson test` checked the `MESON_TESTTHREADS` variable to control
+the amount of parallel jobs to run; this was useful when `meson test` is
+invoked through `ninja test` for example.  With this version, a new variable
+`MESON_NUM_PROCESSES` is supported with a broader scope: in addition to
+`meson test`, it is also used by the `external_project` module and by
+Ninja targets that invoke `clang-tidy`, `clang-format` and `clippy`.

mesonbuild/ast/introspection.py

@@ -283,7 +283,7 @@ def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]:
         kwargs_reduced = {k: v for k, v in kwargs.items() if k in targetclass.known_kwargs and k in {'install', 'build_by_default', 'build_always'}}
         kwargs_reduced = {k: v.value if isinstance(v, ElementaryNode) else v for k, v in kwargs_reduced.items()}
         kwargs_reduced = {k: v for k, v in kwargs_reduced.items() if not isinstance(v, BaseNode)}
-        for_machine = MachineChoice.HOST
+        for_machine = MachineChoice.BUILD if kwargs.get('native', False) else MachineChoice.HOST
         objects: T.List[T.Any] = []
         empty_sources: T.List[T.Any] = []
         # Passing the unresolved sources list causes errors
@@ -294,6 +294,7 @@ def traverse_nodes(inqueue: T.List[BaseNode]) -> T.List[BaseNode]:
 
         new_target = {
             'name': target.get_basename(),
+            'machine': target.for_machine.get_lower_case_name(),
             'id': target.get_id(),
             'type': target.get_typename(),
             'defined_in': os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)),

mesonbuild/backend/backends.py

@@ -41,13 +41,14 @@
     from ..linkers.linkers import StaticLinker
     from ..mesonlib import FileMode, FileOrString
 
-    from typing_extensions import TypedDict
+    from typing_extensions import TypedDict, NotRequired
 
     _ALL_SOURCES_TYPE = T.List[T.Union[File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList]]
 
     class TargetIntrospectionData(TypedDict):
 
         language: str
+        machine: NotRequired[str]
         compiler: T.List[str]
         parameters: T.List[str]
         sources: T.List[str]
@@ -2039,6 +2040,12 @@ def compiler_to_generator_args(self, target: build.BuildTarget,
         commands += [input]
         return commands
 
+    def have_language(self, langname: str) -> bool:
+        for for_machine in MachineChoice:
+            if langname in self.environment.coredata.compilers[for_machine]:
+                return True
+        return False
+
     def compiler_to_generator(self, target: build.BuildTarget,
                               compiler: 'Compiler',
                               sources: _ALL_SOURCES_TYPE,

mesonbuild/backend/ninjabackend.py

@@ -463,6 +463,8 @@ class RustCrate:
 
     display_name: str
     root_module: str
+    crate_type: str
+    target_name: str
     edition: RUST_EDITIONS
     deps: T.List[RustDep]
     cfg: T.List[str]
@@ -837,6 +839,7 @@ def create_target_source_introspection(self, target: build.Target, comp: compile
             # The new entry
             src_block = {
                 'language': lang,
+                'machine': comp.for_machine.get_lower_case_name(),
                 'compiler': comp.get_exelist(),
                 'parameters': parameters,
                 'sources': [],
@@ -1877,6 +1880,7 @@ def __generate_sources_structure(self, root: Path, structured_sources: build.Str
         return orderdeps, first_file
 
     def _add_rust_project_entry(self, name: str, main_rust_file: str, args: CompilerArgs,
+                                crate_type: str, target_name: str,
                                 from_subproject: bool, proc_macro_dylib_path: T.Optional[str],
                                 deps: T.List[RustDep]) -> None:
         raw_edition: T.Optional[str] = mesonlib.first(reversed(args), lambda x: x.startswith('--edition'))
@@ -1894,6 +1898,8 @@ def _add_rust_project_entry(self, name: str, main_rust_file: str, args: Compiler
             len(self.rust_crates),
             name,
             main_rust_file,
+            crate_type,
+            target_name,
             edition,
             deps,
             cfg,
@@ -2133,7 +2139,7 @@ def _link_library(libname: str, static: bool, bundle: bool = False):
 
         self._add_rust_project_entry(target.name,
                                      os.path.abspath(os.path.join(self.environment.build_dir, main_rust_file)),
-                                     args,
+                                     args, cratetype, target_name,
                                      bool(target.subproject),
                                      proc_macro_dylib_path,
                                      project_deps)
@@ -3639,6 +3645,20 @@ def generate_dist(self) -> None:
         elem.add_item('pool', 'console')
         self.add_build(elem)
 
+    def generate_clippy(self) -> None:
+        if 'clippy' in self.all_outputs or not self.have_language('rust'):
+            return
+
+        cmd = self.environment.get_build_command() + \
+            ['--internal', 'clippy', self.environment.build_dir]
+        elem = self.create_phony_target('clippy', 'CUSTOM_COMMAND', 'PHONY')
+        elem.add_item('COMMAND', cmd)
+        elem.add_item('pool', 'console')
+        for crate in self.rust_crates.values():
+            if crate.crate_type in {'rlib', 'dylib', 'proc-macro'}:
+                elem.add_dep(crate.target_name)
+        self.add_build(elem)
+
     def generate_scanbuild(self) -> None:
         if not environment.detect_scanbuild():
             return
@@ -3658,6 +3678,9 @@ def generate_clangtool(self, name: str, extra_arg: T.Optional[str] = None) -> No
         if extra_arg:
             target_name += f'-{extra_arg}'
             extra_args.append(f'--{extra_arg}')
+        colorout = self.environment.coredata.optstore.get_value('b_colorout') \
+            if OptionKey('b_colorout') in self.environment.coredata.optstore else 'always'
+        extra_args.extend(['--color', colorout])
         if not os.path.exists(os.path.join(self.environment.source_dir, '.clang-' + name)) and \
                 not os.path.exists(os.path.join(self.environment.source_dir, '_clang-' + name)):
             return
@@ -3703,6 +3726,7 @@ def generate_utils(self) -> None:
         self.generate_scanbuild()
         self.generate_clangformat()
         self.generate_clangtidy()
+        self.generate_clippy()
         self.generate_tags('etags', 'TAGS')
         self.generate_tags('ctags', 'ctags')
         self.generate_tags('cscope', 'cscope')

mesonbuild/compilers/detect.py

@@ -1042,6 +1042,10 @@ def detect_rust_compiler(env: 'Environment', for_machine: MachineChoice) -> Rust
             version = search_version(out)
 
             cls = rust.ClippyRustCompiler
+            mlog.deprecation(
+                'clippy-driver is not intended as a general purpose compiler. '
+                'You can use "ninja clippy" in order to run clippy on a '
+                'meson project.')
 
         if 'rustc' in out:
             # On Linux and mac rustc will invoke gcc (clang for mac

mesonbuild/compilers/mixins/gnu.py

@@ -8,7 +8,6 @@
 import abc
 import functools
 import os
-import multiprocessing
 import pathlib
 import re
 import subprocess
@@ -617,8 +616,9 @@ def get_lto_compile_args(self, *, threads: int = 0, mode: str = 'default') -> T.
         if threads == 0:
             if self._has_lto_auto_support:
                 return ['-flto=auto']
-            # This matches clang's behavior of using the number of cpus
-            return [f'-flto={multiprocessing.cpu_count()}']
+            # This matches clang's behavior of using the number of cpus, but
+            # obeying meson's MESON_NUM_PROCESSES convention.
+            return [f'-flto={mesonlib.determine_worker_count()}']
         elif threads > 0:
             return [f'-flto={threads}']
         return super().get_lto_compile_args(threads=threads)

mesonbuild/compilers/rust.py

@@ -33,6 +33,35 @@
     's': ['-C', 'opt-level=s'],
 }
 
+def get_rustup_run_and_args(exelist: T.List[str]) -> T.Optional[T.Tuple[T.List[str], T.List[str]]]:
+    """Given the command for a rustc executable, check if it is invoked via
+       "rustup run" and if so separate the "rustup [OPTIONS] run TOOLCHAIN"
+       part from the arguments to rustc.  If the returned value is not None,
+       other tools (for example clippy-driver or rustdoc) can be run by placing
+       the name of the tool between the two elements of the tuple."""
+    e = iter(exelist)
+    try:
+        if os.path.basename(next(e)) != 'rustup':
+            return None
+        # minimum three strings: "rustup run TOOLCHAIN"
+        n = 3
+        opt = next(e)
+
+        # options come first
+        while opt.startswith('-'):
+            n += 1
+            opt = next(e)
+
+        # then "run TOOLCHAIN"
+        if opt != 'run':
+            return None
+
+        next(e)
+        next(e)
+        return exelist[:n], list(e)
+    except StopIteration:
+        return None
+
 class RustCompiler(Compiler):
 
     # rustc doesn't invoke the compiler itself, it doesn't need a LINKER_PREFIX
@@ -65,6 +94,7 @@ def __init__(self, exelist: T.List[str], version: str, for_machine: MachineChoic
         super().__init__([], exelist, version, for_machine, info,
                          is_cross=is_cross, full_version=full_version,
                          linker=linker)
+        self.rustup_run_and_args: T.Optional[T.Tuple[T.List[str], T.List[str]]] = get_rustup_run_and_args(exelist)
         self.base_options.update({OptionKey(o) for o in ['b_colorout', 'b_ndebug']})
         if 'link' in self.linker.id:
             self.base_options.add(OptionKey('b_vscrt'))
@@ -245,6 +275,26 @@ def get_assert_args(self, disable: bool, env: 'Environment') -> T.List[str]:
         action = "no" if disable else "yes"
         return ['-C', f'debug-assertions={action}', '-C', 'overflow-checks=no']
 
+    def get_rust_tool(self, name: str, env: Environment) -> T.List[str]:
+        if self.rustup_run_and_args:
+            rustup_exelist, args = self.rustup_run_and_args
+            # do not use extend so that exelist is copied
+            exelist = rustup_exelist + [name]
+        else:
+            from ..programs import find_external_program
+            for prog in find_external_program(env, self.for_machine, name, name,
+                                              [name], allow_default_for_cross=False):
+                exelist = [prog.path]
+                args = self.exelist[1:]
+                break
+            else:
+                return []
+
+        tool = exelist[0]
+        if os.path.isfile(tool) and os.access(tool, os.X_OK):
+            return exelist + args
+        return []
+
 
 class ClippyRustCompiler(RustCompiler):
 

mesonbuild/mintro.py

@@ -211,6 +211,7 @@ def nodes_to_paths(node_list: T.List[BaseNode]) -> T.List[Path]:
             'build_by_default': i['build_by_default'],
             'target_sources': [{
                 'language': 'unknown',
+                'machine': i['machine'],
                 'compiler': [],
                 'parameters': [],
                 'sources': [str(x) for x in sources],

mesonbuild/mtest.py

@@ -14,7 +14,6 @@
 import datetime
 import enum
 import json
-import multiprocessing
 import os
 import pickle
 import platform
@@ -36,7 +35,8 @@
 from .coredata import MesonVersionMismatchException, major_versions_differ
 from .coredata import version as coredata_version
 from .mesonlib import (MesonException, OrderedSet, RealPathAction,
-                       get_wine_shortpath, join_args, split_args, setup_vsenv)
+                       get_wine_shortpath, join_args, split_args, setup_vsenv,
+                       determine_worker_count)
 from .options import OptionKey
 from .mintro import get_infodir, load_info_file
 from .programs import ExternalProgram
@@ -97,27 +97,6 @@ def uniwidth(s: str) -> int:
         result += UNIWIDTH_MAPPING[w]
     return result
 
-def determine_worker_count() -> int:
-    varname = 'MESON_TESTTHREADS'
-    num_workers = 0
-    if varname in os.environ:
-        try:
-            num_workers = int(os.environ[varname])
-            if num_workers < 0:
-                raise ValueError
-        except ValueError:
-            print(f'Invalid value in {varname}, using 1 thread.')
-            num_workers = 1
-
-    if num_workers == 0:
-        try:
-            # Fails in some weird environments such as Debian
-            # reproducible build.
-            num_workers = multiprocessing.cpu_count()
-        except Exception:
-            num_workers = 1
-    return num_workers
-
 # Note: when adding arguments, please also add them to the completion
 # scripts in $MESONSRC/data/shell-completions/
 def add_arguments(parser: argparse.ArgumentParser) -> None:
@@ -152,7 +131,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
                         help="Run benchmarks instead of tests.")
     parser.add_argument('--logbase', default='testlog',
                         help="Base name for log file.")
-    parser.add_argument('-j', '--num-processes', default=determine_worker_count(), type=int,
+    parser.add_argument('-j', '--num-processes', default=determine_worker_count(['MESON_TESTTHREADS']), type=int,
                         help='How many parallel processes to use.')
     parser.add_argument('-v', '--verbose', default=False, action='store_true',
                         help='Do not redirect stdout and stderr')