build: use cargo messages to locate built cdylib

CHANGELOG.md

@@ -1,12 +1,15 @@
 # Changelog
 
-## 1.4.1 (2022-07-05)
+## Unreleased
+### Changed
+- Locate cdylib artifacts by handling messages from cargo instead of searching target dir (fixes build on MSYS2). [#267](https://github.com/PyO3/setuptools-rust/pull/267)
+
 
+## 1.4.1 (2022-07-05)
 ### Fixed
 - Fix crash when checking Rust version. [#263](https://github.com/PyO3/setuptools-rust/pull/263)
 
 ## 1.4.0 (2022-07-05)
-
 ### Packaging
 - Increase minimum `setuptools` version to 62.4. [#246](https://github.com/PyO3/setuptools-rust/pull/246)
 
@@ -25,7 +28,6 @@
 - If the sysconfig for `BLDSHARED` has no flags, `setuptools-rust` won't crash anymore. [#241](https://github.com/PyO3/setuptools-rust/pull/241)
 
 ## 1.3.0 (2022-04-26)
-
 ### Packaging
 - Increase minimum `setuptools` version to 58. [#222](https://github.com/PyO3/setuptools-rust/pull/222)
 

examples/namespace_package/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2018"
 
 [lib]
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 pyo3 = { version = "0.16.5", features = ["extension-module"] }

setuptools_rust/_utils.py

@@ -1,9 +1,17 @@
 import subprocess
 
 
-def format_called_process_error(e: subprocess.CalledProcessError) -> str:
+def format_called_process_error(
+ e: subprocess.CalledProcessError,
+ *,
+ include_stdout: bool = True,
+ include_stderr: bool = True,
+) -> str:
  """Helper to convert a CalledProcessError to an error message.
 
+ If `include_stdout` or `include_stderr` are True (the default), the
+ respective output stream will be added to the error message (if
+ present in the exception).
 
  >>> format_called_process_error(subprocess.CalledProcessError(
  ... 777, ['ls', '-la'], None, None
@@ -14,18 +22,22 @@ def format_called_process_error(e: subprocess.CalledProcessError) -> str:
  ... ))
  "`cargo 'foo bar'` failed with code 1\\n-- Output captured from stdout:\\nmessage"
  >>> format_called_process_error(subprocess.CalledProcessError(
+ ... 1, ['cargo', 'foo bar'], 'message', None
+ ... ), include_stdout=False)
+ "`cargo 'foo bar'` failed with code 1"
+ >>> format_called_process_error(subprocess.CalledProcessError(
  ... -1, ['cargo'], 'stdout', 'stderr'
  ... ))
  '`cargo` failed with code -1\\n-- Output captured from stdout:\\nstdout\\n-- Output captured from stderr:\\nstderr'
  """
  command = " ".join(_quote_whitespace(arg) for arg in e.cmd)
  message = f"`{command}` failed with code {e.returncode}"
- if e.stdout is not None:
+ if include_stdout and e.stdout is not None:
  message += f"""
 -- Output captured from stdout:
 {e.stdout}"""
 
- if e.stderr is not None:
+ if include_stderr and e.stderr is not None:
  message += f"""
 -- Output captured from stderr:
 {e.stderr}"""

setuptools_rust/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import glob
+import json
 import os
 import platform
 import shutil
@@ -15,7 +16,7 @@
  DistutilsPlatformError,
 )
 from distutils.sysconfig import get_config_var
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple, cast
+from typing import Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, cast
 
 from setuptools.command.build import build as CommandBuild # type: ignore[import]
 from setuptools.command.build_ext import build_ext as CommandBuildExt
@@ -184,6 +185,7 @@ def build_extension(
  self.cargo,
  "rustc",
  "--lib",
+ "--message-format=json-render-diagnostics",
  "--manifest-path",
  ext.path,
  *cargo_args,
@@ -209,13 +211,17 @@ def build_extension(
  try:
  # If quiet, capture all output and only show it in the exception
  # If not quiet, forward all cargo output to stderr
- stdout = subprocess.PIPE if quiet else sys.stderr.fileno()
  stderr = subprocess.PIPE if quiet else None
- subprocess.run(
- command, env=env, stdout=stdout, stderr=stderr, text=True, check=True
+ cargo_messages = subprocess.check_output(
+ command,
+ env=env,
+ stderr=stderr,
+ text=True,
  )
  except subprocess.CalledProcessError as e:
- raise CompileError(format_called_process_error(e))
+ # Don't include stdout in the formatted error as it is a huge dump
+ # of cargo json lines which aren't helpful for the end user.
+ raise CompileError(format_called_process_error(e, include_stdout=False))
 
  except OSError:
  raise DistutilsExecError(
@@ -257,29 +263,25 @@ def build_extension(
  f"unable to find executable '{name}' in '{artifacts_dir}'"
  )
  else:
- platform = sysconfig.get_platform()
- if "win" in platform:
- dylib_ext = "dll"
- elif platform.startswith("macosx"):
- dylib_ext = "dylib"
- elif "wasm32" in platform:
- dylib_ext = "wasm"
- else:
- dylib_ext = "so"
-
- wildcard_so = "*{}.{}".format(ext.get_lib_name(quiet=quiet), dylib_ext)
-
- try:
- dylib_paths.append(
- _BuiltModule(
- ext.name,
- next(glob.iglob(os.path.join(artifacts_dir, wildcard_so))),
- )
+ # Find artifact from cargo messages
+ artifacts = tuple(
+ _find_cargo_artifacts(
+ cargo_messages.splitlines(),
+ manifest_path=os.path.abspath(ext.path),
+ kind="cdylib",
+ )
+ )
+ if len(artifacts) == 0:
+ raise DistutilsExecError(
+ "Rust build failed; unable to find any build artifacts"
  )
- except StopIteration:
+ elif len(artifacts) > 1:
  raise DistutilsExecError(
- f"Rust build failed; unable to find any {wildcard_so} in {artifacts_dir}"
+ f"Rust build failed; expected only one build artifact but found {artifacts}"
  )
+
+ # guaranteed to be just one element after checks above
+ dylib_paths.append(_BuiltModule(ext.name, artifacts[0]))
  return dylib_paths
 
  def install_extension(
@@ -771,3 +773,53 @@ def _split_platform_and_extension(ext_path: str) -> Tuple[str, str, str]:
  # rust.cpython-38-x86_64-linux-gnu to (rust, .cpython-38-x86_64-linux-gnu)
  ext_path, platform_tag = os.path.splitext(ext_path)
  return (ext_path, platform_tag, extension)
+
+
+def _find_cargo_artifacts(
+ cargo_messages: List[str],
+ *,
+ manifest_path: str,
+ kind: str,
+) -> Iterable[str]:
+ """Identifies cargo artifacts built for the given `manifest_path` from the
+ provided cargo_messages.
+
+ >>> list(_find_cargo_artifacts(
+ ... [
+ ... '{"some_irrelevant_message": []}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/foo/bar/Cargo.toml","target":{"kind":["cdylib"]},"filenames":["/some/path/baz.so"]}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/foo/bar/Cargo.toml","target":{"kind":["cdylib", "rlib"]},"filenames":["/file/two/baz.dylib", "/file/two/baz.rlib"]}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/other/path/Cargo.toml","target":{"kind":["cdylib"]},"filenames":["/not/this.so"]}',
+ ... ],
+ ... manifest_path="/foo/bar/Cargo.toml",
+ ... kind="cdylib",
+ ... ))
+ ['/some/path/baz.so', '/file/two/baz.dylib']
+ >>> list(_find_cargo_artifacts(
+ ... [
+ ... '{"some_irrelevant_message": []}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/foo/bar/Cargo.toml","target":{"kind":["cdylib"]},"filenames":["/some/path/baz.so"]}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/foo/bar/Cargo.toml","target":{"kind":["cdylib", "rlib"]},"filenames":["/file/two/baz.dylib", "/file/two/baz.rlib"]}',
+ ... '{"reason":"compiler-artifact","manifest_path":"/other/path/Cargo.toml","target":{"kind":["cdylib"]},"filenames":["/not/this.so"]}',
+ ... ],
+ ... manifest_path="/foo/bar/Cargo.toml",
+ ... kind="rlib",
+ ... ))
+ ['/file/two/baz.rlib']
+ """
+ for message in cargo_messages:
+ # only bother parsing messages that look like a match
+ # (don't bother checking for manifest_path yet because \\ in json paths
+ # on windows makes it a bit fiddly)
+ if "compiler-artifact" in message and kind in message:
+ parsed = json.loads(message)
+ # verify the message is correct
+ if (
+ parsed.get("reason") == "compiler-artifact"
+ and parsed.get("manifest_path") == manifest_path
+ ):
+ for artifact_kind, filename in zip(
+ parsed["target"]["kind"], parsed["filenames"]
+ ):
+ if artifact_kind == kind:
+ yield filename