Skip to content

Enable autopxd to use Microsoft Visual C++ for preprocessing on Windows #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 3, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -18,15 +18,21 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
python-version: ['3.6', '3.7', '3.8', '3.9']
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
exclude:
- os: macos-latest
python-version: '3.6'
- os: macos-latest
python-version: '3.7'
- os: macos-latest
python-version: '3.8'
- os: windows-latest
python-version: '3.6'
- os: windows-latest
python-version: '3.7'
- os: windows-latest
python-version: '3.8'

steps:
- name: Checkout code
@@ -40,7 +46,15 @@ jobs:

- name: Test
shell: bash
if: ${{ matrix.os != 'windows-latest' }}
run: |
set -uexo pipefail
pip install -e file:///$(pwd)#egg=autopxd2[dev]
pytest

- name: Test_Windows
shell: powershell
if: ${{ matrix.os == 'windows-latest' }}
run: |
pip install -e .[dev]
pytest
86 changes: 85 additions & 1 deletion autopxd/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import json
import os
import platform
import re
import subprocess
import sys
import tempfile

import click
from pycparser import (
@@ -34,11 +36,93 @@ def ensure_binary(s, encoding="utf-8", errors="strict"):
raise TypeError(f"not expecting type '{type(s)}'")


def _find_cl():
"""Use vswhere.exe to locate the Microsoft C compiler."""
host_platform = {
"X86": "X86",
"AMD64": "X64",
"ARM64": "ARM64",
}.get(platform.machine(), "X86")
build_platform = {
"X86": "x86",
"AMD64": "x64",
"ARM64": "arm64",
}.get(platform.machine(), "x86")
program_files = os.getenv("ProgramFiles(x86)") or os.getenv("ProgramFiles")
cmd = [
os.path.join(program_files, r"Microsoft Visual Studio\Installer\vswhere.exe"),
"-prerelease",
"-latest",
"-format",
"json",
"-utf8",
"-find",
rf"**\bin\Host{host_platform}\{build_platform}\cl.exe",
]
try:
return json.loads(subprocess.check_output(cmd, encoding="utf-8"))[-1]
except (OSError, subprocess.CalledProcessError, IndexError) as ex:
raise RuntimeError("No suitable compiler available") from ex


def _preprocess_msvc(code, extra_cpp_args, debug):
fd, source_file = tempfile.mkstemp(suffix=".c")
os.close(fd)
with open(source_file, "wb") as f:
f.write(ensure_binary(code))

result = []
try:
cmd = [
_find_cl(),
"/E",
"/X",
"/utf-8",
"/D__attribute__(x)=",
"/D__extension__=",
"/D__inline=",
"/D__asm=",
f"/I{BUILTIN_HEADERS_DIR}",
*(extra_cpp_args or []),
source_file,
]
with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc:
while proc.poll() is None:
result.extend(proc.communicate()[0].decode("utf-8").splitlines())
finally:
os.unlink(source_file)
if proc.returncode:
raise RuntimeError("Invoking C preprocessor failed")

# Normalise the paths in #line pragmas so that they are correctly matched
# later on
def fix_path(match):
file = match.group(1).replace("\\\\", "\\")
if file == source_file:
file = "<stdin>"
return f'"{file}"'

res = "\n".join(re.sub(r'"(.+?)"', fix_path, line) for line in result)

if debug:
sys.stderr.write(res)
return res


def preprocess(code, extra_cpp_args=None, debug=False):
if extra_cpp_args is None:
extra_cpp_args = []
if platform.system() == "Darwin":
cmd = ["clang", "-E", f"-I{DARWIN_HEADERS_DIR}"]
elif platform.system() == "Windows":
# Since Windows may not have GCC installed, we check for a cpp command
# first and if it does not run, then use our MSVC implementation
try:
subprocess.check_call(["cpp", "--version"])
except (OSError, subprocess.CalledProcessError):
return _preprocess_msvc(code, extra_cpp_args, debug)
else:
cmd = ["cpp"]
else:
cmd = ["cpp"]
cmd += (
@@ -85,7 +169,7 @@ def parse(code, extra_cpp_args=None, whitelist=None, debug=False, regex=None):
decls = []
for decl in ast.ext:
if not hasattr(decl, "name") or decl.name not in IGNORE_DECLARATIONS:
if not whitelist or decl.coord.file in whitelist:
if not whitelist or os.path.normpath(decl.coord.file) in whitelist:
decls.append(decl)
ast.ext = decls
return ast
24 changes: 22 additions & 2 deletions test/test_autopxd.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import glob
import os
import re
import sys

import pytest

@@ -9,8 +10,7 @@
FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")


@pytest.mark.parametrize("file_path", glob.iglob(os.path.abspath(os.path.join(FILES_DIR, "*.test"))))
def test_cython_vs_header(file_path):
def do_one_cython_vs_header_test(file_path):
with open(file_path, encoding="utf-8") as f:
data = f.read()
c, cython = re.split("^-+$", data, maxsplit=1, flags=re.MULTILINE)
@@ -34,5 +34,25 @@ def test_cython_vs_header(file_path):
assert cython == actual, f"\nCYTHON:\n{cython}\n\n\nACTUAL:\n{actual}"


@pytest.mark.parametrize("file_path", glob.iglob(os.path.abspath(os.path.join(FILES_DIR, "*.test"))))
def test_cython_vs_header(file_path):
do_one_cython_vs_header_test(file_path)


@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
@pytest.mark.parametrize("file_path", glob.iglob(os.path.abspath(os.path.join(FILES_DIR, "*.test"))))
def test_cython_vs_header_with_msvc(file_path, monkeypatch):
monkeypatch.setattr(autopxd, "preprocess", autopxd._preprocess_msvc) # pylint: disable=protected-access
test_cython_vs_header(file_path)


@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only test")
def test_find_cl():
# In failure cases, this will raise
cl = autopxd._find_cl() # pylint: disable=protected-access
# In success cases, we should have a file path
assert os.path.isfile(cl)


if __name__ == "__main__":
pytest.main([__file__])