Test Python Wheels (#378)

Install the generated python wheels, and test them out across a range of different python versions in github actions

.github/workflows/build.yml

@@ -24,6 +24,7 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
@@ -62,6 +63,7 @@ jobs:
   build-linux-cross:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         target: [i686-musl, armv7-musleabihf, aarch64-musl, x86_64-musl]
     container:
@@ -79,11 +81,40 @@ jobs:
         name: wheels
         path: dist
 
+  test_wheels:
+    name: Test Wheels
+    needs: [build, build-linux-cross]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [2.7.17, 2.7.18, 3.5.4, 3.5.9, 3.5.10, 3.6.7, 3.6.8, 3.6.9, 3.6.10, 3.6.11, 3.6.12, 3.6.13, 3.7.1, 3.7.5, 3.7.6, 3.7.7, 3.7.8, 3.7.9, 3.7.10, 3.8.0, 3.8.1, 3.8.2, 3.8.3, 3.8.4, 3.8.5, 3.8.6, 3.8.7, 3.8.8, 3.8.9, 3.9.0, 3.9.1, 3.9.2, 3.9.3, 3.9.4]
+        # TODO: also test windows
+        os: [ubuntu-latest, macos-latest]
+    steps: 
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheels
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install wheel
+        run: |
+          pip install --no-index --find-links . py-spy
+      - name: Test Wheel
+        run: python tests/integration_test.py
+        if: runner.os != 'macOS'
+      - name: Test macOS Wheel
+        run: sudo "PATH=$PATH" python tests/integration_test.py
+        if: runner.os == 'macOS'
+
+
   release:
     name: Release
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [build-linux-cross, build]
+    needs: [test_wheels]
     steps:
       - uses: actions/download-artifact@v2
         with:

ci/update_python_test_versions.py

@@ -0,0 +1,43 @@
+import requests
+import pkg_resources
+import pathlib
+
+
+_VERSIONS_URL = "https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json"  # noqa
+
+
+def get_github_python_versions():
+    versions_json = requests.get(_VERSIONS_URL).json()
+    raw_versions = [v["version"] for v in versions_json]
+    versions = []
+    for version_str in raw_versions:
+        if '-' in version_str:
+            continue
+
+        v = pkg_resources.parse_version(version_str)
+        if v.major == 3 and v.minor < 5:
+            # we don't support python 3.0/3.1/3.2 , and don't bother testing 3.3/3.4
+            continue
+
+        elif v.major == 2 and v.minor < 7:
+            # we don't test python support before 2.7
+            continue
+
+        versions.append(version_str)
+    return versions
+
+
+if __name__ == "__main__":
+    versions = sorted(get_github_python_versions(), key = lambda x: pkg_resources.parse_version(x))
+    build_yml = pathlib.Path(__file__).parent.parent / ".github" / "workflows" / "build.yml"
+
+    transformed = []
+    for line in open(build_yml):
+        if line.startswith("        python-version: ["):
+            print(line)
+            line = f"        python-version: [{', '.join(v for v in versions)}]\n"
+            print(line)
+        transformed.append(line)
+
+    with open(build_yml, "w") as o:
+        o.write("".join(transformed))

src/python_bindings/mod.rs

@@ -70,8 +70,8 @@ pub mod pyruntime {
                     _ => Some(1416),
                 }
              },
-             Version{major: 3, minor: 8, patch: 1..=7, ..} => { Some(1416) },
-             Version{major: 3, minor: 9, patch: 0..=1, ..} => { Some(616) },
+             Version{major: 3, minor: 8, patch: 1..=9, ..} => { Some(1416) },
+             Version{major: 3, minor: 9, patch: 0..=4, ..} => { Some(616) },
              _ => None
         }
     }
@@ -151,8 +151,8 @@ pub mod pyruntime {
                     _ => Some(1296)
                 }
             },
-            Version{major: 3, minor: 8, patch: 1..=7, ..} => Some(1296),
-            Version{major: 3, minor: 9, patch: 0..=1, ..} => Some(496),
+            Version{major: 3, minor: 8, patch: 1..=9, ..} => Some(1296),
+            Version{major: 3, minor: 9, patch: 0..=4, ..} => Some(496),
             _ => None
         }
     }
@@ -170,8 +170,8 @@ pub mod pyruntime {
                     _ => Some(1224)
                 }
             },
-            Version{major: 3, minor: 8, patch: 1..=7, ..} => Some(1224),
-            Version{major: 3, minor: 9, patch: 0..=1, ..} => Some(424),
+            Version{major: 3, minor: 8, patch: 1..=9, ..} => Some(1224),
+            Version{major: 3, minor: 9, patch: 0..=4, ..} => Some(424),
             _ => None
         }
     }

tests/integration_test.py

@@ -0,0 +1,85 @@
+import json
+import subprocess
+import sys
+import unittest
+import tempfile
+import os
+from collections import defaultdict, namedtuple
+from distutils.spawn import find_executable
+
+
+Frame = namedtuple("Frame", ["file", "name", "line", "col"])
+
+# disable gil checks on windows - just rely on active
+# (doesn't seem to be working quite right - TODO: investigate)
+GIL = ["--gil"] if not sys.platform.startswith("win") else []
+
+
+class TestPyspy(unittest.TestCase):
+    """ Basic tests of using py-spy as a commandline application """
+    def _sample_process(self, script_name, options=None):
+        pyspy = find_executable("py-spy")
+        print("Testing py-spy @", pyspy)
+
+        # for permissions reasons, we really want to run the sampled python process as a
+        # subprocess of the py-spy (works best on linux etc). So we're running the 
+        # record option, and setting different flags. To get the profile output
+        # we're using the speedscope format (since we can read that in as json)
+        with tempfile.NamedTemporaryFile() as profile_file:
+            cmdline = [
+                pyspy,
+                "record",
+                "-o",
+                profile_file.name,
+                "--format",
+                "speedscope",
+                "-d",
+                "1",
+            ]
+            cmdline.extend(options or [])
+            cmdline.extend(["--", sys.executable, script_name])
+
+            subprocess.check_call(cmdline)
+            with open(profile_file.name) as f:
+                profiles = json.load(f)
+
+        frames = profiles["shared"]["frames"]
+        samples = defaultdict(int)
+        for p in profiles["profiles"]:
+            for sample in p["samples"]:
+                samples[tuple(Frame(**frames[frame]) for frame in sample)] += 1
+        return samples
+
+    def test_longsleep(self):
+        # running with the gil flag should have ~ no samples returned
+        profile = self._sample_process(_get_script("longsleep.py"), GIL)
+        assert sum(profile.values()) <= 5
+
+        # running with the idle flag should have > 95%  of samples in the sleep call
+        profile = self._sample_process(_get_script("longsleep.py"), ["--idle"])
+        sample, count = _most_frequent_sample(profile)
+        assert count >= 95
+        assert len(sample) == 2
+        assert sample[0].name == "<module>"
+        assert sample[0].line == 9
+        assert sample[1].name == "longsleep"
+        assert sample[1].line == 5
+
+    def test_busyloop(self):
+        # can't be sure what line we're on, but we should have ~ all samples holding the gil
+        profile = self._sample_process(_get_script("busyloop.py"), GIL)
+        print(profile)
+        assert sum(profile.values()) >= 95
+
+
+
+def _get_script(name):
+    base_dir = os.path.dirname(__file__)
+    return os.path.join(base_dir, "scripts", name)
+
+
+def _most_frequent_sample(samples):
+    return max(samples.items(), key=lambda x: x[1])
+
+if __name__ == "__main__":
+    unittest.main()