fix(python): pin mypy to the exact version

It's been a couple of times `mypy` breaks the pipeline by releasing new
versions that are more strict or start checking things it used to
silently ignored.

In order to avoid this happening in the future (and putting us back in
control), this pins the `mypy` version to the exact release (the last
we successfully built with, in this case), and moves everything we
install though `pip install` into `requirements-dev.txt` files so that
@dependabot can help us stay up-to-date going forward.

Fixes #2464

.github/dependabot.yml

@@ -5,7 +5,6 @@ updates:
     directory: '/'
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
     labels:
       - dependencies
     versioning-strategy: increase
@@ -21,7 +20,6 @@ updates:
     directory: '/packages/@jsii/dotnet-runtime/src'
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
     labels:
       - dependencies
       - language/dotnet
@@ -30,7 +28,6 @@ updates:
     directory: '/packages/@jsii/dotnet-runtime-test/test'
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
     labels:
       - dependencies
       - language/dotnet
@@ -39,7 +36,6 @@ updates:
     directory: '/packages/@jsii/python-runtime'
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
     labels:
       - dependencies
       - language/python
@@ -48,7 +44,22 @@ updates:
     directory: '/gh-pages'
     schedule:
       interval: daily
-    open-pull-requests-limit: 10
+    labels:
+      - dependencies
+      - language/python
+
+  - package-ecosystem: pip
+    directory: '/packages/jsii-pacmak/test/generated-code'
+    schedule:
+      interval: daily
+    labels:
+      - dependencies
+      - language/python
+
+  - package-ecosystem: pip
+    directory: '/packages/jsii-pacmak/lib/targets/python'
+    schedule:
+      interval: daily
     labels:
       - dependencies
       - language/python

packages/@jsii/python-runtime/requirements.txt

@@ -1,7 +1,8 @@
 black~=20.8b1
+mypy==0.782
+pip~=20.3
 pytest~=6.2
 pytest-mypy~=0.8
-pip~=20.3
 setuptools~=51.1
 wheel~=0.36
 

packages/jsii-pacmak/lib/targets/python.ts

@@ -36,7 +36,11 @@ import {
 // eslint-disable-next-line @typescript-eslint/no-var-requires,@typescript-eslint/no-require-imports
 const spdxLicenseList = require('spdx-license-list');
 
-const pythonBuildTools = ['setuptools~=49.3', 'wheel~=0.34'];
+const requirementsFile = path.resolve(
+  __dirname,
+  'python',
+  'requirements-dev.txt',
+);
 
 export default class Python extends Target {
   protected readonly generator: PythonGenerator;
@@ -74,7 +78,7 @@ export default class Python extends Target {
     // Install the necessary things
     await shell(
       python,
-      ['-m', 'pip', 'install', '--no-input', ...pythonBuildTools, 'twine~=3.2'],
+      ['-m', 'pip', 'install', '--no-input', '-r', requirementsFile],
       {
         cwd: sourceDir,
         env,
@@ -1873,9 +1877,15 @@ class Package {
     // TODO: Might be easier to just use a TOML library to write this out.
     code.openFile('pyproject.toml');
     code.line('[build-system]');
-    code.line(
-      `requires = [${pythonBuildTools.map((x) => `"${x}"`).join(', ')}]`,
-    );
+    const buildTools = fs
+      .readFileSync(requirementsFile, { encoding: 'utf-8' })
+      .split('\n')
+      .map((line) => /^\s*(.+)\s*#\s*build-system\s*$/.exec(line)?.[1]?.trim())
+      .reduce(
+        (buildTools, entry) => (entry ? [...buildTools, entry] : buildTools),
+        new Array<string>(),
+      );
+    code.line(`requires = [${buildTools.map((x) => `"${x}"`).join(', ')}]`);
     code.line('build-backend = "setuptools.build_meta"');
     code.closeFile('pyproject.toml');
 

packages/jsii-pacmak/lib/targets/python/requirements-dev.txt

@@ -0,0 +1,9 @@
+# All entries with "# build-system" at the end of the line will be hoisted as
+# part of the build-system declaration of generated packages. Others will only
+# be installed in the virtual environment used for building the distribution
+# package (wheel, sdist), but not declared as build-system dependencies.
+
+setuptools~=52.0.0 # build-system
+wheel~=0.36.0      # build-system
+
+twine~=3.3.0

packages/jsii-pacmak/test/generated-code/harness.ts

@@ -176,7 +176,8 @@ async function runMypy(pythonRoot: string): Promise<void> {
         'pip',
         'install',
         '--no-input',
-        '"mypy>=0.782"',
+        '-r',
+        path.resolve(__dirname, 'requirements-dev.txt'),
         // Note: this resolution is a little ugly, but it's there to avoid creating a dependency cycle
         JSON.stringify(
           path.resolve(

packages/jsii-pacmak/test/generated-code/requirements-dev.txt

@@ -0,0 +1 @@
+mypy==0.782