Merge pull request #256 from joerick/deterministic-builds

Deterministic builds

.circleci/config.yml

@@ -13,6 +13,7 @@ jobs:
       - run:
           name: Test.
           command: flake8 .
+
   osx-python3.6:
     macos:
       xcode: "9.4.1"
@@ -27,6 +28,7 @@ jobs:
       - run:
           name: Test.
           command: venv/bin/python ./bin/run_tests.py
+          no_output_timeout: 30m
 
   osx-python3.7:
     macos:
@@ -42,6 +44,7 @@ jobs:
       - run:
           name: Test.
           command: venv/bin/python ./bin/run_tests.py
+          no_output_timeout: 30m
 
   linux-python3.6:
     docker:
@@ -58,6 +61,7 @@ jobs:
       - run:
           name: Test.
           command: venv/bin/python ./bin/run_tests.py
+          no_output_timeout: 30m
 
 workflows:
   version: 2

.gitignore

@@ -89,8 +89,10 @@ venv3/
 venv2/
 ENV/
 env/
-env3/
 env2/
+env2?/
+env3/
+env3?/
 
 # Spyder project settings
 .spyderproject

LICENSE

@@ -22,3 +22,7 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Get-pip and Pip (bundled in cibuildwheel/resources) are licensed under the MIT
+license. See https://github.com/pypa/get-pip/blob/master/LICENSE.txt

bin/update_constraints.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+
+import configparser
+import os
+import subprocess
+from collections import namedtuple
+
+import requests
+
+os.chdir(os.path.dirname(__file__))
+os.chdir('..')
+
+# CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to
+# regenerate the constraints files
+os.environ['CUSTOM_COMPILE_COMMAND'] = "bin/update_constraints.py"
+subprocess.check_call([
+    'pip-compile',
+    '--allow-unsafe',
+    '--upgrade',
+    'cibuildwheel/resources/constraints.in',
+])
+for python_version in ['27', '35', '36']:
+    subprocess.check_call([
+        f'./env{python_version}/bin/pip-compile',
+        '--allow-unsafe',
+        '--upgrade',
+        'cibuildwheel/resources/constraints.in',
+        '--output-file', f'cibuildwheel/resources/constraints-python{python_version}.txt'
+    ])
+
+Image = namedtuple('Image', [
+    'manylinux_version',
+    'platform',
+    'image_name',
+])
+
+images = [
+    Image('manylinux1', 'x86_64', 'quay.io/pypa/manylinux1_x86_64'),
+    Image('manylinux1', 'i686', 'quay.io/pypa/manylinux1_i686'),
+
+    Image('manylinux2010', 'x86_64', 'quay.io/pypa/manylinux2010_x86_64'),
+    Image('manylinux2010', 'i686', 'quay.io/pypa/manylinux2010_i686'),
+    Image('manylinux2010', 'pypy_x86_64', 'pypywheels/manylinux2010-pypy_x86_64'),
+
+    Image('manylinux2014', 'x86_64', 'quay.io/pypa/manylinux2014_x86_64'),
+    Image('manylinux2014', 'i686', 'quay.io/pypa/manylinux2014_i686'),
+    Image('manylinux2014', 'aarch64', 'quay.io/pypa/manylinux2014_aarch64'),
+    Image('manylinux2014', 'ppc64le', 'quay.io/pypa/manylinux2014_ppc64le'),
+    Image('manylinux2014', 's390x', 'quay.io/pypa/manylinux2014_s390x'),
+]
+
+config = configparser.ConfigParser()
+
+for image in images:
+    # get the tag name whose digest matches 'latest'
+    if image.image_name.startswith('quay.io/'):
+        _, _, repository_name = image.image_name.partition('/')
+        response = requests.get(
+            f'https://quay.io/api/v1/repository/{repository_name}?includeTags=true'
+        )
+        response.raise_for_status()
+        repo_info = response.json()
+        tags_dict = repo_info['tags']
+
+        latest_tag = tags_dict.pop('latest')
+        # find the tag whose manifest matches 'latest'
+        tag_name = next(
+            name
+            for (name, info) in tags_dict.items()
+            if info['manifest_digest'] == latest_tag['manifest_digest']
+        )
+    else:
+        response = requests.get(
+            f'https://hub.docker.com/v2/repositories/{image.image_name}/tags'
+        )
+        response.raise_for_status()
+        tags = response.json()['results']
+
+        latest_tag = next(
+            tag for tag in tags if tag['name'] == 'latest'
+        )
+        # i don't know what it would mean to have multiple images per tag
+        assert len(latest_tag['images']) == 1
+        digest = latest_tag['images'][0]['digest']
+
+        pinned_tag = next(
+            tag for tag in tags if tag['images'][0]['digest'] == digest
+        )
+        tag_name = pinned_tag['name']
+
+    if not config.has_section(image.platform):
+        config[image.platform] = {}
+
+    config[image.platform][image.manylinux_version] = f'{image.image_name}:{tag_name}'
+
+with open('cibuildwheel/resources/pinned_docker_images.cfg', 'w') as f:
+    config.write(f)

cibuildwheel/__main__.py

@@ -3,6 +3,7 @@
 import sys
 import textwrap
 import traceback
+from configparser import ConfigParser
 
 import cibuildwheel
 import cibuildwheel.linux
@@ -14,6 +15,7 @@
 )
 from cibuildwheel.util import (
     BuildSelector,
+    DependencyConstraints,
     Unbuffered
 )
 
@@ -115,6 +117,14 @@ def main():
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
     before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform, default='')
 
+    dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned')
+    if dependency_versions == 'pinned':
+        dependency_constraints = DependencyConstraints.with_defaults()
+    elif dependency_versions == 'latest':
+        dependency_constraints = None
+    else:
+        dependency_constraints = DependencyConstraints(dependency_versions)
+
     if test_extras:
         test_extras = '[{0}]'.format(test_extras)
 
@@ -155,41 +165,43 @@ def main():
         build_selector=build_selector,
         repair_command=repair_command,
         environment=environment,
-        before_test=before_test
+        before_test=before_test,
+        dependency_constraints=dependency_constraints,
     )
 
     if platform == 'linux':
-        manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
-        manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
-        manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010')
-        manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014')
-        manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014')
-        manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014')
-
-        default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64',
-                                           'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64',
-                                           'manylinux2014': 'quay.io/pypa/manylinux2014_x86_64'}
-        default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686',
-                                         'manylinux2010': 'quay.io/pypa/manylinux2010_i686',
-                                         'manylinux2014': 'quay.io/pypa/manylinux2014_i686'}
-        default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'}
-        default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'}
-        default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'}
-        default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'}
+        pinned_docker_images_file = os.path.join(
+            os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
+        )
+        all_pinned_docker_images = ConfigParser()
+        all_pinned_docker_images.read(pinned_docker_images_file)
+        # all_pinned_docker_images looks like a dict of dicts, e.g.
+        # { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
+        #   'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
+        #   'pypy_x86_64': {'manylinux2010': '...' }
+        #   ... }
+
+        manylinux_images = {}
+
+        for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']:
+            pinned_images = all_pinned_docker_images[build_platform]
+
+            config_name = 'CIBW_MANYLINUX_{}_IMAGE'.format(build_platform.upper())
+            config_value = os.environ.get(config_name)
+
+            if config_value is None:
+                # default to manylinux2010 if it's available, otherwise manylinux2014
+                image = pinned_images.get('manylinux2010') or pinned_images.get('manylinux2014')
+            elif config_value in pinned_images:
+                image = pinned_images[config_value]
+            else:
+                image = config_value
+
+            manylinux_images[build_platform] = image
 
         build_options.update(
-            manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image,
-                              'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image,
-                              'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image,
-                              'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image,
-                              'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image,
-                              's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image,
-                              },
+            manylinux_images=manylinux_images
         )
-    elif platform == 'macos':
-        pass
-    elif platform == 'windows':
-        pass
 
     # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
     sys.stdout = Unbuffered(sys.stdout)