Merge pull request #2390 from obfusk/reproducible-builds

reproducible builds

Author: AndreMiras

pythonforandroid/archs.py

@@ -234,6 +234,12 @@ def get_env(self, with_flags_in_cc=True):
 
         env['PATH'] = environ['PATH']
 
+        # for reproducible builds
+        if 'SOURCE_DATE_EPOCH' in environ:
+            for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split():
+                if k in environ:
+                    env[k] = environ[k]
+
         return env
 
 

pythonforandroid/bootstraps/common/build/build.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
+from gzip import GzipFile
+import hashlib
 import json
 from os.path import (
     dirname, join, isfile, realpath,
     relpath, split, exists, basename
 )
-from os import listdir, makedirs, remove
+from os import environ, listdir, makedirs, remove
 import os
 import shlex
 import shutil
@@ -161,16 +163,25 @@ def select(fn):
             return False
         return not is_blacklist(fn)
 
+    def clean(tinfo):
+        """cleaning function (for reproducible builds)"""
+        tinfo.uid = tinfo.gid = 0
+        tinfo.uname = tinfo.gname = ''
+        tinfo.mtime = 0
+        return tinfo
+
     # get the files and relpath file of all the directory we asked for
     files = []
     for sd in source_dirs:
         sd = realpath(sd)
         compile_dir(sd, optimize_python=optimize_python)
         files += [(x, relpath(realpath(x), sd)) for x in listfiles(sd)
                   if select(x)]
+    files.sort()  # deterministic
 
     # create tar.gz of thoses files
-    tf = tarfile.open(tfn, 'w:gz', format=tarfile.USTAR_FORMAT)
+    gf = GzipFile(tfn, 'wb', mtime=0)  # deterministic
+    tf = tarfile.open(None, 'w', gf, format=tarfile.USTAR_FORMAT)
     dirs = []
     for fn, afn in files:
         dn = dirname(afn)
@@ -189,8 +200,9 @@ def select(fn):
                 tf.addfile(tinfo)
 
         # put the file
-        tf.add(fn, afn)
+        tf.add(fn, afn, filter=clean)
     tf.close()
+    gf.close()
 
 
 def compile_dir(dfn, optimize_python=True):
@@ -521,9 +533,18 @@ def make_package(args):
         versioned_name=versioned_name)
 
     # String resources:
+    timestamp = time.time()
+    if 'SOURCE_DATE_EPOCH' in environ:
+        # for reproducible builds
+        timestamp = int(environ['SOURCE_DATE_EPOCH'])
+    private_version = "{} {} {}".format(
+        args.version,
+        args.numeric_version,
+        timestamp
+    )
     render_args = {
         "args": args,
-        "private_version": str(time.time())
+        "private_version": hashlib.sha1(private_version.encode()).hexdigest()
     }
     if get_bootstrap_name() == "sdl2":
         render_args["url_scheme"] = url_scheme

pythonforandroid/recipes/python3/__init__.py

@@ -3,7 +3,7 @@
 import subprocess
 
 from multiprocessing import cpu_count
-from os import environ
+from os import environ, utime
 from os.path import dirname, exists, join
 from pathlib import Path
 from shutil import copy2
@@ -62,6 +62,7 @@ class Python3Recipe(TargetPythonRecipe):
 
     patches = [
         'patches/pyconfig_detection.patch',
+        'patches/reproducible-buildinfo.diff',
 
         # Python 3.7.1
         ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")),
@@ -387,8 +388,14 @@ def create_python_bundle(self, dirn, arch):
         with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
             stdlib_filens = list(walk_valid_filens(
                 '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist))
+            if 'SOURCE_DATE_EPOCH' in environ:
+                # for reproducible builds
+                stdlib_filens.sort()
+                timestamp = int(environ['SOURCE_DATE_EPOCH'])
+                for filen in stdlib_filens:
+                    utime(filen, (timestamp, timestamp))
             info("Zip {} files into the bundle".format(len(stdlib_filens)))
-            shprint(sh.zip, stdlib_zip, *stdlib_filens)
+            shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens)
 
         # copy the site-packages into place
         ensure_dir(join(dirn, 'site-packages'))

pythonforandroid/recipes/python3/patches/reproducible-buildinfo.diff

@@ -0,0 +1,13 @@
+# DP: Build getbuildinfo.o with DATE/TIME values when defined
+
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -785,6 +785,8 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
+ 	      -DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
+ 	      -DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
+ 	      -DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
++	      $(if $(BUILD_DATE),-DDATE='"$(BUILD_DATE)"') \
++	      $(if $(BUILD_TIME),-DTIME='"$(BUILD_TIME)"') \
+ 	      -o $@ $(srcdir)/Modules/getbuildinfo.c
+ 
+ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile