diff --git a/README.md b/README.md index 0504c58765..82f7c1d9d8 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,12 @@ require to target to a **minimum api level of 21**, api level below 21, you must use an old version of python-for-android (<=0.7.1). All this work has been done using android ndk version r17c, and your build should success with that version...but be aware that the -project is in constant development so...the ndk version will change at -some time. +project is in constant development so, as per time of writing, +``we recommend to use android's NDK r19b`` because the toolchains installed by +default with the NDK may be used *in-place* and the python-for-android project +has been adapted to that feature, so, it's mandatory to use, at least, ndk +version r19 (be aware that more modern versions of the +android's ndk may not work). Those mentioned changes has been done this way to make easier the transition between python3 and python2. We will slowly phase out python2 support diff --git a/ci/makefiles/android.mk b/ci/makefiles/android.mk index 44d705838f..d2d4648007 100644 --- a/ci/makefiles/android.mk +++ b/ci/makefiles/android.mk @@ -6,7 +6,7 @@ ifndef target_os endif # Those android NDK/SDK variables can be override when running the file -ANDROID_NDK_VERSION ?= 17c +ANDROID_NDK_VERSION ?= 19b ANDROID_SDK_TOOLS_VERSION ?= 4333796 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 28.0.2 ANDROID_HOME ?= $(HOME)/.android diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 74161d7d8e..b70558d4c2 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -110,7 +110,7 @@ named ``tools``, and you will need to run extra commands to install the SDK packages needed. For Android NDK, note that modern releases will only work on a 64-bit -operating system. **The minimal, and recommended, NDK version to use is r17c:** +operating system. **The minimal, and recommended, NDK version to use is r19b:** - `Go to ndk downloads page `_ - Windows users should create a virtual machine with an GNU Linux os @@ -144,7 +144,7 @@ variables necessary for building on android:: # Adjust the paths! export ANDROIDSDK="$HOME/Documents/android-sdk-27" - export ANDROIDNDK="$HOME/Documents/android-ndk-r17c" + export ANDROIDNDK="$HOME/Documents/android-ndk-r19b" export ANDROIDAPI="27" # Target API version of your application export NDKAPI="21" # Minimum supported API version of your application export ANDROIDNDKVER="r10e" # Version of the NDK you installed diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index 42f143ed21..bf716db3b5 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -1,6 +1,7 @@ from distutils.spawn import find_executable from os import environ -from os.path import (exists, join, dirname, split) +from os.path import join, split +from multiprocessing import cpu_count from glob import glob from pythonforandroid.recipe import Recipe @@ -15,6 +16,35 @@ class Arch(object): command_prefix = None '''The prefix for NDK commands such as gcc.''' + arch = "" + '''Name of the arch such as: `armeabi-v7a`, `arm64-v8a`, `x86`...''' + + arch_cflags = [] + '''Specific arch `cflags`, expect to be overwrote in subclass if needed.''' + + common_cflags = [ + '-target {target}', + '-fomit-frame-pointer' + ] + + common_cppflags = [ + '-DANDROID', + '-D__ANDROID_API__={ctx.ndk_api}', + '-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}', + '-I{python_includes}', + ] + + common_ldflags = ['-L{ctx_libs_dir}'] + + common_ldlibs = ['-lm'] + + common_ldshared = [ + '-pthread', + '-shared', + '-Wl,-O1', + '-Wl,-Bsymbolic-functions', + ] + def __init__(self, ctx): super(Arch, self).__init__() self.ctx = ctx @@ -38,125 +68,159 @@ def include_dirs(self): @property def target(self): - target_data = self.command_prefix.split('-') - return '-'.join( - [target_data[0], 'none', target_data[1], target_data[2]]) - - def get_env(self, with_flags_in_cc=True, clang=False): - env = {} - - cflags = [ - '-DANDROID', - '-fomit-frame-pointer', - '-D__ANDROID_API__={}'.format(self.ctx.ndk_api)] - if not clang: - cflags.append('-mandroid') - else: - cflags.append('-target ' + self.target) - toolchain = '{android_host}-{toolchain_version}'.format( - android_host=self.ctx.toolchain_prefix, - toolchain_version=self.ctx.toolchain_version) - toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, - 'prebuilt', build_platform) - cflags.append('-gcc-toolchain {}'.format(toolchain)) + # As of NDK r19, the toolchains installed by default with the + # NDK may be used in-place. The make_standalone_toolchain.py script + # is no longer needed for interfacing with arbitrary build systems. + # See: https://developer.android.com/ndk/guides/other_build_systems + return '{triplet}{ndk_api}'.format( + triplet=self.command_prefix, ndk_api=self.ctx.ndk_api + ) - env['CFLAGS'] = ' '.join(cflags) + @property + def clang_path(self): + """Full path of the clang compiler""" + llvm_dirname = split( + glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1] + )[-1] + return join( + self.ctx.ndk_dir, + 'toolchains', + llvm_dirname, + 'prebuilt', + build_platform, + 'bin', + ) - # Link the extra global link paths first before anything else - # (such that overriding system libraries with them is possible) - env['LDFLAGS'] = ' ' + " ".join([ - "-L'" + l.replace("'", "'\"'\"'") + "'" # no shlex.quote in py2 - for l in self.extra_global_link_paths - ]) + ' ' - - sysroot = join(self.ctx._ndk_dir, 'sysroot') - if exists(sysroot): - # post-15 NDK per - # https://android.googlesource.com/platform/ndk/+/ndk-r15-release/docs/UnifiedHeaders.md - env['CFLAGS'] += ' -isystem {}/sysroot/usr/include/{}'.format( - self.ctx.ndk_dir, self.ctx.toolchain_prefix) - env['CFLAGS'] += ' -I{}/sysroot/usr/include/{}'.format( - self.ctx.ndk_dir, self.command_prefix) - else: - sysroot = self.ctx.ndk_platform - env['CFLAGS'] += ' -I{}'.format(self.ctx.ndk_platform) - env['CFLAGS'] += ' -isysroot {} '.format(sysroot) - env['CFLAGS'] += '-I' + join(self.ctx.get_python_install_dir(), - 'include/python{}'.format( - self.ctx.python_recipe.version[0:3]) - ) + @property + def clang_exe(self): + """Full path of the clang compiler depending on the android's ndk + version used.""" + return self.get_clang_exe() - env['LDFLAGS'] += '--sysroot={} '.format(self.ctx.ndk_platform) + @property + def clang_exe_cxx(self): + """Full path of the clang++ compiler depending on the android's ndk + version used.""" + return self.get_clang_exe(plus_plus=True) + + def get_clang_exe(self, with_target=False, plus_plus=False): + """Returns the full path of the clang/clang++ compiler, supports two + kwargs: + + - `with_target`: prepend `target` to clang + - `plus_plus`: will return the clang++ compiler (defaults to `False`) + """ + compiler = 'clang' + if with_target: + compiler = '{target}-{compiler}'.format( + target=self.target, compiler=compiler + ) + if plus_plus: + compiler += '++' + return join(self.clang_path, compiler) + + def get_env(self, with_flags_in_cc=True): + env = {} - env["CXXFLAGS"] = env["CFLAGS"] + # CFLAGS/CXXFLAGS: the processor flags + env['CFLAGS'] = ' '.join(self.common_cflags).format(target=self.target) + if self.arch_cflags: + # each architecture may have has his own CFLAGS + env['CFLAGS'] += ' ' + ' '.join(self.arch_cflags) + env['CXXFLAGS'] = env['CFLAGS'] - env["LDFLAGS"] += " ".join(['-lm', '-L' + self.ctx.get_libs_dir(self.arch)]) + # CPPFLAGS (for macros and includes) + env['CPPFLAGS'] = ' '.join(self.common_cppflags).format( + ctx=self.ctx, + command_prefix=self.command_prefix, + python_includes=join( + self.ctx.get_python_install_dir(), + 'include/python{}'.format(self.ctx.python_recipe.version[0:3]), + ), + ) - toolchain_prefix = self.ctx.toolchain_prefix - toolchain_version = self.ctx.toolchain_version - command_prefix = self.command_prefix + # LDFLAGS: Link the extra global link paths first before anything else + # (such that overriding system libraries with them is possible) + env['LDFLAGS'] = ( + ' ' + + " ".join( + [ + "-L'" + + l.replace("'", "'\"'\"'") + + "'" # no shlex.quote in py2 + for l in self.extra_global_link_paths + ] + ) + + ' ' + ' '.join(self.common_ldflags).format( + ctx_libs_dir=self.ctx.get_libs_dir(self.arch) + ) + ) - env['TOOLCHAIN_PREFIX'] = toolchain_prefix - env['TOOLCHAIN_VERSION'] = toolchain_version + # LDLIBS: Library flags or names given to compilers when they are + # supposed to invoke the linker. + env['LDLIBS'] = ' '.join(self.common_ldlibs) + # CCACHE ccache = '' if self.ctx.ccache and bool(int(environ.get('USE_CCACHE', '1'))): # print('ccache found, will optimize builds') ccache = self.ctx.ccache + ' ' env['USE_CCACHE'] = '1' env['NDK_CCACHE'] = self.ctx.ccache - env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')}) - - if clang: - llvm_dirname = split( - glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1])[-1] - clang_path = join(self.ctx.ndk_dir, 'toolchains', llvm_dirname, - 'prebuilt', build_platform, 'bin') - environ['PATH'] = '{clang_path}:{path}'.format( - clang_path=clang_path, path=environ['PATH']) - exe = join(clang_path, 'clang') - execxx = join(clang_path, 'clang++') - else: - exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix) - execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix) + env.update( + {k: v for k, v in environ.items() if k.startswith('CCACHE_')} + ) - cc = find_executable(exe, path=environ['PATH']) + # Compiler: `CC` and `CXX` (and make sure that the compiler exists) + environ['PATH'] = '{clang_path}:{path}'.format( + clang_path=self.clang_path, path=environ['PATH'] + ) + cc = find_executable(self.clang_exe, path=environ['PATH']) if cc is None: print('Searching path are: {!r}'.format(environ['PATH'])) raise BuildInterruptingException( 'Couldn\'t find executable for CC. This indicates a ' 'problem locating the {} executable in the Android ' 'NDK, not that you don\'t have a normal compiler ' - 'installed. Exiting.'.format(exe)) + 'installed. Exiting.'.format(self.clang_exe)) if with_flags_in_cc: env['CC'] = '{ccache}{exe} {cflags}'.format( - exe=exe, + exe=self.clang_exe, ccache=ccache, cflags=env['CFLAGS']) env['CXX'] = '{ccache}{execxx} {cxxflags}'.format( - execxx=execxx, + execxx=self.clang_exe_cxx, ccache=ccache, cxxflags=env['CXXFLAGS']) else: env['CC'] = '{ccache}{exe}'.format( - exe=exe, + exe=self.clang_exe, ccache=ccache) env['CXX'] = '{ccache}{execxx}'.format( - execxx=execxx, + execxx=self.clang_exe_cxx, ccache=ccache) + # Android's binaries + command_prefix = self.command_prefix env['AR'] = '{}-ar'.format(command_prefix) env['RANLIB'] = '{}-ranlib'.format(command_prefix) - env['LD'] = '{}-ld'.format(command_prefix) - env['LDSHARED'] = env["CC"] + " -pthread -shared " +\ - "-Wl,-O1 -Wl,-Bsymbolic-functions " - env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix) - env['MAKE'] = 'make -j5' + env['MAKE'] = 'make -j{}'.format(str(cpu_count())) env['READELF'] = '{}-readelf'.format(command_prefix) env['NM'] = '{}-nm'.format(command_prefix) + env['LD'] = '{}-ld'.format(command_prefix) + # Android's arch/toolchain + env['ARCH'] = self.arch + env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) + env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix + env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version + + # Custom linker options + env['LDSHARED'] = env['CC'] + ' ' + ' '.join(self.common_ldshared) + + # Host python (used by some recipes) hostpython_recipe = Recipe.get_recipe( 'host' + self.ctx.python_recipe.name, self.ctx) env['BUILDLIB_PATH'] = join( @@ -171,9 +235,6 @@ def get_env(self, with_flags_in_cc=True, clang=False): env['PATH'] = environ['PATH'] - env['ARCH'] = self.arch - env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - return env @@ -186,20 +247,21 @@ class ArchARM(Arch): @property def target(self): target_data = self.command_prefix.split('-') - return '-'.join( - ['armv7a', 'none', target_data[1], target_data[2]]) + return '{triplet}{ndk_api}'.format( + triplet='-'.join(['armv7a', target_data[1], target_data[2]]), + ndk_api=self.ctx.ndk_api, + ) class ArchARMv7_a(ArchARM): arch = 'armeabi-v7a' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(ArchARMv7_a, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - (' -march=armv7-a -mfloat-abi=softfp ' - '-mfpu=vfp -mthumb')) - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=armv7-a', + '-mfloat-abi=softfp', + '-mfpu=vfp', + '-mthumb', + '-fPIC', + ] class Archx86(Arch): @@ -207,13 +269,13 @@ class Archx86(Arch): toolchain_prefix = 'x86' command_prefix = 'i686-linux-android' platform_dir = 'arch-x86' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(Archx86, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - ' -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32') - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=i686', + '-mtune=intel', + '-mssse3', + '-mfpmath=sse', + '-m32', + ] class Archx86_64(Arch): @@ -221,13 +283,13 @@ class Archx86_64(Arch): toolchain_prefix = 'x86_64' command_prefix = 'x86_64-linux-android' platform_dir = 'arch-x86_64' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(Archx86_64, self).get_env(with_flags_in_cc, clang=clang) - env['CFLAGS'] = (env['CFLAGS'] + - ' -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel') - env['CXXFLAGS'] = env['CFLAGS'] - return env + arch_cflags = [ + '-march=x86-64', + '-msse4.2', + '-mpopcnt', + '-m64', + '-mtune=intel', + ] class ArchAarch_64(Arch): @@ -235,14 +297,17 @@ class ArchAarch_64(Arch): toolchain_prefix = 'aarch64-linux-android' command_prefix = 'aarch64-linux-android' platform_dir = 'arch-arm64' - - def get_env(self, with_flags_in_cc=True, clang=False): - env = super(ArchAarch_64, self).get_env(with_flags_in_cc, clang=clang) - incpath = ' -I' + join(dirname(__file__), 'includes', 'arm64-v8a') - env['EXTRA_CFLAGS'] = incpath - env['CFLAGS'] += incpath - env['CXXFLAGS'] += incpath - if with_flags_in_cc: - env['CC'] += incpath - env['CXX'] += incpath - return env + arch_cflags = [ + '-march=armv8-a', + # '-I' + join(dirname(__file__), 'includes', 'arm64-v8a'), + ] + + # Note: This `EXTRA_CFLAGS` below should target the commented `include` + # above in `arch_cflags`. The original lines were added during the Sdl2's + # bootstrap creation, and modified/commented during the migration to the + # NDK r19 build system, because it seems that we don't need it anymore, + # do we need them? + # def get_env(self, with_flags_in_cc=True): + # env = super(ArchAarch_64, self).get_env(with_flags_in_cc) + # env['EXTRA_CFLAGS'] = self.arch_cflags[-1] + # return env diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py index 4604aa2bed..c4c4ebcc19 100755 --- a/pythonforandroid/python.py +++ b/pythonforandroid/python.py @@ -12,10 +12,13 @@ import sh from pythonforandroid.recipe import Recipe, TargetPythonRecipe -from pythonforandroid.logger import logger, info, shprint +from pythonforandroid.logger import info, warning, shprint from pythonforandroid.util import ( - current_directory, ensure_dir, walk_valid_filens, - BuildInterruptingException, build_platform) + current_directory, + ensure_dir, + walk_valid_filens, + BuildInterruptingException, +) class GuestPythonRecipe(TargetPythonRecipe): @@ -105,26 +108,9 @@ def __init__(self, *args, **kwargs): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = environ.copy() + env['HOSTARCH'] = arch.command_prefix - android_host = env['HOSTARCH'] = arch.command_prefix - toolchain = '{toolchain_prefix}-{toolchain_version}'.format( - toolchain_prefix=self.ctx.toolchain_prefix, - toolchain_version=self.ctx.toolchain_version) - toolchain = join(self.ctx.ndk_dir, 'toolchains', - toolchain, 'prebuilt', build_platform) - - env['CC'] = ( - '{clang} -target {target} -gcc-toolchain {toolchain}').format( - clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt', - build_platform, 'bin', 'clang'), - target=arch.target, - toolchain=toolchain) - env['AR'] = join(toolchain, 'bin', android_host) + '-ar' - env['LD'] = join(toolchain, 'bin', android_host) + '-ld' - env['RANLIB'] = join(toolchain, 'bin', android_host) + '-ranlib' - env['READELF'] = join(toolchain, 'bin', android_host) + '-readelf' - env['STRIP'] = join(toolchain, 'bin', android_host) + '-strip' - env['STRIP'] += ' --strip-debug --strip-unneeded' + env['CC'] = arch.get_clang_exe(with_target=True) env['PATH'] = ( '{hostpython_dir}:{old_path}').format( @@ -132,43 +118,22 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): 'host' + self.name, self.ctx).get_path_to_python(), old_path=env['PATH']) - ndk_flags = ( - '-fPIC --sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} ' - '-isystem {ndk_android_host} -I{ndk_include}').format( - ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'), - android_api=self.ctx.ndk_api, - ndk_android_host=join( - self.ctx.ndk_dir, 'sysroot', 'usr', 'include', android_host), - ndk_include=join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')) - sysroot = self.ctx.ndk_platform - env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags - env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags - env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format( - sysroot, join(sysroot, 'usr', 'lib')) - - # Manually add the libs directory, and copy some object - # files to the current directory otherwise they aren't - # picked up. This seems necessary because the --sysroot - # setting in LDFLAGS is overridden by the other flags. - # TODO: Work out why this doesn't happen in the original - # bpo-30386 Makefile system. - logger.warning('Doing some hacky stuff to link properly') - lib_dir = join(sysroot, 'usr', 'lib') - if arch.arch == 'x86_64': - lib_dir = join(sysroot, 'usr', 'lib64') - env['LDFLAGS'] += ' -L{}'.format(lib_dir) - shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './') - shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './') - - env['SYSROOT'] = sysroot + env['CFLAGS'] = ' '.join( + [ + '-fPIC', + '-DANDROID', + '-D__ANDROID_API__={}'.format(self.ctx.ndk_api), + ] + ) + env['LDFLAGS'] = env.get('LDFLAGS', '') if sh.which('lld') is not None: # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 - env["LDFLAGS"] += ' -L. -fuse-ld=lld' + env['LDFLAGS'] += ' -L. -fuse-ld=lld' else: - logger.warning('lld not found, linking without it. ' + - 'Consider installing lld if linker errors occur.') + warning('lld not found, linking without it. ' + 'Consider installing lld if linker errors occur.') return env @@ -203,6 +168,33 @@ def add_flags(include_flags, link_dirs, link_libs): recipe = Recipe.get_recipe('openssl', self.ctx) add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch), recipe.link_libs_flags()) + + # python build system contains hardcoded zlib version which prevents + # the build of zlib module, here we search for android's zlib version + # and sets the right flags, so python can be build with android's zlib + info("Activating flags for android's zlib") + zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') + zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_h = join(zlib_includes, 'zlib.h') + try: + with open(zlib_h) as fileh: + zlib_data = fileh.read() + except IOError: + raise BuildInterruptingException( + "Could not determine android's zlib version, no zlib.h ({}) in" + " the NDK dir includes".format(zlib_h) + ) + for line in zlib_data.split('\n'): + if line.startswith('#define ZLIB_VERSION '): + break + else: + raise BuildInterruptingException( + 'Could not parse zlib.h...so we cannot find zlib version,' + 'required by python build,' + ) + env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') + add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') + return env @property diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index ec0d7510ea..f165ce474c 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -430,12 +430,12 @@ def unpack(self, arch): else: info('{} is already unpacked, skipping'.format(self.name)) - def get_recipe_env(self, arch=None, with_flags_in_cc=True, clang=False): + def get_recipe_env(self, arch=None, with_flags_in_cc=True): """Return the env specialized for the recipe """ if arch is None: arch = self.filtered_archs[0] - return arch.get_env(with_flags_in_cc=with_flags_in_cc, clang=clang) + return arch.get_env(with_flags_in_cc=with_flags_in_cc) def prebuild_arch(self, arch): '''Run any pre-build tasks for the Recipe. By default, this checks if diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py index 1969d2c1c5..507ba775e4 100644 --- a/pythonforandroid/recipes/jpeg/__init__.py +++ b/pythonforandroid/recipes/jpeg/__init__.py @@ -63,7 +63,7 @@ def build_arch(self, arch): for lib in glob(join(build_dir, '*.a')): shprint(sh.cp, '-L', lib, self.ctx.libs_dir) - def get_recipe_env(self, arch=None, with_flags_in_cc=False, clang=True): + def get_recipe_env(self, arch=None, with_flags_in_cc=False): env = environ.copy() build_platform = '{system}-{machine}'.format( diff --git a/pythonforandroid/recipes/libffi/__init__.py b/pythonforandroid/recipes/libffi/__init__.py index 31ed9c69b5..3608df6eb0 100644 --- a/pythonforandroid/recipes/libffi/__init__.py +++ b/pythonforandroid/recipes/libffi/__init__.py @@ -12,16 +12,24 @@ class LibffiRecipe(Recipe): - `automake` for the `aclocal` binary - `autoconf` for the `autoreconf` binary - `libltdl-dev` which defines the `LT_SYS_SYMBOL_USCORE` macro + + .. note:: + Some notes about libffi version: + + - v3.2.1 it's from year 2014...it's a little outdated and has + problems with clang (see issue #1525) + - v3.3-rc0 it was released at april 2018 (it's a pre-release), and + it lacks some commits that we are interested, specially those + ones that fixes specific issues for Arm64, you can check those + commits at (search for commit `8fa8837` and look at the below + commits): https://github.com/libffi/libffi/commits/master """ name = 'libffi' - version = '3.2.1' - url = 'https://github.com/libffi/libffi/archive/v{version}.tar.gz' + # Version pinned to post `v3.3RC0` + version = '8fa8837' + url = 'https://github.com/libffi/libffi/archive/{version}.tar.gz' - patches = ['remove-version-info.patch', - # This patch below is already included into libffi's master - # branch and included in the pre-release 3.3rc0...so we should - # remove this when we update the version number for libffi - 'fix-includedir.patch'] + patches = ['remove-version-info.patch'] def should_build(self, arch): return not exists(join(self.ctx.get_libs_dir(arch.arch), 'libffi.so')) @@ -37,14 +45,11 @@ def build_arch(self, arch): '--prefix=' + self.get_build_dir(arch.arch), '--disable-builddir', '--enable-shared', _env=env) - shprint(sh.make, '-j', str(cpu_count()), 'libffi.la', _env=env) - - host_build = self.get_build_dir(arch.arch) ensure_dir(self.ctx.get_libs_dir(arch.arch)) - shprint(sh.cp, - join(host_build, '.libs', 'libffi.so'), - self.ctx.get_libs_dir(arch.arch)) + self.install_libs( + arch, join(self.get_build_dir(arch.arch), '.libs', 'libffi.so') + ) def get_include_dirs(self, arch): return [join(self.get_build_dir(arch.arch), 'include')] diff --git a/pythonforandroid/recipes/libffi/fix-includedir.patch b/pythonforandroid/recipes/libffi/fix-includedir.patch deleted file mode 100644 index 0dc35c70ca..0000000000 --- a/pythonforandroid/recipes/libffi/fix-includedir.patch +++ /dev/null @@ -1,34 +0,0 @@ -From 982b89c01aca99c7bc229914fc1521f96930919b Mon Sep 17 00:00:00 2001 -From: Yen Chi Hsuan -Date: Sun, 13 Nov 2016 19:17:19 +0800 -Subject: [PATCH] Install public headers in the standard path - ---- - include/Makefile.am | 3 +-- - libffi.pc.in | 2 +- - 2 files changed, 2 insertions(+), 3 deletions(-) - -diff --git a/include/Makefile.am b/include/Makefile.am -index bb241e88..c59df9fb 100644 ---- a/include/Makefile.am -+++ b/include/Makefile.am -@@ -6,5 +6,4 @@ DISTCLEANFILES=ffitarget.h - noinst_HEADERS=ffi_common.h ffi_cfi.h - EXTRA_DIST=ffi.h.in - --includesdir = $(libdir)/@PACKAGE_NAME@-@PACKAGE_VERSION@/include --nodist_includes_HEADERS = ffi.h ffitarget.h -+nodist_include_HEADERS = ffi.h ffitarget.h -diff --git a/libffi.pc.in b/libffi.pc.in -index edf6fde5..6fad83b4 100644 ---- a/libffi.pc.in -+++ b/libffi.pc.in -@@ -2,7 +2,7 @@ prefix=@prefix@ - exec_prefix=@exec_prefix@ - libdir=@libdir@ - toolexeclibdir=@toolexeclibdir@ --includedir=${libdir}/@PACKAGE_NAME@-@PACKAGE_VERSION@/include -+includedir=@includedir@ - - Name: @PACKAGE_NAME@ - Description: Library supporting Foreign Function Interfaces diff --git a/pythonforandroid/recipes/libffi/remove-version-info.patch b/pythonforandroid/recipes/libffi/remove-version-info.patch index 7bdc11a641..0a32b7e614 100644 --- a/pythonforandroid/recipes/libffi/remove-version-info.patch +++ b/pythonforandroid/recipes/libffi/remove-version-info.patch @@ -1,12 +1,11 @@ -diff -Naur libffi/Makefile.am b/Makefile.am ---- libffi/Makefile.am 2014-11-12 06:00:59.000000000 -0600 -+++ b/Makefile.am 2015-12-23 15:57:10.363148806 -0600 -@@ -249,7 +249,7 @@ - AM_CFLAGS += -DFFI_DEBUG - endif - --libffi_la_LDFLAGS = -no-undefined -version-info `grep -v '^\#' $(srcdir)/libtool-version` $(LTLDFLAGS) $(AM_LTLDFLAGS) +--- libffi/Makefile.am.orig 2018-12-21 16:11:26.159181262 +0100 ++++ libffi/Makefile.am 2018-12-21 16:14:44.075179374 +0100 +@@ -156,7 +156,7 @@ + libffi.map: $(top_srcdir)/libffi.map.in + $(COMPILE) -D$(TARGET) -E -x assembler-with-cpp -o $@ $< + +-libffi_la_LDFLAGS = -no-undefined $(libffi_version_info) $(libffi_version_script) $(LTLDFLAGS) $(AM_LTLDFLAGS) +libffi_la_LDFLAGS = -no-undefined -avoid-version $(LTLDFLAGS) $(AM_LTLDFLAGS) - + libffi_la_DEPENDENCIES = $(libffi_la_LIBADD) $(libffi_version_dep) + AM_CPPFLAGS = -I. -I$(top_srcdir)/include -Iinclude -I$(top_srcdir)/src - AM_CCASFLAGS = $(AM_CPPFLAGS) diff --git a/pythonforandroid/recipes/numpy/__init__.py b/pythonforandroid/recipes/numpy/__init__.py index 4e47e9d890..027b5465fe 100644 --- a/pythonforandroid/recipes/numpy/__init__.py +++ b/pythonforandroid/recipes/numpy/__init__.py @@ -8,14 +8,12 @@ class NumpyRecipe(CompiledComponentsPythonRecipe): version = '1.16.4' url = 'https://pypi.python.org/packages/source/n/numpy/numpy-{version}.zip' site_packages_name = 'numpy' + depends = ['setuptools'] patches = [ join('patches', 'add_libm_explicitly_to_build.patch'), join('patches', 'do_not_use_system_libs.patch'), join('patches', 'remove_unittest_call.patch'), - join('patches', 'ar.patch'), - join('patches', 'fix_setup_dependencies.patch'), - join('patches', 'fix_environment_detection.patch'), ] call_hostpython_via_targetpython = False @@ -30,28 +28,5 @@ def rebuild_compiled_components(self, arch, env): super(NumpyRecipe, self).rebuild_compiled_components(arch, env) self.setup_extra_args = [] - def get_recipe_env(self, arch): - env = super(NumpyRecipe, self).get_recipe_env(arch) - - flags = " -L{} --sysroot={}".format( - join(self.ctx.ndk_platform, 'usr', 'lib'), - self.ctx.ndk_platform - ) - - py_ver = self.ctx.python_recipe.major_minor_version_string - py_inc_dir = self.ctx.python_recipe.include_root(arch.arch) - py_lib_dir = self.ctx.python_recipe.link_root(arch.arch) - flags += ' -I{}'.format(py_inc_dir) - flags += ' -L{} -lpython{}'.format(py_lib_dir, py_ver) - if 'python3' in self.ctx.python_recipe.name: - flags += 'm' - - if flags not in env['CC']: - env['CC'] += flags - if flags not in env['LD']: - env['LD'] += flags + ' -shared' - - return env - recipe = NumpyRecipe() diff --git a/pythonforandroid/recipes/numpy/patches/ar.patch b/pythonforandroid/recipes/numpy/patches/ar.patch deleted file mode 100644 index c806636dc1..0000000000 --- a/pythonforandroid/recipes/numpy/patches/ar.patch +++ /dev/null @@ -1,41 +0,0 @@ -diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py -index 0fac9b0..94be92a 100644 ---- a/numpy/core/code_generators/generate_umath.py -+++ b/numpy/core/code_generators/generate_umath.py -@@ -982,6 +982,7 @@ def make_arrays(funcdict): - funclist.append('%s_%s' % (tname, name)) - if t.simd is not None: - for vt in t.simd: -+ continue - code2list.append(textwrap.dedent("""\ - #ifdef HAVE_ATTRIBUTE_TARGET_{ISA} - if (npy_cpu_supports("{isa}")) {{ -diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py -index 14451fa..dfd65da 100644 ---- a/numpy/distutils/ccompiler.py -+++ b/numpy/distutils/ccompiler.py -@@ -295,6 +295,7 @@ def CCompiler_compile(self, sources, output_dir=None, macros=None, - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) -+ cc_args += os.environ['CFLAGS'].split() - display = "compile options: '%s'" % (' '.join(cc_args)) - if extra_postargs: - display += "\nextra options: '%s'" % (' '.join(extra_postargs)) -@@ -795,4 +796,3 @@ for _cc in ['msvc9', 'msvc', '_msvc', 'bcpp', 'cygwinc', 'emxc', 'unixc']: - _m = sys.modules.get('distutils.' + _cc + 'compiler') - if _m is not None: - setattr(_m, 'gen_lib_options', gen_lib_options) -- -diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py -index 11b2cce..c3e9f10 100644 ---- a/numpy/distutils/unixccompiler.py -+++ b/numpy/distutils/unixccompiler.py -@@ -111,6 +111,7 @@ def UnixCCompiler_create_static_lib(self, objects, output_libname, - while tmp_objects: - objects = tmp_objects[:50] - tmp_objects = tmp_objects[50:] -+ self.archiver[0] = os.environ['AR'] - display = '%s: adding %d object files to %s' % ( - os.path.basename(self.archiver[0]), - len(objects), output_filename) diff --git a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch b/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch index cdcc41086b..13c1f4bab0 100644 --- a/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch +++ b/pythonforandroid/recipes/numpy/patches/do_not_use_system_libs.patch @@ -4,7 +4,7 @@ index 806f4f7..0d51cfa 100644 +++ b/numpy/distutils/system_info.py @@ -734,6 +734,7 @@ class system_info(object): return self.get_paths(self.section, key) - + def get_libs(self, key, default): + return [] try: diff --git a/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch b/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch deleted file mode 100644 index 3c7251eaa1..0000000000 --- a/pythonforandroid/recipes/numpy/patches/fix_environment_detection.patch +++ /dev/null @@ -1,48 +0,0 @@ -commit 9a09edac303c534a38c5d829d8537176f8a8dfb9 -Author: Alexander Taylor -Date: Fri Jun 28 22:50:45 2019 +0100 - - fix_environment_detection.patch - -diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h -index 64aaaac..e6293f9 100644 ---- a/numpy/core/include/numpy/npy_common.h -+++ b/numpy/core/include/numpy/npy_common.h -@@ -164,12 +164,12 @@ extern long long __cdecl _ftelli64(FILE *); - #endif - #else - #ifdef HAVE_FSEEKO -- #define npy_fseek fseeko -+ #define npy_fseek fseek - #else - #define npy_fseek fseek - #endif - #ifdef HAVE_FTELLO -- #define npy_ftell ftello -+ #define npy_ftell ftell - #else - #define npy_ftell ftell - #endif -@@ -321,13 +321,15 @@ typedef unsigned char npy_bool; - #define NPY_TRUE 1 - - --#if NPY_SIZEOF_LONGDOUBLE == NPY_SIZEOF_DOUBLE -- typedef double npy_longdouble; -- #define NPY_LONGDOUBLE_FMT "g" --#else -- typedef long double npy_longdouble; -- #define NPY_LONGDOUBLE_FMT "Lg" --#endif -+/* #if NPY_SIZEOF_LONGDOUBLE == NPY_SIZEOF_DOUBLE */ -+/* typedef double npy_longdouble; */ -+/* #define NPY_LONGDOUBLE_FMT "g" */ -+/* #else */ -+/* typedef long double npy_longdouble; */ -+/* #define NPY_LONGDOUBLE_FMT "Lg" */ -+/* #endif */ -+typedef long double npy_longdouble; -+#define NPY_LONGDOUBLE_FMT "Lg" - - #ifndef Py_USING_UNICODE - #error Must use Python with unicode enabled. diff --git a/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch b/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch deleted file mode 100644 index 1c38bc6068..0000000000 --- a/pythonforandroid/recipes/numpy/patches/fix_setup_dependencies.patch +++ /dev/null @@ -1,40 +0,0 @@ -diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py -index 42374ac..67fcd98 100644 ---- a/numpy/distutils/misc_util.py -+++ b/numpy/distutils/misc_util.py -@@ -9,7 +9,6 @@ import atexit - import tempfile - import subprocess - import shutil --import multiprocessing - - import distutils - from distutils.errors import DistutilsError -@@ -94,11 +93,7 @@ def get_num_build_jobs(): - - """ - from numpy.distutils.core import get_distribution -- try: -- cpu_count = len(os.sched_getaffinity(0)) -- except AttributeError: -- cpu_count = multiprocessing.cpu_count() -- cpu_count = min(cpu_count, 8) -+ cpu_count = 1 - envjobs = int(os.environ.get("NPY_NUM_BUILD_JOBS", cpu_count)) - dist = get_distribution() - # may be None during configuration -diff --git a/setup.py b/setup.py -index 8b2ded1..431c1b8 100755 ---- a/setup.py -+++ b/setup.py -@@ -389,9 +389,8 @@ def setup_package(): - # Raise errors for unsupported commands, improve help output, etc. - run_build = parse_setuppy_commands() - -- from setuptools import setup -+ from numpy.distutils.core import setup - if run_build: -- from numpy.distutils.core import setup - cwd = os.path.abspath(os.path.dirname(__file__)) - if not os.path.exists(os.path.join(cwd, 'PKG-INFO')): - # Generate Cython sources, unless building from source release diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index d3033a3594..a4aad3df16 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -90,7 +90,7 @@ def should_build(self, arch): 'libcrypto' + self.version + '.so') def get_recipe_env(self, arch=None): - env = super(OpenSSLRecipe, self).get_recipe_env(arch, clang=True) + env = super(OpenSSLRecipe, self).get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version env['MAKE'] = 'make' # This removes the '-j5', which isn't safe env['ANDROID_NDK'] = self.ctx.ndk_dir diff --git a/pythonforandroid/recipes/png/__init__.py b/pythonforandroid/recipes/png/__init__.py index 1ad49cc1ca..a6e281a570 100644 --- a/pythonforandroid/recipes/png/__init__.py +++ b/pythonforandroid/recipes/png/__init__.py @@ -16,15 +16,6 @@ def should_build(self, arch): join(self.get_build_dir(arch.arch), '.libs', 'libpng16.so') ) - def get_recipe_env(self, arch=None): - env = super(PngRecipe, self).get_recipe_env(arch) - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') - env['CFLAGS'] += ' -I{}'.format(ndk_include_dir) - env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) - env['LDFLAGS'] += ' --sysroot={}'.format(self.ctx.ndk_platform) - return env - def build_arch(self, arch): super(PngRecipe, self).build_arch(arch) build_dir = self.get_build_dir(arch.arch) diff --git a/pythonforandroid/recipes/pycrypto/__init__.py b/pythonforandroid/recipes/pycrypto/__init__.py index e8bfab2666..8f70df28e9 100644 --- a/pythonforandroid/recipes/pycrypto/__init__.py +++ b/pythonforandroid/recipes/pycrypto/__init__.py @@ -15,7 +15,7 @@ class PyCryptoRecipe(CompiledComponentsPythonRecipe): call_hostpython_via_targetpython = False patches = ['add_length.patch'] - def get_recipe_env(self, arch=None, clang=True): + def get_recipe_env(self, arch=None): env = super(PyCryptoRecipe, self).get_recipe_env(arch) openssl_recipe = Recipe.get_recipe('openssl', self.ctx) env['CC'] = env['CC'] + openssl_recipe.include_flags(arch) diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py index 25c6e714ea..0f14e5e617 100644 --- a/pythonforandroid/recipes/python2/__init__.py +++ b/pythonforandroid/recipes/python2/__init__.py @@ -20,7 +20,7 @@ class Python2Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python2' - depends = ['hostpython2'] + depends = ['hostpython2', 'libffi'] conflicts = ['python3'] patches = [ @@ -34,6 +34,7 @@ class Python2Recipe(GuestPythonRecipe): 'patches/fix-pwd-gecos.patch', 'patches/fix-ctypes-util-find-library.patch', 'patches/fix-interpreter-version.patch', + 'patches/fix-zlib-version.patch', ] configure_args = ('--host={android_host}', diff --git a/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch b/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch index a098b25634..cb777d3b1f 100644 --- a/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch +++ b/pythonforandroid/recipes/python2/patches/fix-missing-extensions.patch @@ -1,15 +1,3 @@ -diff -Naurp Python-2.7.15/Modules/Setup.dist.orig Python-2.7.15/Modules/Setup.dist ---- Python-2.7.15/Modules/Setup.dist.orig 2018-04-30 00:47:33.000000000 +0200 -+++ Python-2.7.15/Modules/Setup.dist 2018-11-17 20:40:20.153518694 +0100 -@@ -464,7 +464,7 @@ - # Andrew Kuchling's zlib module. - # This require zlib 1.1.3 (or later). - # See http://www.gzip.org/zlib/ --#zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz -+zlib zlibmodule.c -I$(prefix)/include -L$(exec_prefix)/lib -lz - - # Interface to the Expat XML parser - # diff -Naurp Python-2.7.15.orig/Makefile.pre.in Python-2.7.15/Makefile.pre.in --- Python-2.7.15.orig/Makefile.pre.in 2018-04-30 00:47:33.000000000 +0200 +++ Python-2.7.15/Makefile.pre.in 2018-11-18 00:43:58.777379280 +0100 diff --git a/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch b/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch new file mode 100644 index 0000000000..b7c14dbf80 --- /dev/null +++ b/pythonforandroid/recipes/python2/patches/fix-zlib-version.patch @@ -0,0 +1,12 @@ +--- Python-3.7.1/setup.py.orig 2018-10-20 08:04:19.000000000 +0200 ++++ Python-3.7.1/setup.py 2019-02-17 00:24:30.715904412 +0100 +@@ -1416,7 +1416,8 @@ class PyBuildExt(build_ext): + if zlib_inc is not None: + zlib_h = zlib_inc[0] + '/zlib.h' + version = '"0.0.0"' +- version_req = '"1.1.3"' ++ version_req = '"{}"'.format( ++ os.environ.get('ZLIB_VERSION', '1.1.3')) + if host_platform == 'darwin' and is_macosx_sdk_path(zlib_h): + zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:]) + with open(zlib_h) as fp: diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index 963fad635f..57765b642d 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -22,7 +22,8 @@ class Python3Recipe(GuestPythonRecipe): url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python3' - patches = ["patches/fix-ctypes-util-find-library.patch"] + patches = ['patches/fix-ctypes-util-find-library.patch', + 'patches/fix-zlib-version.patch'] if sh.which('lld') is not None: patches = patches + ["patches/remove-fix-cortex-a8.patch"] diff --git a/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch b/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch new file mode 100644 index 0000000000..0dbffae246 --- /dev/null +++ b/pythonforandroid/recipes/python3/patches/fix-zlib-version.patch @@ -0,0 +1,12 @@ +--- Python-3.7.1/setup.py.orig 2018-10-20 08:04:19.000000000 +0200 ++++ Python-3.7.1/setup.py 2019-02-17 00:24:30.715904412 +0100 +@@ -1410,7 +1410,8 @@ class PyBuildExt(build_ext): + if zlib_inc is not None: + zlib_h = zlib_inc[0] + '/zlib.h' + version = '"0.0.0"' +- version_req = '"1.1.3"' ++ version_req = '"{}"'.format( ++ os.environ.get('ZLIB_VERSION', '1.1.3')) + if host_platform == 'darwin' and is_macosx_sdk_path(zlib_h): + zlib_h = os.path.join(macosx_sdk_root(), zlib_h[1:]) + with open(zlib_h) as fp: diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 6cf18eceb9..a4a0531558 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -8,10 +8,10 @@ from pythonforandroid.util import BuildInterruptingException # We only check the NDK major version -MIN_NDK_VERSION = 17 -MAX_NDK_VERSION = 17 +MIN_NDK_VERSION = 19 +MAX_NDK_VERSION = 20 -RECOMMENDED_NDK_VERSION = "17c" +RECOMMENDED_NDK_VERSION = "19b" NDK_DOWNLOAD_URL = "https://developer.android.com/ndk/downloads/" # Important log messages diff --git a/tests/test_archs.py b/tests/test_archs.py index 39bf261654..7cbe5421ad 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -1,6 +1,6 @@ import os - import unittest +from os import environ try: from unittest import mock @@ -8,6 +8,7 @@ # `Python 2` or lower than `Python 3.3` does not # have the `unittest.mock` module built-in import mock + from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe @@ -53,6 +54,7 @@ class ArchSetUpBaseClass(object): """ ctx = None + expected_compiler = "" def setUp(self): self.ctx = Context() @@ -66,6 +68,12 @@ def setUp(self): self.ctx, name="sdl2", recipes=["python3", "kivy"] ) self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) + # Here we define the expected compiler, which, as per ndk >= r19, + # should be the same for all the tests (no more gcc compiler) + self.expected_compiler = ( + "/opt/android/android-ndk/toolchains/" + "llvm/prebuilt/linux-x86_64/bin/clang" + ) class TestArch(ArchSetUpBaseClass, unittest.TestCase): @@ -77,16 +85,8 @@ class TestArch(ArchSetUpBaseClass, unittest.TestCase): def test_arch(self): arch = Arch(self.ctx) - with self.assertRaises(AttributeError) as e1: - arch.__str__() - self.assertEqual( - e1.exception.args[0], "'Arch' object has no attribute 'arch'" - ) - with self.assertRaises(AttributeError) as e2: - getattr(arch, "target") - self.assertEqual( - e2.exception.args[0], "'NoneType' object has no attribute 'split'" - ) + self.assertEqual(arch.__str__(), arch.arch) + self.assertEqual(arch.target, "None21") self.assertIsNone(arch.toolchain_prefix) self.assertIsNone(arch.command_prefix) self.assertIsInstance(arch.include_dirs, list) @@ -98,9 +98,10 @@ class TestArchARM(ArchSetUpBaseClass, unittest.TestCase): will be used to perform tests for :class:`~pythonforandroid.archs.ArchARM`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_arm(self, mock_ensure_dir, mock_find_executable): + def test_arch_arm(self, mock_ensure_dir, mock_find_executable, mock_glob): """ Test that class :class:`~pythonforandroid.archs.ArchARM` returns some expected attributes and environment variables. @@ -115,15 +116,16 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): not exist) """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = ArchARM(self.ctx) self.assertEqual(arch.arch, "armeabi") self.assertEqual(arch.__str__(), "armeabi") self.assertEqual(arch.toolchain_prefix, "arm-linux-androideabi") self.assertEqual(arch.command_prefix, "arm-linux-androideabi") - self.assertEqual(arch.target, "armv7a-none-linux-androideabi") + self.assertEqual(arch.target, "armv7a-linux-androideabi21") self.assertEqual(arch.platform_dir, "arch-arm") arch = ArchARM(self.ctx) @@ -134,9 +136,20 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): expected_env_gcc_keys, set(env.keys()) & expected_env_gcc_keys ) + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + # check gcc compilers - self.assertEqual(env["CC"].split()[0], "arm-linux-androideabi-gcc") - self.assertEqual(env["CXX"].split()[0], "arm-linux-androideabi-g++") + self.assertEqual(env["CC"].split()[0], self.expected_compiler) + self.assertEqual(env["CXX"].split()[0], self.expected_compiler + "++") # check android binaries self.assertEqual(env["AR"], "arm-linux-androideabi-ar") self.assertEqual(env["LD"], "arm-linux-androideabi-ld") @@ -166,9 +179,9 @@ def test_arch_arm(self, mock_ensure_dir, mock_find_executable): self.assertEqual( e.exception.args[0], "Couldn't find executable for CC. This indicates a problem " - "locating the arm-linux-androideabi-gcc executable in the Android " + "locating the {expected_compiler} executable in the Android " "NDK, not that you don't have a normal compiler installed. " - "Exiting.", + "Exiting.".format(expected_compiler=self.expected_compiler), ) @@ -197,7 +210,7 @@ def test_arch_armv7a( tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True mock_glob.return_value = ["llvm"] @@ -206,10 +219,21 @@ def test_arch_armv7a( self.assertEqual(arch.__str__(), "armeabi-v7a") self.assertEqual(arch.toolchain_prefix, "arm-linux-androideabi") self.assertEqual(arch.command_prefix, "arm-linux-androideabi") - self.assertEqual(arch.target, "armv7a-none-linux-androideabi") + self.assertEqual(arch.target, "armv7a-linux-androideabi21") self.assertEqual(arch.platform_dir, "arch-arm") - env = arch.get_env(clang=True) + env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + # check clang build_platform = "{system}-{machine}".format( system=os.uname()[0], machine=os.uname()[-1] @@ -242,29 +266,46 @@ class TestArchX86(ArchSetUpBaseClass, unittest.TestCase): will be used to perform tests for :class:`~pythonforandroid.archs.Archx86`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_x86(self, mock_ensure_dir, mock_find_executable): + def test_arch_x86(self, mock_ensure_dir, mock_find_executable, mock_glob): """ Test that class :class:`~pythonforandroid.archs.Archx86` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = Archx86(self.ctx) self.assertEqual(arch.arch, "x86") self.assertEqual(arch.__str__(), "x86") self.assertEqual(arch.toolchain_prefix, "x86") self.assertEqual(arch.command_prefix, "i686-linux-android") - self.assertEqual(arch.target, "i686-none-linux-android") + self.assertEqual(arch.target, "i686-linux-android21") self.assertEqual(arch.platform_dir, "arch-x86") - # For x86 we expect some extra cflags in our `environment` env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86 we expect some extra cflags in our `environment` self.assertIn( " -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32", env["CFLAGS"], @@ -278,29 +319,49 @@ class TestArchX86_64(ArchSetUpBaseClass, unittest.TestCase): :class:`~pythonforandroid.archs.Archx86_64`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_x86_64(self, mock_ensure_dir, mock_find_executable): + def test_arch_x86_64( + self, mock_ensure_dir, mock_find_executable, mock_glob + ): """ Test that class :class:`~pythonforandroid.archs.Archx86_64` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = Archx86_64(self.ctx) self.assertEqual(arch.arch, "x86_64") self.assertEqual(arch.__str__(), "x86_64") self.assertEqual(arch.toolchain_prefix, "x86_64") self.assertEqual(arch.command_prefix, "x86_64-linux-android") - self.assertEqual(arch.target, "x86_64-none-linux-android") + self.assertEqual(arch.target, "x86_64-linux-android21") self.assertEqual(arch.platform_dir, "arch-x86_64") - # For x86_64 we expect some extra cflags in our `environment` env = arch.get_env() + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86_64 we expect some extra cflags in our `environment` + mock_find_executable.assert_called_once() self.assertIn( " -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel", env["CFLAGS"] ) @@ -313,27 +374,47 @@ class TestArchAArch64(ArchSetUpBaseClass, unittest.TestCase): :class:`~pythonforandroid.archs.ArchAarch_64`. """ + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") @mock.patch("pythonforandroid.build.ensure_dir") - def test_arch_aarch_64(self, mock_ensure_dir, mock_find_executable): + def test_arch_aarch_64( + self, mock_ensure_dir, mock_find_executable, mock_glob + ): """ Test that class :class:`~pythonforandroid.archs.ArchAarch_64` returns some expected attributes and environment variables. - .. note:: Here we mock the same functions than - :meth:`TestArchARM.test_arch_arm` + .. note:: + Here we mock the same functions than + :meth:`TestArchARM.test_arch_arm` plus `glob`, so we make sure that + the glob result is the expected even if the folder doesn't exist, + which is probably the case. This has to be done because here we + tests the `get_env` with clang """ - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = self.expected_compiler mock_ensure_dir.return_value = True + mock_glob.return_value = ["llvm"] arch = ArchAarch_64(self.ctx) self.assertEqual(arch.arch, "arm64-v8a") self.assertEqual(arch.__str__(), "arm64-v8a") self.assertEqual(arch.toolchain_prefix, "aarch64-linux-android") self.assertEqual(arch.command_prefix, "aarch64-linux-android") - self.assertEqual(arch.target, "aarch64-none-linux-android") + self.assertEqual(arch.target, "aarch64-linux-android21") self.assertEqual(arch.platform_dir, "arch-arm64") - # For x86_64 we expect to find an extra key in`environment` env = arch.get_env() - self.assertIn("EXTRA_CFLAGS", env.keys()) + # check glob and find_executable calls + self.assertEqual(mock_glob.call_count, 4) + for glob_call, kw in mock_glob.call_args_list: + self.assertEqual( + glob_call[0], + "{ndk_dir}/toolchains/llvm*".format(ndk_dir=self.ctx._ndk_dir), + ) + mock_find_executable.assert_called_once_with( + self.expected_compiler, path=environ["PATH"] + ) + + # For x86_64 we expect to find an extra key in`environment` + for flag in {"CFLAGS", "CXXFLAGS", "CC", "CXX"}: + self.assertIn("-march=armv8-a", env[flag]) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 6f66925818..e3e75b945f 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -533,15 +533,23 @@ def reset_mocks(): @mock.patch("pythonforandroid.bootstrap.shprint") @mock.patch("pythonforandroid.bootstrap.sh.Command") @mock.patch("pythonforandroid.build.ensure_dir") + @mock.patch("pythonforandroid.archs.glob") @mock.patch("pythonforandroid.archs.find_executable") def test_bootstrap_strip( self, mock_find_executable, + mock_glob, mock_ensure_dir, mock_sh_command, mock_sh_print, ): - mock_find_executable.return_value = "arm-linux-androideabi-gcc" + mock_find_executable.return_value = os.path.join( + self.ctx._ndk_dir, + "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + ) + mock_glob.return_value = [ + os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") + ] # prepare arch, bootstrap, distribution and PythonRecipe arch = ArchARMv7_a(self.ctx) bs = Bootstrap().get_bootstrap(self.bootstrap_name, self.ctx)