Experimental ndk 23 support

ci/constants.py

@@ -32,6 +32,9 @@ class TargetPython(Enum):
     'zope_interface',
     # Requires zope_interface, which is broken.
     'twisted',
+    'icu',
+    'genericndkbuild',
+    'evdev'
 ])
 
 BROKEN_RECIPES = {

ci/makefiles/android.mk

@@ -1,7 +1,7 @@
 # Downloads and installs the Android SDK depending on supplied platform: darwin or linux
 
 # Those android NDK/SDK variables can be override when running the file
-ANDROID_NDK_VERSION ?= 19b
+ANDROID_NDK_VERSION ?= 23b
 ANDROID_SDK_TOOLS_VERSION ?= 6514223
 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 29.0.3
 ANDROID_HOME ?= $(HOME)/.android
@@ -22,7 +22,7 @@ ANDROID_SDK_TOOLS_DL_URL=https://dl.google.com/android/repository/$(ANDROID_SDK_
 
 ANDROID_NDK_HOME=$(ANDROID_HOME)/android-ndk
 ANDROID_NDK_FOLDER=$(ANDROID_HOME)/android-ndk-r$(ANDROID_NDK_VERSION)
-ANDROID_NDK_ARCHIVE=android-ndk-r$(ANDROID_NDK_VERSION)-$(TARGET_OS)-x86_64.zip
+ANDROID_NDK_ARCHIVE=android-ndk-r$(ANDROID_NDK_VERSION)-$(TARGET_OS).zip
 ANDROID_NDK_DL_URL=https://dl.google.com/android/repository/$(ANDROID_NDK_ARCHIVE)
 
 $(info Target install OS is          : $(target_os))

pythonforandroid/archs.py

@@ -1,19 +1,14 @@
 from distutils.spawn import find_executable
 from os import environ
-from os.path import join, split, exists
+from os.path import join
 from multiprocessing import cpu_count
-from glob import glob
 
-from pythonforandroid.logger import warning
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import BuildInterruptingException, build_platform
 
 
 class Arch:
 
-    toolchain_prefix = None
-    '''The prefix for the toolchain dir in the NDK.'''
-
     command_prefix = None
     '''The prefix for NDK commands such as gcc.'''
 
@@ -30,8 +25,7 @@ class Arch:
 
     common_cppflags = [
         '-DANDROID',
-        '-D__ANDROID_API__={ctx.ndk_api}',
-        '-I{ctx.ndk_sysroot}/usr/include/{command_prefix}',
+        '-I{ctx.ndk_sysroot}/usr/include',
         '-I{python_includes}',
     ]
 
@@ -62,20 +56,6 @@ def __str__(self):
     def ndk_lib_dir(self):
         return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api))
 
-    @property
-    def ndk_platform(self):
-        warning("ndk_platform is deprecated and should be avoided in new recipes")
-        ndk_platform = join(
-            self.ctx.ndk_dir,
-            'platforms',
-            'android-{}'.format(self.ctx.ndk_api),
-            self.platform_dir)
-        if not exists(ndk_platform):
-            BuildInterruptingException(
-                "The requested platform folder doesn't exist. If you're building on ndk >= r22, and seeing this error, one of the required recipe is using a removed feature."
-            )
-        return ndk_platform
-
     @property
     def include_dirs(self):
         return [
@@ -97,13 +77,10 @@ def target(self):
     @property
     def clang_path(self):
         """Full path of the clang compiler"""
-        llvm_dirname = split(
-            glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1]
-        )[-1]
         return join(
             self.ctx.ndk_dir,
             'toolchains',
-            llvm_dirname,
+            'llvm',
             'prebuilt',
             build_platform,
             'bin',
@@ -190,12 +167,10 @@ def get_env(self, with_flags_in_cc=True):
             )
 
         # Compiler: `CC` and `CXX` (and make sure that the compiler exists)
-        environ['PATH'] = '{clang_path}:{path}'.format(
-            clang_path=self.clang_path, path=environ['PATH']
-        )
-        cc = find_executable(self.clang_exe, path=environ['PATH'])
+        env['PATH'] = self.ctx.env['PATH']
+        cc = find_executable(self.clang_exe, path=env['PATH'])
         if cc is None:
-            print('Searching path are: {!r}'.format(environ['PATH']))
+            print('Searching path are: {!r}'.format(env['PATH']))
             raise BuildInterruptingException(
                 'Couldn\'t find executable for CC. This indicates a '
                 'problem locating the {} executable in the Android '
@@ -220,20 +195,13 @@ def get_env(self, with_flags_in_cc=True):
                 ccache=ccache)
 
         # Android's binaries
-        command_prefix = self.command_prefix
-        env['AR'] = '{}-ar'.format(command_prefix)
-        env['RANLIB'] = '{}-ranlib'.format(command_prefix)
-        env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
+        env['STRIP'] = f'{self.clang_path}/llvm-strip --strip-unneeded'
         env['MAKE'] = 'make -j{}'.format(str(cpu_count()))
-        env['READELF'] = '{}-readelf'.format(command_prefix)
-        env['NM'] = '{}-nm'.format(command_prefix)
-        env['LD'] = '{}-ld'.format(command_prefix)
+        env['READELF'] = f'{self.clang_path}/llvm-readelf'
 
         # Android's arch/toolchain
         env['ARCH'] = self.arch
         env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
-        env['TOOLCHAIN_PREFIX'] = self.toolchain_prefix
-        env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version
 
         # Custom linker options
         env['LDSHARED'] = env['CC'] + ' ' + ' '.join(self.common_ldshared)
@@ -251,8 +219,6 @@ def get_env(self, with_flags_in_cc=True):
             ),
         )
 
-        env['PATH'] = environ['PATH']
-
         # for reproducible builds
         if 'SOURCE_DATE_EPOCH' in environ:
             for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split():
@@ -264,9 +230,7 @@ def get_env(self, with_flags_in_cc=True):
 
 class ArchARM(Arch):
     arch = "armeabi"
-    toolchain_prefix = 'arm-linux-androideabi'
     command_prefix = 'arm-linux-androideabi'
-    platform_dir = 'arch-arm'
 
     @property
     def target(self):
@@ -290,12 +254,9 @@ class ArchARMv7_a(ArchARM):
 
 class Archx86(Arch):
     arch = 'x86'
-    toolchain_prefix = 'x86'
     command_prefix = 'i686-linux-android'
-    platform_dir = 'arch-x86'
     arch_cflags = [
         '-march=i686',
-        '-mtune=intel',
         '-mssse3',
         '-mfpmath=sse',
         '-m32',
@@ -304,26 +265,22 @@ class Archx86(Arch):
 
 class Archx86_64(Arch):
     arch = 'x86_64'
-    toolchain_prefix = 'x86_64'
     command_prefix = 'x86_64-linux-android'
-    platform_dir = 'arch-x86_64'
     arch_cflags = [
         '-march=x86-64',
         '-msse4.2',
         '-mpopcnt',
         '-m64',
-        '-mtune=intel',
         '-fPIC',
     ]
 
 
 class ArchAarch_64(Arch):
     arch = 'arm64-v8a'
-    toolchain_prefix = 'aarch64-linux-android'
     command_prefix = 'aarch64-linux-android'
-    platform_dir = 'arch-arm64'
     arch_cflags = [
         '-march=armv8-a',
+        '-fPIC'
         # '-I' + join(dirname(__file__), 'includes', 'arm64-v8a'),
     ]
 

pythonforandroid/bootstraps/common/build/gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-all.zip

pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle

@@ -5,7 +5,7 @@ buildscript {
        jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.5.4'
+        classpath 'com.android.tools.build:gradle:7.0.1'
     }
 }
 

pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml

@@ -62,7 +62,7 @@
                  {{ args.extra_manifest_application_arguments }}
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
                  android:hardwareAccelerated="true"
-                 >
+                 android:extractNativeLibs="true" >
         {% for l in args.android_used_libs %}
         <uses-library android:name="{{ l }}" />
         {% endfor %}

pythonforandroid/bootstraps/service_only/build/templates/AndroidManifest.tmpl.xml

@@ -50,7 +50,8 @@
                  android:allowBackup="{{ args.allow_backup }}"
                  {% if args.backup_rules %}android:fullBackupContent="@xml/{{ args.backup_rules }}"{% endif %}
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
-                 android:hardwareAccelerated="true" >
+                 android:hardwareAccelerated="true"
+                 android:extractNativeLibs="true" >
         {% for l in args.android_used_libs %}
         <uses-library android:name="{{ l }}" />
         {% endfor %}

pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml

@@ -52,6 +52,7 @@
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
                  android:hardwareAccelerated="true"
                  android:usesCleartextTraffic="true"
+                 android:extractNativeLibs="true"
                  {% if debug %}android:debuggable="true"{% endif %}
                  >
         {% for l in args.android_used_libs %}

pythonforandroid/build.py

@@ -1,6 +1,5 @@
 from os.path import (
-    abspath, join, realpath, dirname, expanduser, exists,
-    split, isdir
+    abspath, join, realpath, dirname, expanduser, exists
 )
 from os import environ
 import copy
@@ -40,78 +39,6 @@ def get_ndk_sysroot(ndk_dir):
     return sysroot, sysroot_exists
 
 
-def get_toolchain_versions(ndk_dir, arch):
-    toolchain_versions = []
-    toolchain_path_exists = True
-    toolchain_prefix = arch.toolchain_prefix
-    toolchain_path = join(ndk_dir, 'toolchains')
-    if isdir(toolchain_path):
-        toolchain_contents = glob.glob('{}/{}-*'.format(toolchain_path,
-                                                        toolchain_prefix))
-        toolchain_versions = [split(path)[-1][len(toolchain_prefix) + 1:]
-                              for path in toolchain_contents]
-    else:
-        warning('Could not find toolchain subdirectory!')
-        toolchain_path_exists = False
-    return toolchain_versions, toolchain_path_exists
-
-
-def select_and_check_toolchain_version(sdk_dir, ndk_dir, arch, ndk_sysroot_exists, py_platform):
-    toolchain_versions, toolchain_path_exists = get_toolchain_versions(ndk_dir, arch)
-    ok = ndk_sysroot_exists and toolchain_path_exists
-    toolchain_versions.sort()
-
-    toolchain_versions_gcc = []
-    for toolchain_version in toolchain_versions:
-        if toolchain_version[0].isdigit():
-            # GCC toolchains begin with a number
-            toolchain_versions_gcc.append(toolchain_version)
-
-    if toolchain_versions:
-        info('Found the following toolchain versions: {}'.format(
-            toolchain_versions))
-        info('Picking the latest gcc toolchain, here {}'.format(
-            toolchain_versions_gcc[-1]))
-        toolchain_version = toolchain_versions_gcc[-1]
-    else:
-        warning('Could not find any toolchain for {}!'.format(
-            arch.toolchain_prefix))
-        ok = False
-
-    # Modify the path so that sh finds modules appropriately
-    environ['PATH'] = (
-        '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'
-        'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'
-        '{toolchain_prefix}-{toolchain_version}/prebuilt/'
-        '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'
-        'tools:{path}').format(
-            sdk_dir=sdk_dir, ndk_dir=ndk_dir,
-            toolchain_prefix=arch.toolchain_prefix,
-            toolchain_version=toolchain_version,
-            py_platform=py_platform, path=environ.get('PATH'))
-
-    for executable in (
-        "pkg-config",
-        "autoconf",
-        "automake",
-        "libtoolize",
-        "tar",
-        "bzip2",
-        "unzip",
-        "make",
-        "gcc",
-        "g++",
-    ):
-        if not sh.which(executable):
-            warning(f"Missing executable: {executable} is not installed")
-
-    if not ok:
-        raise BuildInterruptingException(
-            'python-for-android cannot continue due to the missing executables above')
-
-    return toolchain_version
-
-
 def get_targets(sdk_dir):
     if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
         avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager'))
@@ -441,11 +368,14 @@ def prepare_build_environment(self,
         self.ndk_sysroot, ndk_sysroot_exists = get_ndk_sysroot(self.ndk_dir)
         self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include')
 
-        for arch in self.archs:
-            # We assume that the toolchain version is the same for all the archs.
-            self.toolchain_version = select_and_check_toolchain_version(
-                self.sdk_dir, self.ndk_dir, arch, ndk_sysroot_exists, py_platform
-            )
+        self.env["PATH"] = ":".join(
+            [
+                f"{ndk_dir}/toolchains/llvm/prebuilt/{py_platform}-x86_64/bin",
+                ndk_dir,
+                f"{sdk_dir}/tools",
+                environ.get("PATH"),
+            ]
+        )
 
     def __init__(self):
         self.include_dirs = []
@@ -458,8 +388,6 @@ def __init__(self):
         self._ndk_api = None
         self.ndk = None
 
-        self.toolchain_version = None
-
         self.local_recipes = None
         self.copy_libs = False
 

pythonforandroid/recipe.py

@@ -818,7 +818,7 @@ def build_arch(self, arch, *extra_args):
         env = self.get_recipe_env(arch)
         with current_directory(self.get_build_dir(arch.arch)):
             shprint(
-                sh.ndk_build,
+                sh.Command(f"{self.ctx.ndk_dir}/ndk-build"),
                 'V=1',
                 'NDK_DEBUG=' + ("1" if self.ctx.build_as_debuggable else "0"),
                 'APP_PLATFORM=android-' + str(self.ctx.ndk_api),
@@ -1142,7 +1142,6 @@ def get_recipe_env(self, arch, with_flags_in_cc=True):
         env['LDSHARED'] = env['CC'] + ' -shared'
         # shprint(sh.whereis, env['LDSHARED'], _env=env)
         env['LIBLINK'] = 'NOTNONE'
-        env['NDKPLATFORM'] = self.ctx.ndk_sysroot  # FIXME?
         if self.ctx.copy_libs:
             env['COPYLIBS'] = '1'
 

pythonforandroid/recipes/audiostream/__init__.py

@@ -25,7 +25,8 @@ def get_recipe_env(self, arch):
                               jni_path=join(self.ctx.bootstrap.build_dir, 'jni'),
                               sdl_include=sdl_include,
                               sdl_mixer_include=sdl_mixer_include)
-        env['NDKPLATFORM'] = arch.ndk_platform
+        # NDKPLATFORM is our switch for detecting Android platform, so can't be None
+        env['NDKPLATFORM'] = "NOTNONE"
         env['LIBLINK'] = 'NOTNONE'  # Hacky fix. Needed by audiostream setup.py
         return env
 

pythonforandroid/recipes/evdev/__init__.py

@@ -18,7 +18,7 @@ class EvdevRecipe(CompiledComponentsPythonRecipe):
 
     def get_recipe_env(self, arch=None):
         env = super().get_recipe_env(arch)
-        env['NDKPLATFORM'] = arch.ndk_platform
+        env['SYSROOT'] = self.ctx.ndk_sysroot
         return env
 
 

pythonforandroid/recipes/evdev/include-dir.patch

@@ -6,7 +6,7 @@ diff -Naur orig/setup.py v0.4.7/setup.py
  #-----------------------------------------------------------------------------
  def create_ecodes():
 -    header = '/usr/include/linux/input.h'
-+    header = os.environ['NDKPLATFORM'] + '/usr/include/linux/input.h'
++    header = os.environ['SYSROOT'] + '/usr/include/linux/input.h'
 
      if not os.path.isfile(header):
          msg = '''\