diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bc1f71dbdf..40850ffbc0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -41,14 +41,10 @@ jobs: pip install tox>=2.0 make test - build: + build_apk: name: Unit test apk needs: [flake8] runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - build-arch: ['arm64-v8a', 'armeabi-v7a', 'x86_64', 'x86'] steps: - name: Checkout python-for-android uses: actions/checkout@v2 @@ -64,15 +60,43 @@ jobs: - name: Pull docker image run: | make docker/pull - - name: Build apk Python 3 ${{ matrix.build-arch }} + - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | mkdir -p apks - make docker/run/make/with-artifact/testapps-with-numpy/${{ matrix.build-arch }} + make docker/run/make/with-artifact/apk/testapps-with-numpy - uses: actions/upload-artifact@v1 with: - name: bdist_test_app_unittests__${{ matrix.build-arch }}-debug-1.1.apk + name: bdist_unit_tests_app-debug-1.1-.apk path: apks + build_aab: + name: Unit test aab + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@v2 + # helps with GitHub runner getting out of space + - name: Free disk space + run: | + df -h + sudo swapoff -a + sudo rm -f /swapfile + sudo apt -y clean + docker rmi $(docker image ls -aq) + df -h + - name: Pull docker image + run: | + make docker/pull + - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) + run: | + mkdir -p aabs + make docker/run/make/with-artifact/aab/testapps-with-numpy-aab + - uses: actions/upload-artifact@v1 + with: + name: bdist_unit_tests_app-release-1.1-.aab + path: aabs + rebuild_updated_recipes: name: Test updated recipes needs: [flake8] diff --git a/Makefile b/Makefile index be6c1b7c89..80fc8fcf1f 100644 --- a/Makefile +++ b/Makefile @@ -34,12 +34,17 @@ rebuild_updated_recipes: virtualenv ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \ $(PYTHON) ci/rebuild_updated_recipes.py -testapps-with-numpy/%: virtualenv - $(eval $@_APP_ARCH := $(shell basename $*)) +testapps-with-numpy: virtualenv . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ - --arch=$($@_APP_ARCH) + --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 + +testapps-with-numpy-aab: virtualenv + . $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \ + python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \ + --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \ + --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release testapps/%: virtualenv $(eval $@_APP_ARCH := $(shell basename $*)) @@ -69,14 +74,18 @@ docker/run/test: docker/build docker/run/command: docker/build docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)" -docker/run/make/%: docker/build - docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* +docker/run/make/with-artifact/apk/%: docker/build + docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1-.apk ./apks + docker rm -fv p4a-latest -docker/run/make/with-artifact/%: docker/build - $(eval $@_APP_ARCH := $(shell basename $*)) +docker/run/make/with-artifact/aab/%: docker/build docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $* - docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app__$($@_APP_ARCH)-debug-1.1-.apk ./apks + docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1-.aab ./aabs docker rm -fv p4a-latest +docker/run/make/%: docker/build + docker run --rm --env-file=.env $(DOCKER_IMAGE) make $* + docker/run/shell: docker/build docker run --rm --env-file=.env -it $(DOCKER_IMAGE) diff --git a/README.md b/README.md index 519e2762fb..9f69f1bc5c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ python-for-android python-for-android is a packaging tool for Python apps on Android. You can create your own Python distribution including the modules and -dependencies you want, and bundle it in an APK along with your own code. +dependencies you want, and bundle it in an APK or AAB along with your own code. Features include: @@ -19,6 +19,7 @@ Features include: sqlalchemy. - Multiple architecture targets, for APKs optimised on any given device. +- AAB: Android App Bundle support. For documentation and support, see: @@ -30,7 +31,7 @@ For documentation and support, see: Follow the [quickstart instructions]() -to install and begin creating APKs. +to install and begin creating APKs and AABs. **Quick instructions**: install python-for-android with: @@ -52,6 +53,8 @@ With everything installed, build an APK with SDL2 with e.g.: p4a apk --requirements=kivy --private /home/username/devel/planewave_frozen/ --package=net.inclem.planewavessdl2 --name="planewavessdl2" --version=0.5 --bootstrap=sdl2 +**If you need to deploy your app on Google Play, Android App Bundle (aab) is required since 1 August 2021:** + **For full instructions and parameter options,** see [the documentation](https://python-for-android.readthedocs.io/en/latest/quickstart/#usage). @@ -109,6 +112,9 @@ api level below 21, you should use an older version of python-for-android On March of 2020 we dropped support for creating apps that use Python 2. The latest python-for-android release that supported building Python 2 was version 2019.10.6. +On August of 2021, we added support for Android App Bundle (aab). As a collateral, +now We support multi-arch apk. + ## Contributors This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. diff --git a/doc/source/commands.rst b/doc/source/commands.rst index 952c31d6b2..5a0884aa5e 100644 --- a/doc/source/commands.rst +++ b/doc/source/commands.rst @@ -71,8 +71,9 @@ supply those that you need. Whether the distribution must be compiled from scratch. ``--arch`` - The architecture to build for. Currently only one architecture can be - targeted at a time, and a given distribution can only include one architecture. + The architecture to build for. You can specify multiple architectures to build for + at the same time. As an example ``p4a ... --arch arm64-v8a --arch armeabi-v7a ...`` + will build a distribution for both ``arm64-v8a`` and ``armeabi-v7a``. ``--bootstrap BOOTSTRAP`` The Java bootstrap to use for your application. You mostly don't diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst index 90c15cd461..5aa910812a 100644 --- a/doc/source/quickstart.rst +++ b/doc/source/quickstart.rst @@ -213,6 +213,24 @@ You can also replace flask with another web framework. Replace ``--port=5000`` with the port on which your app will serve a website. The default for Flask is 5000. +Exporting the Android App Bundle (aab) for distributing it on Google Play +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Starting from August 2021 for new apps and from November 2021 for updates to existings apps, +Google Play Console will require the Android App Bundle instead of the long lived apk. + +python-for-android handles by itself the needed work to accomplish the new requirements: + + p4a aab --private $HOME/code/myapp --package=org.example.myapp --name="My App" --version 0.1 --bootstrap=sdl2 --requirements=python3,kivy --arch=arm64-v8a --arch=armeabi-v7a --release + +This `p4a aab ...` command builds a distribution with `python3`, +`kivy`, and everything else you specified in the requirements. +It will be packaged using a SDL2 bootstrap, and produce +an `.aab` file that contains binaries for both `armeabi-v7a` and `arm64-v8a` ABIs. + +The Android App Bundle, is supposed to be used for distributing your app. +If you need to test it locally, on your device, you can use `bundletool ` + Other options ~~~~~~~~~~~~~ diff --git a/doc/source/troubleshooting.rst b/doc/source/troubleshooting.rst index 884d24d384..4d04c9954e 100644 --- a/doc/source/troubleshooting.rst +++ b/doc/source/troubleshooting.rst @@ -85,31 +85,35 @@ At the top level, this will always contain the same set of files:: AndroidManifest.xml classes.dex META-INF res assets lib YourApk.apk resources.arsc -The Python distribution is in the assets folder:: +The user app data (code, images, fonts ..) is packaged into a single tarball contained in the assets folder:: $ cd assets $ ls - private.mp3 + private.tar -``private.mp3`` is actually a tarball containing all your packaged -data, and the Python distribution. Extract it:: +``private.tar`` is a tarball containing all your packaged +data. Extract it:: - $ tar xf private.mp3 + $ tar xf private.tar -This will reveal all the Python-related files:: +This will reveal all the user app data (the files shown below are from the touchtracer demo):: $ ls - android_runnable.pyo include interpreter_subprocess main.kv pipinterface.kv settings.pyo - assets __init__.pyo interpreterwrapper.pyo main.pyo pipinterface.pyo utils.pyo - editor.kv interpreter.kv _python_bundle menu.kv private.mp3 widgets.pyo - editor.pyo interpreter.pyo libpymodules.so menu.pyo settings.kv + README.txt android.txt icon.png main.pyc p4a_env_vars.txt particle.png + private.tar touchtracer.kv -Most of these files have been included by the user (in this case, they -come from one of my own apps), the rest relate to the python -distribution. +Due to how We're required to ship ABI-specific things in Android App Bundle, +the Python installation is packaged separately, as (most of it) is ABI-specific. + +For example, the Python installation for ``arm64-v8a`` is available in ``lib/arm64-v8a/libpybundle.so`` + +``libpybundle.so`` is a tarball (but named like a library for packaging requirements), that contains our ``_python_bundle``:: + + $ tar xf libpybundle.so + $ cd _python_bundle + $ ls + modules site-packages stdlib.zip -The python installation, along with all side-packages, is mostly contained -inside the `_python_bundle` folder. Common errors diff --git a/pythonforandroid/archs.py b/pythonforandroid/archs.py index aa661fecce..95d94b5f1b 100644 --- a/pythonforandroid/archs.py +++ b/pythonforandroid/archs.py @@ -1,9 +1,10 @@ from distutils.spawn import find_executable from os import environ -from os.path import join, split +from os.path import join, split, exists from multiprocessing import cpu_count from glob import glob +from pythonforandroid.logger import warning from pythonforandroid.recipe import Recipe from pythonforandroid.util import BuildInterruptingException, build_platform @@ -30,7 +31,7 @@ class Arch: common_cppflags = [ '-DANDROID', '-D__ANDROID_API__={ctx.ndk_api}', - '-I{ctx.ndk_dir}/sysroot/usr/include/{command_prefix}', + '-I{ctx.ndk_sysroot}/usr/include/{command_prefix}', '-I{python_includes}', ] @@ -57,6 +58,24 @@ def __init__(self, ctx): def __str__(self): return self.arch + @property + def ndk_lib_dir(self): + return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api)) + + @property + def ndk_platform(self): + warning("ndk_platform is deprecated and should be avoided in new recipes") + ndk_platform = join( + self.ctx.ndk_dir, + 'platforms', + 'android-{}'.format(self.ctx.ndk_api), + self.platform_dir) + if not exists(ndk_platform): + BuildInterruptingException( + "The requested platform folder doesn't exist. If you're building on ndk >= r22, and seeing this error, one of the required recipe is using a removed feature." + ) + return ndk_platform + @property def include_dirs(self): return [ @@ -133,7 +152,7 @@ def get_env(self, with_flags_in_cc=True): ctx=self.ctx, command_prefix=self.command_prefix, python_includes=join( - self.ctx.get_python_install_dir(), + self.ctx.get_python_install_dir(self.arch), 'include/python{}'.format(self.ctx.python_recipe.version[0:3]), ), ) @@ -213,7 +232,7 @@ def get_env(self, with_flags_in_cc=True): # Android's arch/toolchain env['ARCH'] = self.arch env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api)) - env['TOOLCHAIN_PREFIX'] = self.ctx.toolchain_prefix + env['TOOLCHAIN_PREFIX'] = self.toolchain_prefix env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version # Custom linker options diff --git a/pythonforandroid/bdistapk.py b/pythonforandroid/bdistapk.py index d4b2c7953a..bfc5279736 100644 --- a/pythonforandroid/bdistapk.py +++ b/pythonforandroid/bdistapk.py @@ -128,21 +128,23 @@ def prepare_build_dir(self): class BdistAPK(Bdist): - """ - distutil command handler for 'apk' - """ + """distutil command handler for 'apk'.""" description = 'Create an APK with python-for-android' package_type = 'apk' class BdistAAR(Bdist): - """ - distutil command handler for 'aar' - """ + """distutil command handler for 'aar'.""" description = 'Create an AAR with python-for-android' package_type = 'aar' +class BdistAAB(Bdist): + """distutil command handler for 'aab'.""" + description = 'Create an AAB with python-for-android' + package_type = 'aab' + + def _set_user_options(): # This seems like a silly way to do things, but not sure if there's a # better way to pass arbitrary options onwards to p4a @@ -156,6 +158,7 @@ def _set_user_options(): user_options.append((arg[2:], None, None)) BdistAPK.user_options = user_options + BdistAAB.user_options = user_options _set_user_options() diff --git a/pythonforandroid/bootstrap.py b/pythonforandroid/bootstrap.py index a9e7f4d911..0a5225e526 100755 --- a/pythonforandroid/bootstrap.py +++ b/pythonforandroid/bootstrap.py @@ -374,7 +374,7 @@ def strip_libraries(self, arch): if len(tokens) > 1: strip = strip.bake(tokens[1:]) - libs_dir = join(self.dist_dir, '_python_bundle', + libs_dir = join(self.dist_dir, f'_python_bundle__{arch.arch}', '_python_bundle', 'modules') filens = shprint(sh.find, libs_dir, join(self.dist_dir, 'libs'), '-iname', '*.so', _env=env).stdout.decode('utf-8') diff --git a/pythonforandroid/bootstraps/common/build/build.py b/pythonforandroid/bootstraps/common/build/build.py index f7c2314c1e..dcb6d2ac3b 100644 --- a/pythonforandroid/bootstraps/common/build/build.py +++ b/pythonforandroid/bootstraps/common/build/build.py @@ -267,7 +267,7 @@ def make_package(args): # Package up the private data (public not supported). use_setup_py = get_dist_info_for("use_setup_py", error_if_missing=False) is True - tar_dirs = [env_vars_tarpath] + private_tar_dirs = [env_vars_tarpath] _temp_dirs_to_clean = [] try: if args.private: @@ -277,7 +277,7 @@ def make_package(args): ): print('No setup.py/pyproject.toml used, copying ' 'full private data into .apk.') - tar_dirs.append(args.private) + private_tar_dirs.append(args.private) else: print("Copying main.py's ONLY, since other app data is " "expected in site-packages.") @@ -309,10 +309,7 @@ def make_package(args): ) # Append directory with all main.py's to result apk paths: - tar_dirs.append(main_py_only_dir) - for python_bundle_dir in ('private', '_python_bundle'): - if exists(python_bundle_dir): - tar_dirs.append(python_bundle_dir) + private_tar_dirs.append(main_py_only_dir) if get_bootstrap_name() == "webview": for asset in listdir('webview_includes'): shutil.copy(join('webview_includes', asset), join(assets_dir, asset)) @@ -326,8 +323,13 @@ def make_package(args): shutil.copytree(realpath(asset_src), join(assets_dir, asset_dest)) if args.private or args.launcher: + for arch in get_dist_info_for("archs"): + libs_dir = f"libs/{arch}" + make_tar( + join(libs_dir, 'libpybundle.so'), [f'_python_bundle__{arch}'], args.ignore_path, + optimize_python=args.optimize_python) make_tar( - join(assets_dir, 'private.mp3'), tar_dirs, args.ignore_path, + join(assets_dir, 'private.tar'), private_tar_dirs, args.ignore_path, optimize_python=args.optimize_python) finally: for directory in _temp_dirs_to_clean: @@ -358,9 +360,6 @@ def make_package(args): print("WARNING: Received an --icon_fg or an --icon_bg argument, but not both. " "Ignoring.") - if args.enable_androidx: - shutil.copy('templates/gradle.properties', 'gradle.properties') - if get_bootstrap_name() != "service_only": lottie_splashscreen = join(res_dir, 'raw/splashscreen.json') if args.presplash_lottie: @@ -409,15 +408,17 @@ def make_package(args): version_code = 0 if not args.numeric_version: - # Set version code in format (arch-minsdk-app_version) - arch = get_dist_info_for("archs")[0] - arch_dict = {"x86_64": "9", "arm64-v8a": "8", "armeabi-v7a": "7", "x86": "6"} - arch_code = arch_dict.get(arch, '1') + """ + Set version code in format (10 + minsdk + app_version) + Historically versioning was (arch + minsdk + app_version), + with arch expressed with a single digit from 6 to 9. + Since the multi-arch support, has been changed to 10. + """ min_sdk = args.min_sdk_version for i in args.version.split('.'): version_code *= 100 version_code += int(i) - args.numeric_version = "{}{}{}".format(arch_code, min_sdk, version_code) + args.numeric_version = "{}{}{}".format("10", min_sdk, version_code) if args.intent_filters: with open(args.intent_filters) as fd: @@ -549,6 +550,12 @@ def make_package(args): is_library=(get_bootstrap_name() == 'service_library'), ) + # gradle properties + render( + 'gradle.tmpl.properties', + 'gradle.properties', + args=args) + # ant build templates render( 'build.tmpl.xml', diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java index d67570e18a..7b3e45f739 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java @@ -127,7 +127,7 @@ protected static void recursiveDelete(File f) { f.delete(); } - public static void unpackData( + public static void unpackAsset( Context ctx, final String resource, File target, @@ -170,7 +170,7 @@ public static void unpackData( target.mkdirs(); AssetExtract ae = new AssetExtract(ctx); - if (!ae.extractTar(resource + ".mp3", target.getAbsolutePath())) { + if (!ae.extractTar(resource + ".tar", target.getAbsolutePath(), "private")) { String msg = "Could not extract " + resource + " data."; if (ctx instanceof Activity) { toastError((Activity)ctx, msg); @@ -192,4 +192,33 @@ public static void unpackData( } } } + + public static void unpackPyBundle( + Context ctx, + final String resource, + File target, + boolean cleanup_on_version_update) { + + Log.v(TAG, "Unpacking " + resource + " " + target.getName()); + + // FIXME: Implement a versioning logic to speed-up the startup process (maybe hash-based?). + + // If the disk data is out of date, extract it and write the version file. + Log.v(TAG, "Extracting " + resource + " assets."); + + if (cleanup_on_version_update) { + recursiveDelete(target); + } + target.mkdirs(); + + AssetExtract ae = new AssetExtract(ctx); + if (!ae.extractTar(resource + ".so", target.getAbsolutePath(), "pybundle")) { + String msg = "Could not extract " + resource + " data."; + if (ctx instanceof Activity) { + toastError((Activity)ctx, msg); + } else { + Log.v(TAG, msg); + } + } + } } diff --git a/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java b/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java index e7383258f1..0a5dda6567 100644 --- a/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java +++ b/pythonforandroid/bootstraps/common/build/src/main/java/org/renpy/android/AssetExtract.java @@ -13,6 +13,7 @@ import java.io.FileOutputStream; import java.io.FileNotFoundException; import java.io.File; +import java.io.FileInputStream; import java.util.zip.GZIPInputStream; @@ -28,7 +29,7 @@ public AssetExtract(Context context) { mAssetManager = context.getAssets(); } - public boolean extractTar(String asset, String target) { + public boolean extractTar(String asset, String target, String method) { byte buf[] = new byte[1024 * 1024]; @@ -36,7 +37,12 @@ public boolean extractTar(String asset, String target) { TarInputStream tis = null; try { - assetStream = mAssetManager.open(asset, AssetManager.ACCESS_STREAMING); + if(method == "private"){ + assetStream = mAssetManager.open(asset, AssetManager.ACCESS_STREAMING); + } else if (method == "pybundle") { + assetStream = new FileInputStream(asset); + } + tis = new TarInputStream(new BufferedInputStream(new GZIPInputStream(new BufferedInputStream(assetStream, 8192)), 8192)); } catch (IOException e) { Log.e("python", "opening up extract tar", e); diff --git a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle index b59ee6df41..4d9e287b4b 100644 --- a/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle +++ b/pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle @@ -5,7 +5,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:3.5.2' + classpath 'com.android.tools.build:gradle:3.5.4' } } @@ -39,11 +39,16 @@ android { manifestPlaceholders = {{ args.manifest_placeholders}} } - {% if debug_build -%} + packagingOptions { - doNotStrip '**/*.so' + {% if debug_build -%} + doNotStrip '**/*.so' + {% else %} + exclude 'lib/**/gdbserver' + exclude 'lib/**/gdb.setup' + {%- endif %} } - {%- endif %} + {% if args.sign -%} signingConfigs { diff --git a/pythonforandroid/bootstraps/common/build/templates/gradle.properties b/pythonforandroid/bootstraps/common/build/templates/gradle.properties deleted file mode 100644 index 646c51b977..0000000000 --- a/pythonforandroid/bootstraps/common/build/templates/gradle.properties +++ /dev/null @@ -1,2 +0,0 @@ -android.useAndroidX=true -android.enableJetifier=true diff --git a/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties b/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties new file mode 100644 index 0000000000..334714c1f7 --- /dev/null +++ b/pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties @@ -0,0 +1,5 @@ +{% if args.enable_androidx %} +android.useAndroidX=true +android.enableJetifier=true +{% endif %} +android.bundle.enableUncompressedNativeLibs=false \ No newline at end of file diff --git a/pythonforandroid/bootstraps/sdl2/__init__.py b/pythonforandroid/bootstraps/sdl2/__init__.py index 5f7c9cee9a..662d43c0ef 100644 --- a/pythonforandroid/bootstraps/sdl2/__init__.py +++ b/pythonforandroid/bootstraps/sdl2/__init__.py @@ -15,12 +15,7 @@ class SDL2GradleBootstrap(Bootstrap): def assemble_distribution(self): info_main("# Creating Android project ({})".format(self.name)) - arch = self.ctx.archs[0] - - if len(self.ctx.archs) > 1: - raise ValueError("SDL2/gradle support only one arch") - - info("Copying SDL2/gradle build for {}".format(arch)) + info("Copying SDL2/gradle build") shprint(sh.rm, "-rf", self.dist_dir) shprint(sh.cp, "-r", self.build_dir, self.dist_dir) @@ -33,23 +28,24 @@ def assemble_distribution(self): with current_directory(self.dist_dir): info("Copying Python distribution") - python_bundle_dir = join('_python_bundle', '_python_bundle') - - self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - ensure_dir(python_bundle_dir) - site_packages_dir = self.ctx.python_recipe.create_python_bundle( - join(self.dist_dir, python_bundle_dir), arch) + for arch in self.ctx.archs: + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') + ensure_dir(python_bundle_dir) + + self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)]) + site_packages_dir = self.ctx.python_recipe.create_python_bundle( + join(self.dist_dir, python_bundle_dir), arch) + if not self.ctx.with_debug_symbols: + self.strip_libraries(arch) + self.fry_eggs(site_packages_dir) if 'sqlite3' not in self.ctx.recipe_build_order: with open('blacklist.txt', 'a') as fileh: fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n') - if not self.ctx.with_debug_symbols: - self.strip_libraries(arch) - self.fry_eggs(site_packages_dir) super().assemble_distribution() diff --git a/pythonforandroid/bootstraps/sdl2/build/blacklist.txt b/pythonforandroid/bootstraps/sdl2/build/blacklist.txt index 96f8af8133..d5e230c89a 100644 --- a/pythonforandroid/bootstraps/sdl2/build/blacklist.txt +++ b/pythonforandroid/bootstraps/sdl2/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs diff --git a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java index a0d3ee0a2b..f1fd943a34 100644 --- a/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/sdl2/build/src/main/java/org/kivy/android/PythonActivity.java @@ -104,7 +104,8 @@ private class UnpackFilesTask extends AsyncTask { protected String doInBackground(String... params) { File app_root_file = new File(params[0]); Log.v(TAG, "Ready to unpack"); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); return null; } diff --git a/pythonforandroid/bootstraps/service_only/__init__.py b/pythonforandroid/bootstraps/service_only/__init__.py index 41016d73d0..f00baddf8a 100644 --- a/pythonforandroid/bootstraps/service_only/__init__.py +++ b/pythonforandroid/bootstraps/service_only/__init__.py @@ -37,7 +37,7 @@ def assemble_distribution(self): self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - python_bundle_dir = join('_python_bundle', '_python_bundle') + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) diff --git a/pythonforandroid/bootstraps/service_only/build/blacklist.txt b/pythonforandroid/bootstraps/service_only/build/blacklist.txt index f5d05d44d5..53cc634b7d 100644 --- a/pythonforandroid/bootstraps/service_only/build/blacklist.txt +++ b/pythonforandroid/bootstraps/service_only/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs diff --git a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java index 8abff40bb6..919c42b0ea 100644 --- a/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/service_only/build/src/main/java/org/kivy/android/PythonActivity.java @@ -74,7 +74,8 @@ protected void onCreate(Bundle savedInstanceState) { Log.v(TAG, "Ready to unpack"); File app_root_file = new File(getAppRoot()); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); Log.v(TAG, "About to do super onCreate"); super.onCreate(savedInstanceState); diff --git a/pythonforandroid/bootstraps/webview/__init__.py b/pythonforandroid/bootstraps/webview/__init__.py index 24b4cf4bce..c7a0117b98 100644 --- a/pythonforandroid/bootstraps/webview/__init__.py +++ b/pythonforandroid/bootstraps/webview/__init__.py @@ -34,7 +34,7 @@ def assemble_distribution(self): self.distribute_javaclasses(self.ctx.javaclass_dir, dest_dir=join("src", "main", "java")) - python_bundle_dir = join('_python_bundle', '_python_bundle') + python_bundle_dir = join(f'_python_bundle__{arch.arch}', '_python_bundle') ensure_dir(python_bundle_dir) site_packages_dir = self.ctx.python_recipe.create_python_bundle( join(self.dist_dir, python_bundle_dir), arch) diff --git a/pythonforandroid/bootstraps/webview/build/blacklist.txt b/pythonforandroid/bootstraps/webview/build/blacklist.txt index f5d05d44d5..53cc634b7d 100644 --- a/pythonforandroid/bootstraps/webview/build/blacklist.txt +++ b/pythonforandroid/bootstraps/webview/build/blacklist.txt @@ -1,5 +1,7 @@ # prevent user to include invalid extensions *.apk +*.aab +*.apks *.pxd # eggs diff --git a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java index 4dcddec1f0..b8499849da 100644 --- a/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java +++ b/pythonforandroid/bootstraps/webview/build/src/main/java/org/kivy/android/PythonActivity.java @@ -106,7 +106,8 @@ private class UnpackFilesTask extends AsyncTask { protected String doInBackground(String... params) { File app_root_file = new File(params[0]); Log.v(TAG, "Ready to unpack"); - PythonUtil.unpackData(mActivity, "private", app_root_file, true); + PythonUtil.unpackAsset(mActivity, "private", app_root_file, true); + PythonUtil.unpackPyBundle(mActivity, getApplicationInfo().nativeLibraryDir + "/" + "libpybundle", app_root_file, false); return null; } diff --git a/pythonforandroid/build.py b/pythonforandroid/build.py index 6c1975a8fd..b76deb21a8 100644 --- a/pythonforandroid/build.py +++ b/pythonforandroid/build.py @@ -24,20 +24,20 @@ from pythonforandroid.recommendations import ( check_ndk_version, check_target_api, check_ndk_api, RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API) +from pythonforandroid.util import build_platform -def get_ndk_platform_dir(ndk_dir, ndk_api, arch): - ndk_platform_dir_exists = True - platform_dir = arch.platform_dir - ndk_platform = join( - ndk_dir, - 'platforms', - 'android-{}'.format(ndk_api), - platform_dir) - if not exists(ndk_platform): - warning("ndk_platform doesn't exist: {}".format(ndk_platform)) - ndk_platform_dir_exists = False - return ndk_platform, ndk_platform_dir_exists +def get_ndk_standalone(ndk_dir): + return join(ndk_dir, 'toolchains', 'llvm', 'prebuilt', build_platform) + + +def get_ndk_sysroot(ndk_dir): + sysroot = join(get_ndk_standalone(ndk_dir), 'sysroot') + sysroot_exists = True + if not exists(sysroot): + warning("sysroot doesn't exist: {}".format(sysroot)) + sysroot_exists = False + return sysroot, sysroot_exists def get_toolchain_versions(ndk_dir, arch): @@ -56,6 +56,62 @@ def get_toolchain_versions(ndk_dir, arch): return toolchain_versions, toolchain_path_exists +def select_and_check_toolchain_version(sdk_dir, ndk_dir, arch, ndk_sysroot_exists, py_platform): + toolchain_versions, toolchain_path_exists = get_toolchain_versions(ndk_dir, arch) + ok = ndk_sysroot_exists and toolchain_path_exists + toolchain_versions.sort() + + toolchain_versions_gcc = [] + for toolchain_version in toolchain_versions: + if toolchain_version[0].isdigit(): + # GCC toolchains begin with a number + toolchain_versions_gcc.append(toolchain_version) + + if toolchain_versions: + info('Found the following toolchain versions: {}'.format( + toolchain_versions)) + info('Picking the latest gcc toolchain, here {}'.format( + toolchain_versions_gcc[-1])) + toolchain_version = toolchain_versions_gcc[-1] + else: + warning('Could not find any toolchain for {}!'.format( + arch.toolchain_prefix)) + ok = False + + # Modify the path so that sh finds modules appropriately + environ['PATH'] = ( + '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' + 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' + '{toolchain_prefix}-{toolchain_version}/prebuilt/' + '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' + 'tools:{path}').format( + sdk_dir=sdk_dir, ndk_dir=ndk_dir, + toolchain_prefix=arch.toolchain_prefix, + toolchain_version=toolchain_version, + py_platform=py_platform, path=environ.get('PATH')) + + for executable in ( + "pkg-config", + "autoconf", + "automake", + "libtoolize", + "tar", + "bzip2", + "unzip", + "make", + "gcc", + "g++", + ): + if not sh.which(executable): + warning(f"Missing executable: {executable} is not installed") + + if not ok: + raise BuildInterruptingException( + 'python-for-android cannot continue due to the missing executables above') + + return toolchain_version + + def get_targets(sdk_dir): if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')): avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin', 'avdmanager')) @@ -114,7 +170,9 @@ class Context: ccache = None # whether to use ccache - ndk_platform = None # the ndk platform directory + ndk_standalone = None + ndk_sysroot = None + ndk_include_dir = None # usr/include bootstrap = None bootstrap_build_dir = None @@ -162,8 +220,8 @@ def python_installs_dir(self): ensure_dir(directory) return directory - def get_python_install_dir(self): - return join(self.python_installs_dir, self.bootstrap.distribution.name) + def get_python_install_dir(self, arch): + return join(self.python_installs_dir, self.bootstrap.distribution.name, arch) def setup_dirs(self, storage_dir): '''Calculates all the storage and build dirs, and makes sure @@ -251,8 +309,6 @@ def prepare_build_environment(self, if self._build_env_prepared: return - ok = True - # Work out where the Android SDK is sdk_dir = None if user_sdk_dir: @@ -296,7 +352,9 @@ def prepare_build_environment(self, android_api = int(android_api) self.android_api = android_api - check_target_api(android_api, self.archs[0].arch) + for arch in self.archs: + # Maybe We could remove this one in a near future (ARMv5 is definitely old) + check_target_api(android_api, arch) apis = get_available_apis(self.sdk_dir) info('Available Android APIs are ({})'.format( ', '.join(map(str, apis)))) @@ -375,60 +433,19 @@ def prepare_build_environment(self, ' a python 3 target (which is the default)' ' then THINGS WILL BREAK.') - # This would need to be changed if supporting multiarch APKs - arch = self.archs[0] - toolchain_prefix = arch.toolchain_prefix - self.ndk_platform, ndk_platform_dir_exists = get_ndk_platform_dir( - self.ndk_dir, self.ndk_api, arch) - ok = ok and ndk_platform_dir_exists - py_platform = sys.platform if py_platform in ['linux2', 'linux3']: py_platform = 'linux' - toolchain_versions, toolchain_path_exists = get_toolchain_versions( - self.ndk_dir, arch) - ok = ok and toolchain_path_exists - toolchain_versions.sort() - - toolchain_versions_gcc = [] - for toolchain_version in toolchain_versions: - if toolchain_version[0].isdigit(): - # GCC toolchains begin with a number - toolchain_versions_gcc.append(toolchain_version) - - if toolchain_versions: - info('Found the following toolchain versions: {}'.format( - toolchain_versions)) - info('Picking the latest gcc toolchain, here {}'.format( - toolchain_versions_gcc[-1])) - toolchain_version = toolchain_versions_gcc[-1] - else: - warning('Could not find any toolchain for {}!'.format( - toolchain_prefix)) - ok = False - - self.toolchain_prefix = toolchain_prefix - self.toolchain_version = toolchain_version - # Modify the path so that sh finds modules appropriately - environ['PATH'] = ( - '{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/' - 'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/' - '{toolchain_prefix}-{toolchain_version}/prebuilt/' - '{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/' - 'tools:{path}').format( - sdk_dir=self.sdk_dir, ndk_dir=self.ndk_dir, - toolchain_prefix=toolchain_prefix, - toolchain_version=toolchain_version, - py_platform=py_platform, path=environ.get('PATH')) - - for executable in ("pkg-config", "autoconf", "automake", "libtoolize", - "tar", "bzip2", "unzip", "make", "gcc", "g++"): - if not sh.which(executable): - warning(f"Missing executable: {executable} is not installed") - - if not ok: - raise BuildInterruptingException( - 'python-for-android cannot continue due to the missing executables above') + + self.ndk_standalone = get_ndk_standalone(self.ndk_dir) + self.ndk_sysroot, ndk_sysroot_exists = get_ndk_sysroot(self.ndk_dir) + self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include') + + for arch in self.archs: + # We assume that the toolchain version is the same for all the archs. + self.toolchain_version = select_and_check_toolchain_version( + self.sdk_dir, self.ndk_dir, arch, ndk_sysroot_exists, py_platform + ) def __init__(self): self.include_dirs = [] @@ -441,7 +458,6 @@ def __init__(self): self._ndk_api = None self.ndk = None - self.toolchain_prefix = None self.toolchain_version = None self.local_recipes = None @@ -492,11 +508,11 @@ def prepare_bootstrap(self, bootstrap): def prepare_dist(self): self.bootstrap.prepare_dist_dir() - def get_site_packages_dir(self, arch=None): + def get_site_packages_dir(self, arch): '''Returns the location of site-packages in the python-install build dir. ''' - return self.get_python_install_dir() + return self.get_python_install_dir(arch.arch) def get_libs_dir(self, arch): '''The libs dir for a given arch.''' @@ -600,10 +616,11 @@ def build_recipes(build_order, python_modules, ctx, project_dir, recipe.postbuild_arch(arch) info_main('# Installing pure Python modules') - run_pymodules_install( - ctx, python_modules, project_dir, - ignore_setup_py=ignore_project_setup_py - ) + for arch in ctx.archs: + run_pymodules_install( + ctx, arch, python_modules, project_dir, + ignore_setup_py=ignore_project_setup_py + ) def project_has_setup_py(project_dir): @@ -613,7 +630,7 @@ def project_has_setup_py(project_dir): )) -def run_setuppy_install(ctx, project_dir, env=None): +def run_setuppy_install(ctx, project_dir, env=None, arch=None): env = env or {} with current_directory(project_dir): @@ -651,7 +668,7 @@ def run_setuppy_install(ctx, project_dir, env=None): # Reference: # https://github.com/pypa/pip/issues/6223 ctx_site_packages_dir = os.path.normpath( - os.path.abspath(ctx.get_site_packages_dir()) + os.path.abspath(ctx.get_site_packages_dir(arch)) ) venv_site_packages_dir = os.path.normpath(os.path.join( ctx.build_dir, "venv", "lib", [ @@ -690,7 +707,7 @@ def run_setuppy_install(ctx, project_dir, env=None): ctx.build_dir, "venv", "bin", "pip" ).replace("'", "'\"'\"'") + "' " + "install -c ._tmp_p4a_recipe_constraints.txt -v ." - ).format(ctx.get_site_packages_dir(). + ).format(ctx.get_site_packages_dir(arch). replace("'", "'\"'\"'")), _env=copy.copy(env)) @@ -724,7 +741,7 @@ def run_setuppy_install(ctx, project_dir, env=None): os.remove("._tmp_p4a_recipe_constraints.txt") -def run_pymodules_install(ctx, modules, project_dir=None, +def run_pymodules_install(ctx, arch, modules, project_dir=None, ignore_setup_py=False): """ This function will take care of all non-recipe things, by: @@ -736,8 +753,9 @@ def run_pymodules_install(ctx, modules, project_dir=None, """ - info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE ***') - modules = list(filter(ctx.not_has_package, modules)) + info('*** PYTHON PACKAGE / PROJECT INSTALL STAGE FOR ARCH: {} ***'.format(arch)) + + modules = [m for m in modules if ctx.not_has_package(m, arch)] # We change current working directory later, so this has to be an absolute # path or `None` in case that we didn't supply the `project_dir` via kwargs @@ -754,14 +772,20 @@ def run_pymodules_install(ctx, modules, project_dir=None, # Output messages about what we're going to do: if modules: - info('The requirements ({}) don\'t have recipes, attempting to ' - 'install them with pip'.format(', '.join(modules))) - info('If this fails, it may mean that the module has compiled ' - 'components and needs a recipe.') + info( + "The requirements ({}) don\'t have recipes, attempting to " + "install them with pip".format(', '.join(modules)) + ) + info( + "If this fails, it may mean that the module has compiled " + "components and needs a recipe." + ) if project_dir is not None and \ project_has_setup_py(project_dir) and not ignore_setup_py: - info('Will process project install, if it fails then the ' - 'project may not be compatible for Android install.') + info( + "Will process project install, if it fails then the " + "project may not be compatible for Android install." + ) # Use our hostpython to create the virtualenv host_python = sh.Command(ctx.hostpython) @@ -770,7 +794,7 @@ def run_pymodules_install(ctx, modules, project_dir=None, # Prepare base environment and upgrade pip: base_env = dict(copy.copy(os.environ)) - base_env["PYTHONPATH"] = ctx.get_site_packages_dir() + base_env["PYTHONPATH"] = ctx.get_site_packages_dir(arch) info('Upgrade pip to latest version') shprint(sh.bash, '-c', ( "source venv/bin/activate && pip install -U pip" @@ -793,7 +817,7 @@ def run_pymodules_install(ctx, modules, project_dir=None, # Make sure our build package dir is available, and the virtualenv # site packages come FIRST (so the proper pip version is used): - env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir() + env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir(arch) env["PYTHONPATH"] = os.path.abspath(join( ctx.build_dir, "venv", "lib", "python" + ctx.python_recipe.major_minor_version_string, @@ -814,32 +838,32 @@ def run_pymodules_install(ctx, modules, project_dir=None, fileh.write(line) info('Installing Python modules with pip') - info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. ' - 'A reason for this is often modules compiling ' - 'native code that is unaware of Android cross-compilation ' - 'and does not work without additional ' - 'changes / workarounds.') + info( + "IF THIS FAILS, THE MODULES MAY NEED A RECIPE. " + "A reason for this is often modules compiling " + "native code that is unaware of Android cross-compilation " + "and does not work without additional " + "changes / workarounds." + ) shprint(sh.bash, '-c', ( "venv/bin/pip " + "install -v --target '{0}' --no-deps -r requirements.txt" - ).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")), + ).format(ctx.get_site_packages_dir(arch).replace("'", "'\"'\"'")), _env=copy.copy(env)) # Afterwards, run setup.py if present: if project_dir is not None and ( project_has_setup_py(project_dir) and not ignore_setup_py ): - run_setuppy_install(ctx, project_dir, env) + run_setuppy_install(ctx, project_dir, env, arch.arch) elif not ignore_setup_py: - info("No setup.py found in project directory: " + - str(project_dir) - ) + info("No setup.py found in project directory: " + str(project_dir)) # Strip object files after potential Cython or native code builds: if not ctx.with_debug_symbols: standard_recipe.strip_object_files( - ctx.archs[0], env, build_dir=ctx.build_dir + arch, env, build_dir=ctx.build_dir ) @@ -879,7 +903,7 @@ def biglink(ctx, arch): # Move to the directory containing crtstart_so.o and crtend_so.o # This is necessary with newer NDKs? A gcc bug? - with current_directory(join(ctx.ndk_platform, 'usr', 'lib')): + with current_directory(arch.ndk_lib_dir): do_biglink( join(ctx.get_libs_dir(arch.arch), 'libpymodules.so'), obj_dir.split(' '), diff --git a/pythonforandroid/distribution.py b/pythonforandroid/distribution.py index 8607766eba..ff97f92bfe 100644 --- a/pythonforandroid/distribution.py +++ b/pythonforandroid/distribution.py @@ -46,7 +46,7 @@ def get_distribution( cls, ctx, *, - arch_name, # required keyword argument: there is no sensible default + archs, # required keyword argument: there is no sensible default name=None, recipes=[], ndk_api=None, @@ -70,8 +70,8 @@ def get_distribution( ndk_api : int The NDK API to compile against, included in the dist because it cannot be changed later during APK packaging. - arch_name : str - The target architecture name to compile against, included in the dist because + archs : list + The target architectures list to compile against, included in the dist because it cannot be changed later during APK packaging. recipes : list The recipes that the distribution must contain. @@ -99,7 +99,7 @@ def get_distribution( if name is not None and name: possible_dists = [ d for d in possible_dists if - (d.name == name) and (arch_name in d.archs)] + (d.name == name) and all(arch_name in d.archs for arch_name in archs)] if possible_dists: # There should only be one folder with a given dist name *and* arch. @@ -136,7 +136,7 @@ def get_distribution( continue if ndk_api is not None and dist.ndk_api != ndk_api: continue - if arch_name is not None and arch_name not in dist.archs: + if not all(arch_name in dist.archs for arch_name in archs): continue if (set(dist.recipes) == set(recipes) or (set(recipes).issubset(set(dist.recipes)) and @@ -176,14 +176,10 @@ def get_distribution( dist.name = name dist.dist_dir = join( ctx.dist_dir, - generate_dist_folder_name( - name, - [arch_name] if arch_name is not None else None, - ) - ) + name) dist.recipes = recipes dist.ndk_api = ctx.ndk_api - dist.archs = [arch_name] + dist.archs = archs return dist @@ -265,23 +261,3 @@ def pretty_log_dists(dists, log_func=info): for line in infos: log_func('\t' + line) - - -def generate_dist_folder_name(base_dist_name, arch_names=None): - """Generate the distribution folder name to use, based on a - combination of the input arguments. - - Parameters - ---------- - base_dist_name : str - The core distribution identifier string - arch_names : list of str - The architecture compile targets - """ - if arch_names is None: - arch_names = ["no_arch_specified"] - - return '{}__{}'.format( - base_dist_name, - '_'.join(arch_names) - ) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index 87a9ae9ac4..b28a947367 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -949,7 +949,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): def should_build(self, arch): name = self.folder_name - if self.ctx.has_package(name): + if self.ctx.has_package(name, arch): info('Python package already exists in site-packages') return False info('{} apparently isn\'t already in site-packages'.format(name)) @@ -976,7 +976,7 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): hpenv = env.copy() with current_directory(self.get_build_dir(arch.arch)): shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=hpenv, *self.setup_extra_args) @@ -1142,7 +1142,7 @@ def get_recipe_env(self, arch, with_flags_in_cc=True): env['LDSHARED'] = env['CC'] + ' -shared' # shprint(sh.whereis, env['LDSHARED'], _env=env) env['LIBLINK'] = 'NOTNONE' - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = self.ctx.ndk_sysroot # FIXME? if self.ctx.copy_libs: env['COPYLIBS'] = '1' diff --git a/pythonforandroid/recipes/Pillow/__init__.py b/pythonforandroid/recipes/Pillow/__init__.py index 08abba0f24..a2da43c278 100644 --- a/pythonforandroid/recipes/Pillow/__init__.py +++ b/pythonforandroid/recipes/Pillow/__init__.py @@ -35,9 +35,9 @@ class PillowRecipe(CompiledComponentsPythonRecipe): def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) - env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr') - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + env['ANDROID_ROOT'] = join(arch.ndk_platform, 'usr') + ndk_lib_dir = arch.ndk_lib_dir + ndk_include_dir = self.ctx.ndk_include_dir png = self.get_recipe('png', self.ctx) png_lib_dir = join(png.get_build_dir(arch.arch), '.libs') diff --git a/pythonforandroid/recipes/audiostream/__init__.py b/pythonforandroid/recipes/audiostream/__init__.py index 69e5ab4c49..93f007a5f6 100644 --- a/pythonforandroid/recipes/audiostream/__init__.py +++ b/pythonforandroid/recipes/audiostream/__init__.py @@ -25,7 +25,7 @@ def get_recipe_env(self, arch): jni_path=join(self.ctx.bootstrap.build_dir, 'jni'), sdl_include=sdl_include, sdl_mixer_include=sdl_mixer_include) - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = arch.ndk_platform env['LIBLINK'] = 'NOTNONE' # Hacky fix. Needed by audiostream setup.py return env diff --git a/pythonforandroid/recipes/cffi/__init__.py b/pythonforandroid/recipes/cffi/__init__.py index dbe805eced..06966e0138 100644 --- a/pythonforandroid/recipes/cffi/__init__.py +++ b/pythonforandroid/recipes/cffi/__init__.py @@ -35,9 +35,7 @@ def get_recipe_env(self, arch=None): self.ctx.get_libs_dir(arch.arch)) env['LDFLAGS'] += ' -L{}'.format(os.path.join(self.ctx.bootstrap.build_dir, 'libs', arch.arch)) # required for libc and libdl - ndk_dir = self.ctx.ndk_platform - ndk_lib_dir = os.path.join(ndk_dir, 'usr', 'lib') - env['LDFLAGS'] += ' -L{}'.format(ndk_lib_dir) + env['LDFLAGS'] += ' -L{}'.format(arch.ndk_lib_dir) env['PYTHONPATH'] = ':'.join([ self.ctx.get_site_packages_dir(), env['BUILDLIB_PATH'], diff --git a/pythonforandroid/recipes/evdev/__init__.py b/pythonforandroid/recipes/evdev/__init__.py index 2d53f9b0b8..1973612fa3 100644 --- a/pythonforandroid/recipes/evdev/__init__.py +++ b/pythonforandroid/recipes/evdev/__init__.py @@ -18,7 +18,7 @@ class EvdevRecipe(CompiledComponentsPythonRecipe): def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) - env['NDKPLATFORM'] = self.ctx.ndk_platform + env['NDKPLATFORM'] = arch.ndk_platform return env diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py index 00dbeec6a1..130f6708c8 100644 --- a/pythonforandroid/recipes/freetype/__init__.py +++ b/pythonforandroid/recipes/freetype/__init__.py @@ -47,8 +47,8 @@ def get_recipe_env(self, arch=None, with_harfbuzz=False): ) # android's zlib support - zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') - zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_lib_path = arch.ndk_lib_dir + zlib_includes = self.ctx.ndk_include_dir def add_flag_if_not_added(flag, env_key): if flag not in env[env_key]: diff --git a/pythonforandroid/recipes/icu/__init__.py b/pythonforandroid/recipes/icu/__init__.py index 56e2d49798..f6c43100e0 100644 --- a/pythonforandroid/recipes/icu/__init__.py +++ b/pythonforandroid/recipes/icu/__init__.py @@ -114,7 +114,7 @@ def install_libraries(self, arch): src_include = join( self.get_build_dir(arch.arch), "icu_build", "include") dst_include = join( - self.ctx.get_python_install_dir(), "include", "icu") + self.ctx.get_python_install_dir(arch.arch), "include", "icu") ensure_dir(dst_include) shprint(sh.cp, "-r", join(src_include, "layout"), dst_include) shprint(sh.cp, "-r", join(src_include, "unicode"), dst_include) diff --git a/pythonforandroid/recipes/kivy3/__init__.py b/pythonforandroid/recipes/kivy3/__init__.py index 43b55a4e5d..6f27f62cc9 100644 --- a/pythonforandroid/recipes/kivy3/__init__.py +++ b/pythonforandroid/recipes/kivy3/__init__.py @@ -15,7 +15,7 @@ class Kivy3Recipe(PythonRecipe): def build_arch(self, arch): super().build_arch(arch) suffix = '/kivy3/default.glsl' - shutil.copyfile(self.get_build_dir(arch.arch) + suffix, self.ctx.get_python_install_dir() + suffix) + shutil.copyfile(self.get_build_dir(arch.arch) + suffix, self.ctx.get_python_install_dir(arch.arch) + suffix) recipe = Kivy3Recipe() diff --git a/pythonforandroid/recipes/libiconv/__init__.py b/pythonforandroid/recipes/libiconv/__init__.py index 530497a2ed..111e422d2f 100644 --- a/pythonforandroid/recipes/libiconv/__init__.py +++ b/pythonforandroid/recipes/libiconv/__init__.py @@ -21,7 +21,7 @@ def build_arch(self, arch): shprint( sh.Command('./configure'), '--host=' + arch.command_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), _env=env) shprint(sh.make, '-j' + str(cpu_count()), _env=env) diff --git a/pythonforandroid/recipes/libogg/__init__.py b/pythonforandroid/recipes/libogg/__init__.py index a96eca9c34..51320f429d 100644 --- a/pythonforandroid/recipes/libogg/__init__.py +++ b/pythonforandroid/recipes/libogg/__init__.py @@ -12,7 +12,7 @@ def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) flags = [ - '--with-sysroot=' + self.ctx.ndk_platform, + '--with-sysroot=' + arch.ndk_platform, '--host=' + arch.toolchain_prefix, ] configure = sh.Command('./configure') diff --git a/pythonforandroid/recipes/librt/__init__.py b/pythonforandroid/recipes/librt/__init__.py index 9eb56b3b18..fcd7d5048c 100644 --- a/pythonforandroid/recipes/librt/__init__.py +++ b/pythonforandroid/recipes/librt/__init__.py @@ -18,11 +18,8 @@ class LibRt(Recipe): libc, so we create a symbolic link which we will remove when our build finishes''' - @property - def libc_path(self): - return join(self.ctx.ndk_platform, 'usr', 'lib', 'libc') - def build_arch(self, arch): + libc_path = join(arch.ndk_platform, 'usr', 'lib', 'libc') # Create a temporary folder to add to link path with a fake librt.so: fake_librt_temp_folder = join( self.get_build_dir(arch.arch), @@ -35,13 +32,13 @@ def build_arch(self, arch): if exists(join(fake_librt_temp_folder, "librt.so")): remove(join(fake_librt_temp_folder, "librt.so")) shprint(sh.ln, '-sf', - self.libc_path + '.so', + libc_path + '.so', join(fake_librt_temp_folder, "librt.so"), ) if exists(join(fake_librt_temp_folder, "librt.a")): remove(join(fake_librt_temp_folder, "librt.a")) shprint(sh.ln, '-sf', - self.libc_path + '.a', + libc_path + '.a', join(fake_librt_temp_folder, "librt.a"), ) diff --git a/pythonforandroid/recipes/libsecp256k1/__init__.py b/pythonforandroid/recipes/libsecp256k1/__init__.py index caa5a6fc37..b2e10cb8f2 100644 --- a/pythonforandroid/recipes/libsecp256k1/__init__.py +++ b/pythonforandroid/recipes/libsecp256k1/__init__.py @@ -20,7 +20,7 @@ def build_arch(self, arch): shprint( sh.Command('./configure'), '--host=' + arch.toolchain_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), '--enable-shared', '--enable-module-recovery', '--enable-experimental', diff --git a/pythonforandroid/recipes/libvorbis/__init__.py b/pythonforandroid/recipes/libvorbis/__init__.py index 5f1e3254c1..4599d319a8 100644 --- a/pythonforandroid/recipes/libvorbis/__init__.py +++ b/pythonforandroid/recipes/libvorbis/__init__.py @@ -21,7 +21,7 @@ def build_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): env = self.get_recipe_env(arch) flags = [ - '--with-sysroot=' + self.ctx.ndk_platform, + '--with-sysroot=' + arch.ndk_platform, '--host=' + arch.toolchain_prefix, ] configure = sh.Command('./configure') diff --git a/pythonforandroid/recipes/libzbar/__init__.py b/pythonforandroid/recipes/libzbar/__init__.py index 7a9c15650d..a4b5292abd 100644 --- a/pythonforandroid/recipes/libzbar/__init__.py +++ b/pythonforandroid/recipes/libzbar/__init__.py @@ -34,7 +34,7 @@ def build_arch(self, arch): sh.Command('./configure'), '--host=' + arch.command_prefix, '--target=' + arch.toolchain_prefix, - '--prefix=' + self.ctx.get_python_install_dir(), + '--prefix=' + self.ctx.get_python_install_dir(arch.arch), # Python bindings are compiled in a separated recipe '--with-python=no', '--with-gtk=no', diff --git a/pythonforandroid/recipes/lxml/__init__.py b/pythonforandroid/recipes/lxml/__init__.py index 75e7fcd9f7..13e95bf28e 100644 --- a/pythonforandroid/recipes/lxml/__init__.py +++ b/pythonforandroid/recipes/lxml/__init__.py @@ -51,8 +51,8 @@ def get_recipe_env(self, arch): env['LIBS'] += ' -lxml2' # android's ndk flags - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') - ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + ndk_lib_dir = arch.ndk_lib_dir + ndk_include_dir = self.ndk_include_dir cflags += ' -I' + ndk_include_dir env['LDFLAGS'] += ' -L' + ndk_lib_dir env['LIBS'] += ' -lz -lm -lc' diff --git a/pythonforandroid/recipes/openal/__init__.py b/pythonforandroid/recipes/openal/__init__.py index cfb62f6148..1fc72159c7 100644 --- a/pythonforandroid/recipes/openal/__init__.py +++ b/pythonforandroid/recipes/openal/__init__.py @@ -22,7 +22,7 @@ def build_arch(self, arch): env = self.get_recipe_env(arch) cmake_args = [ '-DCMAKE_TOOLCHAIN_FILE={}'.format('XCompile-Android.txt'), - '-DHOST={}'.format(self.ctx.toolchain_prefix) + '-DHOST={}'.format(arch.toolchain_prefix) ] shprint( sh.cmake, '.', diff --git a/pythonforandroid/recipes/openssl/__init__.py b/pythonforandroid/recipes/openssl/__init__.py index da4fdb6e61..20a93ac4ab 100644 --- a/pythonforandroid/recipes/openssl/__init__.py +++ b/pythonforandroid/recipes/openssl/__init__.py @@ -96,7 +96,8 @@ def get_recipe_env(self, arch=None): env = super().get_recipe_env(arch) env['OPENSSL_VERSION'] = self.version env['MAKE'] = 'make' # This removes the '-j5', which isn't safe - env['ANDROID_NDK'] = self.ctx.ndk_dir + env['CC'] = 'clang' + env['ANDROID_NDK_HOME'] = self.ctx.ndk_dir return env def select_build_arch(self, arch): diff --git a/pythonforandroid/recipes/pandas/__init__.py b/pythonforandroid/recipes/pandas/__init__.py index 40da2fb73c..a217ab635a 100644 --- a/pythonforandroid/recipes/pandas/__init__.py +++ b/pythonforandroid/recipes/pandas/__init__.py @@ -20,7 +20,7 @@ def get_recipe_env(self, arch): # we need the includes from our installed numpy at site packages # because we need some includes generated at numpy's compile time env['NUMPY_INCLUDES'] = join( - self.ctx.get_python_install_dir(), "numpy/core/include", + self.ctx.get_python_install_dir(arch.arch), "numpy/core/include", ) # this flag below is to fix a runtime error: diff --git a/pythonforandroid/recipes/protobuf_cpp/__init__.py b/pythonforandroid/recipes/protobuf_cpp/__init__.py index 9bde198376..5c43e33828 100644 --- a/pythonforandroid/recipes/protobuf_cpp/__init__.py +++ b/pythonforandroid/recipes/protobuf_cpp/__init__.py @@ -115,7 +115,7 @@ def install_python_package(self, arch): hpenv = env.copy() shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=hpenv, *self.setup_extra_args) diff --git a/pythonforandroid/recipes/psycopg2/__init__.py b/pythonforandroid/recipes/psycopg2/__init__.py index e35694e6ce..1d946e7d42 100644 --- a/pythonforandroid/recipes/psycopg2/__init__.py +++ b/pythonforandroid/recipes/psycopg2/__init__.py @@ -43,7 +43,7 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True): shprint(hostpython, 'setup.py', 'build_ext', '--static-libpq', _env=env) shprint(hostpython, 'setup.py', 'install', '-O2', - '--root={}'.format(self.ctx.get_python_install_dir()), + '--root={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--install-lib=.', _env=env) diff --git a/pythonforandroid/recipes/pycrypto/__init__.py b/pythonforandroid/recipes/pycrypto/__init__.py index ae2a375df4..f142d3776d 100644 --- a/pythonforandroid/recipes/pycrypto/__init__.py +++ b/pythonforandroid/recipes/pycrypto/__init__.py @@ -36,7 +36,7 @@ def build_compiled_components(self, arch): with current_directory(self.get_build_dir(arch.arch)): configure = sh.Command('./configure') shprint(configure, '--host=arm-eabi', - '--prefix={}'.format(self.ctx.get_python_install_dir()), + '--prefix={}'.format(self.ctx.get_python_install_dir(arch.arch)), '--enable-shared', _env=env) super().build_compiled_components(arch) diff --git a/pythonforandroid/recipes/pygame/__init__.py b/pythonforandroid/recipes/pygame/__init__.py index 3088b6e8c0..8ec416617d 100644 --- a/pythonforandroid/recipes/pygame/__init__.py +++ b/pythonforandroid/recipes/pygame/__init__.py @@ -28,9 +28,7 @@ def prebuild_arch(self, arch): with current_directory(self.get_build_dir(arch.arch)): setup_template = open(join("buildconfig", "Setup.Android.SDL2.in")).read() env = self.get_recipe_env(arch) - env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr') - - ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib') + env['ANDROID_ROOT'] = join(arch.ndk_platform, 'usr') png = self.get_recipe('png', self.ctx) png_lib_dir = join(png.get_build_dir(arch.arch), '.libs') @@ -43,7 +41,7 @@ def prebuild_arch(self, arch): sdl_includes=( " -I" + join(self.ctx.bootstrap.build_dir, 'jni', 'SDL', 'include') + " -L" + join(self.ctx.bootstrap.build_dir, "libs", str(arch)) + - " -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + ndk_lib_dir), + " -L" + png_lib_dir + " -L" + jpeg_lib_dir + " -L" + arch.ndk_lib_dir), sdl_ttf_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_ttf'), sdl_image_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_image'), sdl_mixer_includes="-I"+join(self.ctx.bootstrap.build_dir, 'jni', 'SDL2_mixer'), diff --git a/pythonforandroid/recipes/pyicu/__init__.py b/pythonforandroid/recipes/pyicu/__init__.py index 17b7c619b3..d1e3749fb7 100644 --- a/pythonforandroid/recipes/pyicu/__init__.py +++ b/pythonforandroid/recipes/pyicu/__init__.py @@ -13,7 +13,7 @@ def get_recipe_env(self, arch): env = super().get_recipe_env(arch) icu_include = join( - self.ctx.get_python_install_dir(), "include", "icu") + self.ctx.get_python_install_dir(arch.arch), "include", "icu") icu_recipe = self.get_recipe('icu', self.ctx) icu_link_libs = icu_recipe.built_libraries.keys() diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py index cb03709a30..7d5c488feb 100644 --- a/pythonforandroid/recipes/python3/__init__.py +++ b/pythonforandroid/recipes/python3/__init__.py @@ -265,8 +265,8 @@ def add_flags(include_flags, link_dirs, link_libs): # the build of zlib module, here we search for android's zlib version # and sets the right flags, so python can be build with android's zlib info("Activating flags for android's zlib") - zlib_lib_path = join(self.ctx.ndk_platform, 'usr', 'lib') - zlib_includes = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include') + zlib_lib_path = arch.ndk_lib_dir + zlib_includes = self.ctx.ndk_include_dir zlib_h = join(zlib_includes, 'zlib.h') try: with open(zlib_h) as fileh: @@ -370,7 +370,7 @@ def create_python_bundle(self, dirn, arch): # Compile to *.pyc/*.pyo the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) # Compile to *.pyc/*.pyo the other python packages (site-packages) - self.compile_python_files(self.ctx.get_python_install_dir()) + self.compile_python_files(self.ctx.get_python_install_dir(arch.arch)) # Bundle compiled python modules to a folder modules_dir = join(dirn, 'modules') @@ -399,9 +399,9 @@ def create_python_bundle(self, dirn, arch): # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) - ensure_dir(self.ctx.get_python_install_dir()) + ensure_dir(self.ctx.get_python_install_dir(arch.arch)) # TODO: Improve the API around walking and copying the files - with current_directory(self.ctx.get_python_install_dir()): + with current_directory(self.ctx.get_python_install_dir(arch.arch)): filens = list(walk_valid_filens( '.', self.site_packages_dir_blacklist, self.site_packages_filen_blacklist)) diff --git a/pythonforandroid/recipes/pyzbar/__init__.py b/pythonforandroid/recipes/pyzbar/__init__.py index 32aaf89185..cf78a558cd 100644 --- a/pythonforandroid/recipes/pyzbar/__init__.py +++ b/pythonforandroid/recipes/pyzbar/__init__.py @@ -16,7 +16,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/recipes/scipy/__init__.py b/pythonforandroid/recipes/scipy/__init__.py index de22a79c54..6d9a2cdda4 100644 --- a/pythonforandroid/recipes/scipy/__init__.py +++ b/pythonforandroid/recipes/scipy/__init__.py @@ -33,7 +33,7 @@ def get_recipe_env(self, arch): sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}" sysroot_include = f'{self.ctx.ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include' libgfortran = f'{self.ctx.ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}/{prefix}/{LIB}' - numpylib = self.ctx.get_python_install_dir() + '/numpy/core/lib' + numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy/core/lib' LDSHARED_opts = env['LDSHARED'].split('clang')[1] env['LAPACK'] = f'{lapack_dir}/lib' diff --git a/pythonforandroid/recipes/zbar/__init__.py b/pythonforandroid/recipes/zbar/__init__.py index 1656895795..c24971e21d 100644 --- a/pythonforandroid/recipes/zbar/__init__.py +++ b/pythonforandroid/recipes/zbar/__init__.py @@ -23,7 +23,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/recipes/zbarlight/__init__.py b/pythonforandroid/recipes/zbarlight/__init__.py index 0f1d718835..36365cd03d 100644 --- a/pythonforandroid/recipes/zbarlight/__init__.py +++ b/pythonforandroid/recipes/zbarlight/__init__.py @@ -16,7 +16,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch, with_flags_in_cc) libzbar = self.get_recipe('libzbar', self.ctx) libzbar_dir = libzbar.get_build_dir(arch.arch) - env['PYTHON_ROOT'] = self.ctx.get_python_install_dir() + env['PYTHON_ROOT'] = self.ctx.get_python_install_dir(arch.arch) env['CFLAGS'] += ' -I' + join(libzbar_dir, 'include') env['LDFLAGS'] += ' -L' + join(libzbar_dir, 'zbar', '.libs') env['LIBS'] = env.get('LIBS', '') + ' -landroid -lzbar' diff --git a/pythonforandroid/recommendations.py b/pythonforandroid/recommendations.py index 00caad1484..5550861282 100644 --- a/pythonforandroid/recommendations.py +++ b/pythonforandroid/recommendations.py @@ -153,6 +153,7 @@ def check_target_api(api, arch): recommendation """ + # FIXME: Should We remove support for armeabi (ARMv5)? if api >= ARMEABI_MAX_TARGET_API and arch == 'armeabi': raise BuildInterruptingException( UNSUPPORTED_NDK_API_FOR_ARMEABI_MESSAGE.format( diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index aa242a4170..2636eaca5b 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -99,8 +99,6 @@ def check_python_dependencies(): toolchain_dir = dirname(__file__) sys.path.insert(0, join(toolchain_dir, "tools", "external")) -APK_SUFFIX = '.apk' - def add_boolean_option(parser, names, no_names=None, default=True, dest=None, description=None): @@ -163,7 +161,7 @@ def dist_from_args(ctx, args): ctx, name=args.dist_name, recipes=split_argument_list(args.requirements), - arch_name=args.arch, + archs=args.arch, ndk_api=args.ndk_api, force_build=args.force_build, require_perfect_match=args.require_perfect_match, @@ -313,8 +311,8 @@ def __init__(self): '(default: {})'.format(default_storage_dir))) generic_parser.add_argument( - '--arch', help='The arch to build for.', - default='armeabi-v7a') + '--arch', help='The archs to build for.', + action='append', default=[]) # Options for specifying the Distribution generic_parser.add_argument( @@ -563,6 +561,11 @@ def add_parser(subparsers, *args, **kwargs): 'apk', help='Build an APK', parents=[parser_packaging]) + add_parser( + subparsers, + 'aab', help='Build an AAB', + parents=[parser_packaging]) + add_parser( subparsers, 'create', help='Compile a set of requirements into a dist', @@ -712,7 +715,7 @@ def add_parser(subparsers, *args, **kwargs): self.ctx.symlink_bootstrap_files = args.symlink_bootstrap_files self.ctx.java_build_tool = args.java_build_tool - self._archs = split_argument_list(args.arch) + self._archs = args.arch self.ctx.local_recipes = args.local_recipes self.ctx.copy_libs = args.copy_libs @@ -1028,7 +1031,7 @@ def _build_package(self, args, package_type): """ Creates an android package using gradle :param args: parser args - :param package_type: one of 'apk', 'aar' + :param package_type: one of 'apk', 'aar', 'aab' :return (gradle output, build_args) """ ctx = self.ctx @@ -1076,9 +1079,17 @@ def _build_package(self, args, package_type): _tail=20, _critical=True, _env=env ) if args.build_mode == "debug": + if package_type == "aab": + raise BuildInterruptingException( + "aab is meant only for distribution and is not available in debug mode. " + "Instead, you can use apk while building for debugging purposes." + ) gradle_task = "assembleDebug" elif args.build_mode == "release": - gradle_task = "assembleRelease" + if package_type == "apk": + gradle_task = "assembleRelease" + elif package_type == "aab": + gradle_task = "bundleRelease" else: raise BuildInterruptingException( "Unknown build mode {} for apk()".format(args.build_mode)) @@ -1092,7 +1103,7 @@ def _finish_package(self, args, output, build_args, package_type, output_dir): :param args: the parser args :param output: RunningCommand output :param build_args: build args as returned by build.parse_args - :param package_type: one of 'apk', 'aar' + :param package_type: one of 'apk', 'aar', 'aab' :param output_dir: where to put the package file """ @@ -1129,11 +1140,12 @@ def _finish_package(self, args, output, build_args, package_type, output_dir): raise BuildInterruptingException('Couldn\'t find the built APK') info_main('# Found android package file: {}'.format(package_file)) + package_extension = f".{package_type}" if package_add_version: info('# Add version number to android package') - package_name = basename(package_file)[:-len(APK_SUFFIX)] + package_name = basename(package_file)[:-len(package_extension)] package_file_dest = "{}-{}-{}".format( - package_name, build_args.version, APK_SUFFIX) + package_name, build_args.version, package_extension) info('# Android package renamed to {}'.format(package_file_dest)) shprint(sh.cp, package_file, package_file_dest) else: @@ -1151,6 +1163,12 @@ def aar(self, args): output_dir = join(self._dist.dist_dir, "build", "outputs", 'aar') self._finish_package(args, output, build_args, 'aar', output_dir) + @require_prebuilt_dist + def aab(self, args): + output, build_args = self._build_package(args, package_type='aab') + output_dir = join(self._dist.dist_dir, "build", "outputs", 'bundle', args.build_mode) + self._finish_package(args, output, build_args, 'aab', output_dir) + @require_prebuilt_dist def create(self, args): """Create a distribution directory if it doesn't already exist, run diff --git a/setup.py b/setup.py index ef9ee0cfb2..18bd3e4e45 100644 --- a/setup.py +++ b/setup.py @@ -103,6 +103,7 @@ def recursively_include(results, directory, patterns): 'distutils.commands': [ 'apk = pythonforandroid.bdistapk:BdistAPK', 'aar = pythonforandroid.bdistapk:BdistAAR', + 'aab = pythonforandroid.bdistapk:BdistAAB', ], }, classifiers=[ diff --git a/testapps/on_device_unit_tests/setup.py b/testapps/on_device_unit_tests/setup.py index be33963e89..fb9570af3f 100644 --- a/testapps/on_device_unit_tests/setup.py +++ b/testapps/on_device_unit_tests/setup.py @@ -38,6 +38,20 @@ # define a basic test app, which can be override passing the proper args to cli options = { 'apk': + { + 'requirements': + 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' + 'chardet,idna', + 'android-api': 27, + 'ndk-api': 21, + 'dist-name': 'bdist_unit_tests_app', + 'arch': 'armeabi-v7a', + 'bootstrap' : 'sdl2', + 'permissions': ['INTERNET', 'VIBRATE'], + 'orientation': 'sensor', + 'service': 'P4a_test_service:app_service.py', + }, + 'aab': { 'requirements': 'sqlite3,libffi,openssl,pyjnius,kivy,python3,requests,urllib3,' diff --git a/tests/recipes/recipe_ctx.py b/tests/recipes/recipe_ctx.py index a162e8f0dc..056ca88454 100644 --- a/tests/recipes/recipe_ctx.py +++ b/tests/recipes/recipe_ctx.py @@ -39,15 +39,13 @@ def setUp(self): self.ctx.setup_dirs(os.getcwd()) self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) self.ctx.bootstrap.distribution = Distribution.get_distribution( - self.ctx, name="sdl2", recipes=self.recipes, arch_name=self.TEST_ARCH, + self.ctx, name="sdl2", recipes=self.recipes, archs=[self.TEST_ARCH], ) self.ctx.recipe_build_order = self.recipe_build_order self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) self.arch = ArchAarch_64(self.ctx) - self.ctx.ndk_platform = ( - f"{self.ctx._ndk_dir}/platforms/" - f"android-{self.ctx.ndk_api}/{self.arch.platform_dir}" - ) + self.ctx.ndk_sysroot = f'{self.ctx._ndk_dir}/sysroot' + self.ctx.ndk_include_dir = f'{self.ctx.ndk_sysroot}/usr/include' self.recipe = Recipe.get_recipe(self.recipe_name, self.ctx) def tearDown(self): diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 39c29047f7..de062d7231 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -53,7 +53,6 @@ def test_build_arch( mock_archs_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") ] - self.ctx.toolchain_prefix = self.arch.toolchain_prefix self.ctx.toolchain_version = "4.9" self.recipe.build_arch(self.arch) diff --git a/tests/recipes/test_pandas.py b/tests/recipes/test_pandas.py index 23cefb43c9..3ac34d1d3b 100644 --- a/tests/recipes/test_pandas.py +++ b/tests/recipes/test_pandas.py @@ -38,7 +38,7 @@ def test_get_recipe_env( self.ctx.recipe_build_order ) numpy_includes = join( - self.ctx.get_python_install_dir(), "numpy/core/include", + self.ctx.get_python_install_dir(self.arch.arch), "numpy/core/include", ) env = self.recipe.get_recipe_env(self.arch) self.assertIn(numpy_includes, env["NUMPY_INCLUDES"]) diff --git a/tests/recipes/test_python3.py b/tests/recipes/test_python3.py index 66698c9162..481c4b3e24 100644 --- a/tests/recipes/test_python3.py +++ b/tests/recipes/test_python3.py @@ -194,7 +194,7 @@ def test_create_python_bundle( expected_sp_paths = [ modules_build_dir, join(recipe_build_dir, 'Lib'), - self.ctx.get_python_install_dir(), + self.ctx.get_python_install_dir(self.arch.arch), ] for n, (sp_call, kw) in enumerate(mock_subprocess.call_args_list): self.assertEqual(sp_call[0][-1], expected_sp_paths[n]) diff --git a/tests/test_archs.py b/tests/test_archs.py index 3b6b86e218..bfd4bebfcc 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -64,7 +64,7 @@ def setUp(self): self.ctx, name="sdl2", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) self.ctx.python_recipe = Recipe.get_recipe("python3", self.ctx) # Here we define the expected compiler, which, as per ndk >= r19, diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 8fcedb53a7..64e11c52ae 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -8,7 +8,7 @@ from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) -from pythonforandroid.distribution import Distribution, generate_dist_folder_name +from pythonforandroid.distribution import Distribution from pythonforandroid.recipe import Recipe from pythonforandroid.archs import ArchARMv7_a from pythonforandroid.build import Context @@ -50,7 +50,7 @@ def setUp_distribution_with_bootstrap(self, bs): self.ctx.bootstrap.distribution = Distribution.get_distribution( self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) def tearDown(self): @@ -85,7 +85,7 @@ def test_attributes(self): # test dist_dir success self.setUp_distribution_with_bootstrap(bs) - expected_folder_name = generate_dist_folder_name('test_prj', [self.TEST_ARCH]) + expected_folder_name = 'test_prj' self.assertTrue( bs.dist_dir.endswith(f"dists/{expected_folder_name}")) @@ -438,8 +438,8 @@ def test_assemble_distribution( mock_strip_libraries.assert_called() expected__python_bundle = os.path.join( self.ctx.dist_dir, - f"{self.ctx.bootstrap.distribution.name}__{self.TEST_ARCH}", - "_python_bundle", + self.ctx.bootstrap.distribution.name, + f"_python_bundle__{self.TEST_ARCH}", "_python_bundle", ) self.assertIn( diff --git a/tests/test_build.py b/tests/test_build.py index 27e3f40a24..1ffa46fc52 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -4,6 +4,7 @@ import jinja2 from pythonforandroid.build import run_pymodules_install +from pythonforandroid.archs import ArchARMv7_a, ArchAarch_64 class TestBuildBasic(unittest.TestCase): @@ -14,10 +15,11 @@ def test_run_pymodules_install_optional_project_dir(self): `project_dir` optional parameter is None, refs #1898 """ ctx = mock.Mock() + ctx.archs = [ArchARMv7_a(ctx), ArchAarch_64(ctx)] modules = [] project_dir = None with mock.patch('pythonforandroid.build.info') as m_info: - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_info.call_args_list[-1] == mock.call( 'No Python modules and no setup.py to process, skipping') @@ -42,13 +44,13 @@ def test_strip_if_with_debug_symbols(self): # Make sure it is NOT called when `with_debug_symbols` is true: ctx.with_debug_symbols = True - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_CythonRecipe().strip_object_files.called is False # Make sure strip object files IS called when # `with_debug_symbols` is fasle: ctx.with_debug_symbols = False - assert run_pymodules_install(ctx, modules, project_dir) is None + assert run_pymodules_install(ctx, ctx.archs[0], modules, project_dir) is None assert m_CythonRecipe().strip_object_files.called is True diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 3342b5f143..423d572252 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -53,7 +53,7 @@ def setUp_distribution_with_bootstrap(self, bs, **kwargs): self.ctx, name=kwargs.pop("name", "test_prj"), recipes=kwargs.pop("recipes", ["python3", "kivy"]), - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], **kwargs ) @@ -111,7 +111,7 @@ def test_get_distribution_no_name(self, mock_exists): returns the proper result which should `unnamed_dist_1`.""" mock_exists.return_value = False self.ctx.bootstrap = Bootstrap().get_bootstrap("sdl2", self.ctx) - dist = Distribution.get_distribution(self.ctx, arch_name=self.TEST_ARCH) + dist = Distribution.get_distribution(self.ctx, archs=[self.TEST_ARCH]) self.assertEqual(dist.name, "unnamed_dist_1") @mock.patch("pythonforandroid.util.chdir") @@ -213,7 +213,7 @@ def test_get_distributions_error_ndk_api_mismatch( self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) mock_get_dists.return_value = [expected_dist] mock_glob.return_value = ["sdl2-python3"] @@ -264,7 +264,7 @@ def test_get_distributions_possible_dists(self, mock_get_dists): self.ctx, name="test_prj", recipes=["python3", "kivy"], - arch_name=self.TEST_ARCH, + archs=[self.TEST_ARCH], ) mock_get_dists.return_value = [expected_dist] self.setUp_distribution_with_bootstrap( diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index d7c73c6319..c6747885ca 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -1,10 +1,12 @@ import io import sys +from os.path import join import pytest from unittest import mock from pythonforandroid.recipe import Recipe from pythonforandroid.toolchain import ToolchainCL from pythonforandroid.util import BuildInterruptingException +from pythonforandroid.build import get_ndk_standalone def patch_sys_argv(argv): @@ -62,14 +64,16 @@ def test_create(self): '--dist-name=test_toolchain', '--activity-class-name=abc.myapp.android.CustomPythonActivity', '--service-class-name=xyz.myapp.android.CustomPythonService', + '--arch=arm64-v8a', + '--arch=armeabi-v7a' ] with patch_sys_argv(argv), mock.patch( 'pythonforandroid.build.get_available_apis' ) as m_get_available_apis, mock.patch( 'pythonforandroid.build.get_toolchain_versions' ) as m_get_toolchain_versions, mock.patch( - 'pythonforandroid.build.get_ndk_platform_dir' - ) as m_get_ndk_platform_dir, mock.patch( + 'pythonforandroid.build.get_ndk_sysroot' + ) as m_get_ndk_sysroot, mock.patch( 'pythonforandroid.toolchain.build_recipes' ) as m_build_recipes, mock.patch( 'pythonforandroid.bootstraps.service_only.' @@ -77,8 +81,10 @@ def test_create(self): ) as m_run_distribute: m_get_available_apis.return_value = [27] m_get_toolchain_versions.return_value = (['4.9'], True) - m_get_ndk_platform_dir.return_value = ( - '/tmp/android-ndk/platforms/android-21/arch-arm', True) + m_get_ndk_sysroot.return_value = ( + join(get_ndk_standalone("/tmp/android-ndk"), "sysroot"), + True, + ) tchain = ToolchainCL() assert tchain.ctx.activity_class_name == 'abc.myapp.android.CustomPythonActivity' assert tchain.ctx.service_class_name == 'xyz.myapp.android.CustomPythonService' @@ -86,10 +92,11 @@ def test_create(self): [mock.call('/tmp/android-sdk')], # linux case [mock.call('/private/tmp/android-sdk')] # macos case ] - assert m_get_toolchain_versions.call_args_list in [ - [mock.call('/tmp/android-ndk', mock.ANY)], # linux case - [mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case - ] + for callargs in m_get_toolchain_versions.call_args_list: + assert callargs in [ + mock.call("/tmp/android-ndk", mock.ANY), # linux case + mock.call("/private/tmp/android-ndk", mock.ANY), # macos case + ] build_order = [ 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', @@ -116,7 +123,7 @@ def test_create_no_sdk_dir(self): """ The `--sdk-dir` is mandatory to `create` a distribution. """ - argv = ['toolchain.py', 'create'] + argv = ['toolchain.py', 'create', '--arch=arm64-v8a', '--arch=armeabi-v7a'] with patch_sys_argv(argv), pytest.raises( BuildInterruptingException ) as ex_info: