From 13fa87652b1ba0e645b6c215b742c464207112a4 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 24 Dec 2023 16:23:39 +0000 Subject: [PATCH] Simplify pre-release test procedure --- release/README.md | 83 +++++++++++++++------------- server/pypi/README-internal.md | 3 +- server/pypi/packages/h5py/test.py | 3 + server/pypi/pkgtest/app/build.gradle | 39 +++---------- 4 files changed, 58 insertions(+), 70 deletions(-) diff --git a/release/README.md b/release/README.md index 6e1fb9c60b..3f3075860a 100644 --- a/release/README.md +++ b/release/README.md @@ -1,32 +1,33 @@ # Product release procedure -## Runtime +## Integration tests -Run `gradlew runtime:check`. +These are fully automated in GitHub Actions. Check the most recent run succeeded, and +make a note of its commit, because we'll be tagging it later. -Run `gradlew -P cmakeBuildType=Release publish`. +Download the `maven` artifact from GitHub Actions, and unpack it into the local `maven` +directory on all machines that will be used by the subsequent tests. -Record pkgtest app sizes and startup times (#5683), and investigate if significantly worse than -the previous version. Remember that the tests and the packages themselves may have changed. -Open demo app, and run the Java and Python unit tests on any device under the following -conditions: +## Unit tests + +Open the demo app in Android Studio, and run the Java and Python unit tests on a +representative device under the following conditions: * Set build variant to "debug". * Clean install, then run tests twice in the same process. * Kill the process, then run tests again in a new process. +* Test "release" variant". * Test "releaseMinify" variant (minify is not enabled in the "release" variant, because it could prevent users importing classes in the Python console). * Change back to "debug" variant, and test a single-ABI build by temporarily changing abiFilters. -* Re-enable all ABIs. -Run Build > Generate Signed APK with the "release" variant. Save a copy of this APK, -because we'll be releasing it on Google Play later. +Download the `demo` artifact from GitHub Actions, and unpack the APK from it. -Use `adb` to install and test the APK on the following devices, with at least one device being -a clean install, and at least one being an upgrade from the previous public release with the -tests already run. +Install the APK and run the Java and Python unit tests on the following devices, with at +least one device being a clean install, and at least one being an upgrade from the +previous public release with the tests already run. * x86 emulator with minSdkVersion * x86_64 emulator with minSdkVersion @@ -34,34 +35,39 @@ tests already run. * Any armeabi-v7a device * Any arm64-v8a device -Record test times in #5683, and investigate if significantly worse than the previous version. -Remember that the tests themselves may have changed. - Test all the UI elements of the app on both minSdkVersion and targetSdkVersion. -Test all the non-default Python versions on the same devices. +For each of the non-default Python versions, run the Java and Python unit tests on the +same devices. -## Gradle plugin +## Performance tests -On one test machine, run `gradlew -P cmakeBuildType=Release publish`. Then, on each test -machine: +Open the pkgtest app in Android Studio, and temporarily edit the top-level build.gradle +file to use the local Chaquopy version. -* Copy `gradle`, `runtime` and (if necessary) `target` from the first test machine. -* Pull the current version of this repository. -* To make sure the artifacts are not overwritten, temporarily disable the `dependsOn - publish` line in `gradle-plugin/build.gradle`. -* Run `gradlew --continue -P cmakeBuildType=Release gradle:check`. +Record sizes and startup times (#5683), and investigate if significantly worse than the +previous version. Remember that the tests and the packages themselves may have changed. ## Package tests -The following builds and tests can take a long time, and it's helpful to parallelize them -as much as possible. So after each build, copy the APK out of the build directory and -install it while running the next build. This is why we test the slowest devices first. +Open the pkgtest app in Android Studio, and temporarily edit the top-level build.gradle +file to use the local Chaquopy version. + +Temporarily edit the app/build.gradle file to set `PACKAGES` to the top 40 recipes, +ordered by number of PyPI downloads: + +* Get the PyPI statistics as described in server/pypi/README-internal.md. +* `cd server/pypi/packages` +* `cat pypi-downloads.csv | cut -d, -f1 | while read name; do if [ -e $name ]; then echo $name; fi; done | head -n40 | tr '\n' ' '` + +As of 2023-12, this is: -Temporarily edit pkgtest/app/build.gradle to call -`addPackages(delegate, PACKAGE_GROUPS[1])`. + numpy cryptography cffi pandas aiohttp yarl greenlet frozenlist grpcio lxml psutil multidict pillow scipy bcrypt matplotlib pynacl scikit-learn kiwisolver regex ruamel-yaml-clib google-crc32c pycryptodomex contourpy pyzmq pycryptodome zope-interface tensorflow h5py tokenizers torch shapely numba llvmlite xgboost scikit-image statsmodels sentencepiece opencv-python torchvision + +Search the package test scripts for the word "Android", and consider adding any packages +which test Chaquopy in a way that isn't covered by the unit tests. Set `abiFilters` to each of the following values (this tests the single-ABI case), and test on a corresponding device: @@ -73,12 +79,9 @@ Set `abiFilters` to `"x86", "x86_64"` (this tests the multi-ABI case), and test following devices, with at least one being a clean install: * x86 emulator with minSdkVersion -* x86_64 emulator with minSdkVersion - * TensorFlow will fail because of #5626, so test that on API 23. +* x86_64 emulator with minSdkVersion (TensorFlow is expected to fail because of #669) * x86_64 emulator with targetSdkVersion -Repeat with `PACKAGE_GROUPS[2]`. - ## Public release @@ -88,7 +91,7 @@ Maven Central](https://central.sonatype.org/publish/publish-manual/#bundle-creat * `com.chaquo.python.gradle.plugin` * `gradle` * `runtime/*` -* `target` (if updated) +* `target` (if necessary) As a backup, also upload them to . @@ -114,8 +117,10 @@ and update all the public repositories as necessary. Open each public app in Android Studio and test it on any device, with a clean install. -Take the signed APK of the demo app which was built above, and release it on Google Play, -updating description and screenshots if necessary. +Close the projects to make sure .idea files are written. + +Take the demo app APK which was tested above, and release it on Google Play, updating +the description and screenshots if necessary. Set reminder to check for Google Play crash reports regularly over the next month. @@ -142,7 +147,9 @@ If major.minor version number has changed: Commit and push all example app repositories. -Commit this repository, add version number tag, and push. +Tag the commit the GitHub Actions artifacts were built from, and push the tag. + +Commit and push this repository. Increment VERSION.txt for next version number. diff --git a/server/pypi/README-internal.md b/server/pypi/README-internal.md index 2a7391c076..d134f4284f 100644 --- a/server/pypi/README-internal.md +++ b/server/pypi/README-internal.md @@ -54,7 +54,8 @@ GROUP BY file.project ORDER BY downloads DESC LIMIT 10000 ``` -Use a 7-day window to avoid any bias from weekday/weekend differences. +Use a 7-day window to avoid any bias from weekday/weekend differences, and make it end +at least 2 days in the past in case there's any delay. ### Run the tests diff --git a/server/pypi/packages/h5py/test.py b/server/pypi/packages/h5py/test.py index a620f5f111..5fd1d928ce 100644 --- a/server/pypi/packages/h5py/test.py +++ b/server/pypi/packages/h5py/test.py @@ -1,6 +1,9 @@ import unittest +# The pyzmq recipe has a test for duplicate module basenames which depends on h5py. Add +# the word "Android" here so it'll show up during the pre-release package tests (see +# release/README.md). class TestH5py(unittest.TestCase): def test_basic(self): diff --git a/server/pypi/pkgtest/app/build.gradle b/server/pypi/pkgtest/app/build.gradle index 0c31c45786..5391f5f956 100644 --- a/server/pypi/pkgtest/app/build.gradle +++ b/server/pypi/pkgtest/app/build.gradle @@ -3,44 +3,20 @@ plugins { id 'com.chaquo.python' } -def PACKAGE_GROUPS = [ - // Group 0 is not included in regular pre-release testing. - [ - // Not in the public package repository. - "cmake-example", "cython-example", "python-example", "rpi-gpio", - - // Conflicts with opencv-python and opencv-contrib-python. - "opencv-contrib-python-headless", "opencv-python-headless", - ], - [ - // Conflicts with more popular packages - "opencv-contrib-python", // opencv-python - "pycrypto", // pycryptodome - - // Test tensorflow and related packages separately to avoid suspected address space - // fragmentation problems on some 32-bit devices (#5719). - "grpcio", "h5py", "tensorflow", "tflite-runtime", - ], -] - -def defaultGroup = [] // Contains every package which isn't listed above. ext.PACKAGES_DIR = "../../packages" -for (f in file(ext.PACKAGES_DIR).listFiles()) { - if (f.isDirectory() && - ! f.name.startsWith("chaquopy-") && - ! PACKAGE_GROUPS.any { f.name in it }) { - defaultGroup.add(f.name) - } -} -PACKAGE_GROUPS.add(defaultGroup) -// Extra dependencies required to run the tests. Each entry is a pip requirement specifier. def SPACY_MODEL = "en_core_web_sm-2.2.5" + +// Tensorflow 2.1.0 is incompatible with protobuf 4.25 +// (https://stackoverflow.com/q/77569103) +def PROTOBUF_REQ = "protobuf<4.25" + +// Extra dependencies required to run the tests. Each entry is a pip requirement specifier. ext.TEST_PACKAGES = [ "argon2-cffi-bindings": ["argon2-cffi"], "backports-zoneinfo": ["tzdata"], "dlib": ["numpy"], - "grpcio": ["protobuf"], + "grpcio": [PROTOBUF_REQ], "photutils": ["scipy"], "pycurl": ["certifi"], "pyzbar": ["pillow"], @@ -49,6 +25,7 @@ ext.TEST_PACKAGES = [ "shapely": ["numpy"], "spacy": ["https://github.com/explosion/spacy-models/releases/download/$SPACY_MODEL/${SPACY_MODEL}.tar.gz"], "soundfile": ["numpy"], + "tensorflow": [PROTOBUF_REQ] ]