[Android] Introducing mlc4j and app packaging

android/MLCChat/README.md

@@ -0,0 +1,6 @@
+# MLC-LLM Android
+
+Checkout [Documentation page](https://llm.mlc.ai/docs/deploy/android.html) for more information.
+
+- run `mlc_llm package`
+- open this `MLCChat/` folder as a project in Android Studio

android/app/build.gradle → android/MLCChat/app/build.gradle

@@ -47,7 +47,7 @@ android {
 }
 
 dependencies {
-    implementation project(":library")
+    implementation project(":mlc4j")
     implementation 'androidx.core:core-ktx:1.10.1'
     implementation 'androidx.lifecycle:lifecycle-runtime-ktx:2.6.1'
     implementation 'androidx.activity:activity-compose:1.7.1'

android/app/src/main/java/ai/mlc/mlcchat/AppViewModel.kt → android/MLCChat/app/src/main/java/ai/mlc/mlcchat/AppViewModel.kt

@@ -38,7 +38,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
     private val modelIdSet = emptySet<String>().toMutableSet()
 
     companion object {
-        const val AppConfigFilename = "app-config.json"
+        const val AppConfigFilename = "mlc-app-config.json"
         const val ModelConfigFilename = "mlc-chat-config.json"
         const val ParamsConfigFilename = "ndarray-cache.json"
         const val ModelUrlSuffix = "resolve/main/"

android/MLCChat/bundle_weight.py

@@ -0,0 +1,65 @@
+import argparse
+import os
+import subprocess
+from pathlib import Path
+
+from mlc_llm.support import logging
+
+logging.enable_logging()
+logger = logging.getLogger(__name__)
+
+
+def main(apk_path: Path, package_output_path: Path):
+    """Push weights to the android device with adb"""
+    # - Install the apk on device.
+    logger.info('Install apk "%s" to device', str(apk_path.absolute()))
+    subprocess.run(["adb", "install", str(apk_path)], check=True, env=os.environ)
+    # - Create the weight directory for the app.
+    device_weihgt_dir = "/storage/emulated/0/Android/data/ai.mlc.mlcchat/files/"
+    logger.info('Creating directory "%s" on device', device_weihgt_dir)
+    subprocess.run(
+        ["adb", "shell", "mkdir", "-p", device_weihgt_dir],
+        check=True,
+        env=os.environ,
+    )
+    for model_weight_dir in (package_output_path / "bundle").iterdir():
+        if model_weight_dir.is_dir():
+            src_path = str(model_weight_dir.absolute())
+            dst_path = "/data/local/tmp/" + model_weight_dir.name
+            logger.info('Pushing local weights "%s" to device location "%s"', src_path, dst_path)
+            subprocess.run(["adb", "push", src_path, dst_path], check=True, env=os.environ)
+
+            src_path = dst_path
+            dst_path = "/storage/emulated/0/Android/data/ai.mlc.mlcchat/files/"
+            logger.info('Move weights from "%s" to "%s"', src_path, dst_path)
+            subprocess.run(["adb", "shell", "mv", src_path, dst_path], check=True, env=os.environ)
+    logger.info("All finished.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("MLC LLM Android Weight Bundle")
+
+    def _parse_apk_path(path: str) -> Path:
+        path = Path(path)
+        if not path.exists():
+            raise ValueError(
+                f"Path {str(path)} is expected to be an apk file, but the file does not exist."
+            )
+        if not path.is_file():
+            raise ValueError(f"Path {str(path)} is expected to be an apk file.")
+        return path
+
+    parser.add_argument(
+        "--apk-path",
+        type=_parse_apk_path,
+        default="app/release/app-release.apk",
+        help="The path to generated MLCChat apk file.",
+    )
+    parser.add_argument(
+        "--package-output-path",
+        type=Path,
+        default="dist",
+        help='The path to the output directory of "mlc_llm package".',
+    )
+    args = parser.parse_args()
+    main(args.apk_path, args.package_output_path)

android/MLCChat/mlc-package-config.json

@@ -0,0 +1,38 @@
+{
+    "device": "android",
+    "model_list": [
+        {
+            "model": "HF://mlc-ai/gemma-2b-it-q4f16_1-MLC",
+            "model_id": "gemma-2b-q4f16_1",
+            "estimated_vram_bytes": 3000000000
+        },
+        {
+            "model": "HF://mlc-ai/Llama-2-7b-chat-hf-q4f16_1-MLC",
+            "estimated_vram_bytes": 4348727787,
+            "model_id": "Llama-2-7b-chat-hf-q4f16_1",
+            "overrides": {
+                "context_window_size": 768,
+                "prefill_chunk_size": 256
+            }
+        },
+        {
+            "model": "HF://mlc-ai/RedPajama-INCITE-Chat-3B-v1-q4f16_1-MLC",
+            "estimated_vram_bytes": 1948348579,
+            "model_id": "RedPajama-INCITE-Chat-3B-v1-q4f16_1"
+        },
+        {
+            "model": "HF://mlc-ai/Mistral-7B-Instruct-v0.2-q4f16_1-MLC",
+            "estimated_vram_bytes": 4275453296,
+            "model_id": "Mistral-7B-Instruct-v0.2-q4f16_1",
+            "overrides": {
+                "sliding_window_size": 768,
+                "prefill_chunk_size": 256
+            }
+        },
+        {
+            "model": "HF://mlc-ai/phi-2-q4f16_1-MLC",
+            "estimated_vram_bytes": 2036816936,
+            "model_id": "phi-2-q4f16_1"
+        }
+    ]
+}

android/settings.gradle → android/MLCChat/settings.gradle

@@ -14,4 +14,5 @@ dependencyResolutionManagement {
 }
 rootProject.name = "MLCChat"
 include ':app'
-include ':library'
+include ':mlc4j'
+project(':mlc4j').projectDir = file('dist/lib/mlc4j')

android/library/CMakeLists.txt → android/mlc4j/CMakeLists.txt

@@ -37,7 +37,7 @@ add_custom_command(
 )
 
 add_library(model_android STATIC IMPORTED)
-set_target_properties(model_android PROPERTIES IMPORTED_LOCATION ${ANDROID_BIN_DIR}/model_lib/libmodel_android.a)
+set_target_properties(model_android PROPERTIES IMPORTED_LOCATION ${ANDROID_BIN_DIR}/lib/libmodel_android.a)
 
 add_library(tvm4j_runtime_packed SHARED ${TVM_HOME}/jvm/native/src/main/native/org_apache_tvm_native_c_api.cc)
 

android/library/build.gradle → android/mlc4j/build.gradle

@@ -19,13 +19,13 @@ android {
     }
     sourceSets {
         main {
-            jniLibs.srcDirs = ['build/output']
+            jniLibs.srcDirs = ['output']
         }
     }
 }
 
 dependencies {
-    implementation fileTree(dir: 'build/output', include: ['*.jar'])
+    implementation fileTree(dir: 'output', include: ['*.jar'])
     implementation 'androidx.core:core-ktx:1.9.0'
     implementation 'androidx.appcompat:appcompat:1.6.1'
     implementation 'com.google.android.material:material:1.10.0'