From 37d21e120d2881c035883d394785f240b8bc6b06 Mon Sep 17 00:00:00 2001 From: Yagna Srinath Reddy Battula Date: Fri, 20 Mar 2020 17:29:39 -0700 Subject: [PATCH] TVM android camera demo (#5005) --- apps/android_camera/.gitignore | 15 + apps/android_camera/README.md | 28 + apps/android_camera/app/build.gradle | 79 +++ .../app/src/main/AndroidManifest.xml | 50 ++ .../Camera2BasicFragment.java | 615 ++++++++++++++++++ .../androidcamerademo/MainActivity.java | 88 +++ .../app/src/main/jni/Android.mk | 59 ++ .../app/src/main/jni/Application.mk | 46 ++ apps/android_camera/app/src/main/jni/build.sh | 26 + .../app/src/main/jni/make/config.mk | 54 ++ .../app/src/main/jni/tvm_runtime.h | 79 +++ .../src/main/res/drawable/item_selector.xml | 23 + .../app/src/main/res/layout/activity_main.xml | 23 + .../res/layout/fragment_camera2_basic.xml | 85 +++ .../app/src/main/res/layout/listview_row.xml | 28 + .../app/src/main/res/values/colors.xml | 20 + .../app/src/main/res/values/strings.xml | 18 + .../app/src/main/res/values/styles.xml | 31 + .../app/src/main/res/xml/provider_paths.xml | 20 + .../app/src/main/rs/yuv420888.rs | 49 ++ apps/android_camera/build.gradle | 49 ++ apps/android_camera/dev_tools/gen_keystore.sh | 20 + apps/android_camera/dev_tools/sign_apk.sh | 24 + apps/android_camera/gradle.properties | 21 + apps/android_camera/models/prepare_model.py | 123 ++++ apps/android_camera/settings.gradle | 18 + tests/lint/add_asf_header.py | 1 - 27 files changed, 1691 insertions(+), 1 deletion(-) create mode 100644 apps/android_camera/.gitignore create mode 100644 apps/android_camera/README.md create mode 100644 apps/android_camera/app/build.gradle create mode 100644 apps/android_camera/app/src/main/AndroidManifest.xml create mode 100644 apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java create mode 100644 apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java create mode 100644 apps/android_camera/app/src/main/jni/Android.mk create mode 100644 apps/android_camera/app/src/main/jni/Application.mk create mode 100755 apps/android_camera/app/src/main/jni/build.sh create mode 100644 apps/android_camera/app/src/main/jni/make/config.mk create mode 100644 apps/android_camera/app/src/main/jni/tvm_runtime.h create mode 100644 apps/android_camera/app/src/main/res/drawable/item_selector.xml create mode 100644 apps/android_camera/app/src/main/res/layout/activity_main.xml create mode 100644 apps/android_camera/app/src/main/res/layout/fragment_camera2_basic.xml create mode 100644 apps/android_camera/app/src/main/res/layout/listview_row.xml create mode 100644 apps/android_camera/app/src/main/res/values/colors.xml create mode 100644 apps/android_camera/app/src/main/res/values/strings.xml create mode 100644 apps/android_camera/app/src/main/res/values/styles.xml create mode 100644 apps/android_camera/app/src/main/res/xml/provider_paths.xml create mode 100644 apps/android_camera/app/src/main/rs/yuv420888.rs create mode 100644 apps/android_camera/build.gradle create mode 100755 apps/android_camera/dev_tools/gen_keystore.sh create mode 100755 apps/android_camera/dev_tools/sign_apk.sh create mode 100644 apps/android_camera/gradle.properties create mode 100644 apps/android_camera/models/prepare_model.py create mode 100644 apps/android_camera/settings.gradle diff --git a/apps/android_camera/.gitignore b/apps/android_camera/.gitignore new file mode 100644 index 000000000000..6c2f76d2093c --- /dev/null +++ b/apps/android_camera/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures +.externalNativeBuild +/app/src/main/jni/jni_helper_func.h +/app/src/main/jni/org_apache_tvm_native_c_api.cc +/app/src/main/jni/org_apache_tvm_native_c_api.h +/app/src/main/obj/ +gradle/ +app/src/main/assets/models diff --git a/apps/android_camera/README.md b/apps/android_camera/README.md new file mode 100644 index 000000000000..c292ce40c582 --- /dev/null +++ b/apps/android_camera/README.md @@ -0,0 +1,28 @@ +[//]: # Licensed to the Apache Software Foundation (ASF) under one +[//]: # or more contributor license agreements. See the NOTICE file +[//]: # distributed with this work for additional information +[//]: # regarding copyright ownership. The ASF licenses this file +[//]: # to you under the Apache License, Version 2.0 (the +[//]: # "License"); you may not use this file except in compliance +[//]: # with the License. You may obtain a copy of the License at +[//]: # +[//]: # http://www.apache.org/licenses/LICENSE-2.0 +[//]: # +[//]: # Unless required by applicable law or agreed to in writing, +[//]: # software distributed under the License is distributed on an +[//]: # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +[//]: # KIND, either express or implied. See the License for the +[//]: # specific language governing permissions and limitations +[//]: # under the License. + +Android Camera Demo Sample App +============================== + +The Android Camera Demo Sample App provides a basic implementation of an Android +app that uses the tvm runtime to perform image classification in real time. + +Converting Models +----------------- + +The `models/prepare_models.py` script provides a example flow for dumping model +parameter files for use by the app. diff --git a/apps/android_camera/app/build.gradle b/apps/android_camera/app/build.gradle new file mode 100644 index 000000000000..8a772a3d29f3 --- /dev/null +++ b/apps/android_camera/app/build.gradle @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +apply plugin: 'com.android.application' + +task buildJni(type: Exec, description: 'Build JNI libs') { + commandLine 'sh', 'src/main/jni/build.sh' +} + +tasks.withType(JavaCompile) { + //compileTask -> compileTask.dependsOn buildJni +} + +android { + compileSdkVersion 29 + defaultConfig { + applicationId "ml.apache.tvm.android.androidcamerademo" + minSdkVersion 24 + targetSdkVersion 29 + renderscriptTargetApi 18 + renderscriptSupportModeEnabled true + versionCode 1 + versionName "1.0" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } + sourceSets { + main { + jni.srcDirs = [] + jniLibs.srcDirs = ['src/main/libs'] + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + buildToolsVersion = '29.0.3' +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { + exclude group: 'com.android.support', module: 'support-annotations' + }) + implementation 'androidx.appcompat:appcompat:1.1.0' + implementation 'androidx.constraintlayout:constraintlayout:1.1.3' + implementation 'com.google.android.material:material:1.1.0' + implementation 'org.apache.tvm:tvm4j-core:0.0.1-SNAPSHOT' + testImplementation 'junit:junit:4.13' + + implementation("androidx.concurrent:concurrent-futures:1.0.0") + implementation "androidx.camera:camera-core:1.0.0-beta01" + implementation "androidx.camera:camera-camera2:1.0.0-beta01" + // If you want to use the CameraX View class + implementation "androidx.camera:camera-view:1.0.0-alpha08" + // If you want to use the CameraX Extensions library + implementation "androidx.camera:camera-extensions:1.0.0-alpha08" + // If you want to use the CameraX Lifecycle library + implementation "androidx.camera:camera-lifecycle:1.0.0-beta01" +} diff --git a/apps/android_camera/app/src/main/AndroidManifest.xml b/apps/android_camera/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000000..0821286d5543 --- /dev/null +++ b/apps/android_camera/app/src/main/AndroidManifest.xml @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + diff --git a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java new file mode 100644 index 000000000000..f598f8e698f8 --- /dev/null +++ b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java @@ -0,0 +1,615 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tvm.android.androidcamerademo; + +import android.annotation.SuppressLint; +import android.content.res.AssetManager; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.media.Image; +import android.os.AsyncTask; +import android.os.Bundle; +import android.os.SystemClock; +import android.util.Log; +import android.util.Size; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.ArrayAdapter; +import android.widget.ListView; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.appcompat.widget.AppCompatTextView; +import androidx.camera.core.Camera; +import androidx.camera.core.CameraSelector; +import androidx.camera.core.ImageAnalysis; +import androidx.camera.core.ImageProxy; +import androidx.camera.core.Preview; +import androidx.camera.lifecycle.ProcessCameraProvider; +import androidx.camera.view.PreviewView; +import androidx.core.content.ContextCompat; +import androidx.fragment.app.Fragment; +import androidx.renderscript.Allocation; +import androidx.renderscript.Element; +import androidx.renderscript.RenderScript; +import androidx.renderscript.Script; +import androidx.renderscript.Type; + +import com.google.common.util.concurrent.ListenableFuture; + +import org.apache.tvm.Function; +import org.apache.tvm.Module; +import org.apache.tvm.NDArray; +import org.apache.tvm.TVMContext; +import org.apache.tvm.TVMType; +import org.apache.tvm.TVMValue; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.PriorityQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public class Camera2BasicFragment extends Fragment { + private static final String TAG = Camera2BasicFragment.class.getSimpleName(); + + // TVM constants + private static final int OUTPUT_INDEX = 0; + private static final int IMG_CHANNEL = 3; + private static final boolean EXE_GPU = false; + private static final int MODEL_INPUT_SIZE = 224; + private static final String MODEL_CL_LIB_FILE = "deploy_lib_opencl.so"; + private static final String MODEL_CPU_LIB_FILE = "deploy_lib_cpu.so"; + private static final String MODEL_GRAPH_FILE = "deploy_graph.json"; + private static final String MODEL_PARAM_FILE = "deploy_param.params"; + private static final String MODEL_LABEL_FILE = "image_net_labels.json"; + private static final String MODELS = "models"; + private static String INPUT_NAME = "input_1"; + private static String[] models; + private static String mCurModel = ""; + private final float[] mCHW = new float[MODEL_INPUT_SIZE * MODEL_INPUT_SIZE * IMG_CHANNEL]; + private final float[] mCHW2 = new float[MODEL_INPUT_SIZE * MODEL_INPUT_SIZE * IMG_CHANNEL]; + private final Semaphore isProcessingDone = new Semaphore(1); + private final ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor( + 3, + 3, + 1, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>() + ); + // rs creation just for demo. Create rs just once in onCreate and use it again. + private RenderScript rs; + private ScriptC_yuv420888 mYuv420; + private boolean mRunClassifier = false; + + private AppCompatTextView mResultView; + private AppCompatTextView mInfoView; + private ListView mModelView; + private AssetManager assetManager; + private Module graphRuntimeModule; + private JSONObject labels; + private ListenableFuture cameraProviderFuture; + private PreviewView previewView; + private ImageAnalysis imageAnalysis; + + static Camera2BasicFragment newInstance() { + return new Camera2BasicFragment(); + } + + private static Matrix getTransformationMatrix( + final int srcWidth, + final int srcHeight, + final int dstWidth, + final int dstHeight, + final int applyRotation, + final boolean maintainAspectRatio) { + final Matrix matrix = new Matrix(); + + if (applyRotation != 0) { + if (applyRotation % 90 != 0) { + Log.w(TAG, "Rotation of %d % 90 != 0 " + applyRotation); + } + + // Translate so center of image is at origin. + matrix.postTranslate(-srcWidth / 2.0f, -srcHeight / 2.0f); + + // Rotate around origin. + matrix.postRotate(applyRotation); + } + + // Account for the already applied rotation, if any, and then determine how + // much scaling is needed for each axis. + final boolean transpose = (Math.abs(applyRotation) + 90) % 180 == 0; + + final int inWidth = transpose ? srcHeight : srcWidth; + final int inHeight = transpose ? srcWidth : srcHeight; + + // Apply scaling if necessary. + if (inWidth != dstWidth || inHeight != dstHeight) { + final float scaleFactorX = dstWidth / (float) inWidth; + final float scaleFactorY = dstHeight / (float) inHeight; + + if (maintainAspectRatio) { + // Scale by minimum factor so that dst is filled completely while + // maintaining the aspect ratio. Some image may fall off the edge. + final float scaleFactor = Math.max(scaleFactorX, scaleFactorY); + matrix.postScale(scaleFactor, scaleFactor); + } else { + // Scale exactly to fill dst from src. + matrix.postScale(scaleFactorX, scaleFactorY); + } + } + + if (applyRotation != 0) { + // Translate back from origin centered reference to destination frame. + matrix.postTranslate(dstWidth / 2.0f, dstHeight / 2.0f); + } + + return matrix; + } + + private String[] getModels() { + String[] models; + try { + models = getActivity().getAssets().list(MODELS); + } catch (IOException e) { + return null; + } + return models; + } + + @SuppressLint("DefaultLocale") + private String[] inference(float[] chw) { + NDArray inputNdArray = NDArray.empty(new long[]{1, IMG_CHANNEL, MODEL_INPUT_SIZE, MODEL_INPUT_SIZE}, new TVMType("float32")); + inputNdArray.copyFrom(chw); + Function setInputFunc = graphRuntimeModule.getFunction("set_input"); + setInputFunc.pushArg(INPUT_NAME).pushArg(inputNdArray).invoke(); + // release tvm local variables + inputNdArray.release(); + setInputFunc.release(); + + // get the function from the module(run it) + Function runFunc = graphRuntimeModule.getFunction("run"); + runFunc.invoke(); + // release tvm local variables + runFunc.release(); + + // get the function from the module(get output data) + NDArray outputNdArray = NDArray.empty(new long[]{1, 1000}, new TVMType("float32")); + Function getOutputFunc = graphRuntimeModule.getFunction("get_output"); + getOutputFunc.pushArg(OUTPUT_INDEX).pushArg(outputNdArray).invoke(); + float[] output = outputNdArray.asFloatArray(); + // release tvm local variables + outputNdArray.release(); + getOutputFunc.release(); + + if (null != output) { + String[] results = new String[5]; + // top-5 + PriorityQueue pq = new PriorityQueue<>(1000, (Integer idx1, Integer idx2) -> output[idx1] > output[idx2] ? -1 : 1); + + // display the result from extracted output data + for (int j = 0; j < output.length; ++j) { + pq.add(j); + } + for (int l = 0; l < 5; l++) { + //noinspection ConstantConditions + int idx = pq.poll(); + if (idx < labels.length()) { + try { + results[l] = String.format("%.2f", output[idx]) + " : " + labels.getString(Integer.toString(idx)); + } catch (JSONException e) { + Log.e(TAG, "index out of range", e); + } + } else { + results[l] = "???: unknown"; + } + } + return results; + } + return new String[5]; + } + + private void updateActiveModel() { + Log.i(TAG, "updating active model..."); + new LoadModelAsyncTask().execute(); + } + + @Override + public void onViewCreated(final View view, Bundle savedInstanceState) { + + mResultView = view.findViewById(R.id.resultTextView); + mInfoView = view.findViewById(R.id.infoTextView); + mModelView = view.findViewById(R.id.modelListView); + if (assetManager == null) { + assetManager = getActivity().getAssets(); + } + + mModelView.setChoiceMode(ListView.CHOICE_MODE_SINGLE); + models = getModels(); + + ArrayAdapter modelAdapter = + new ArrayAdapter<>( + getContext(), R.layout.listview_row, R.id.listview_row_text, models); + mModelView.setAdapter(modelAdapter); + mModelView.setItemChecked(0, true); + mModelView.setOnItemClickListener( + (parent, view1, position, id) -> updateActiveModel()); + + new LoadModelAsyncTask().execute(); + } + + @Override + public void onActivityCreated(Bundle savedInstanceState) { + super.onActivityCreated(savedInstanceState); + } + + @Override + public void onDestroy() { + // release tvm local variables + if (null != graphRuntimeModule) + graphRuntimeModule.release(); + super.onDestroy(); + } + + /** + * Read file from assets and return byte array. + * + * @param assets The asset manager to be used to load assets. + * @param fileName The filepath of read file. + * @return byte[] file content + * @throws IOException + */ + private byte[] getBytesFromFile(AssetManager assets, String fileName) throws IOException { + InputStream is = assets.open(fileName); + int length = is.available(); + byte[] bytes = new byte[length]; + // Read in the bytes + int offset = 0; + int numRead; + try { + while (offset < bytes.length + && (numRead = is.read(bytes, offset, bytes.length - offset)) >= 0) { + offset += numRead; + } + } finally { + is.close(); + } + // Ensure all the bytes have been read in + if (offset < bytes.length) { + throw new IOException("Could not completely read file " + fileName); + } + return bytes; + } + + /** + * Get application cache path where to place compiled functions. + * + * @param fileName library file name. + * @return String application cache folder path + * @throws IOException + */ + private String getTempLibFilePath(String fileName) throws IOException { + File tempDir = File.createTempFile("tvm4j_demo_", ""); + if (!tempDir.delete() || !tempDir.mkdir()) { + throw new IOException("Couldn't create directory " + tempDir.getAbsolutePath()); + } + return (tempDir + File.separator + fileName); + } + + private Bitmap YUV_420_888_toRGB(Image image, int width, int height) { + // Get the three image planes + Image.Plane[] planes = image.getPlanes(); + ByteBuffer buffer = planes[0].getBuffer(); + byte[] y = new byte[buffer.remaining()]; + buffer.get(y); + + buffer = planes[1].getBuffer(); + byte[] u = new byte[buffer.remaining()]; + buffer.get(u); + + buffer = planes[2].getBuffer(); + byte[] v = new byte[buffer.remaining()]; + buffer.get(v); + + int yRowStride = planes[0].getRowStride(); + int uvRowStride = planes[1].getRowStride(); + int uvPixelStride = planes[1].getPixelStride(); + + + // Y,U,V are defined as global allocations, the out-Allocation is the Bitmap. + // Note also that uAlloc and vAlloc are 1-dimensional while yAlloc is 2-dimensional. + Type.Builder typeUcharY = new Type.Builder(rs, Element.U8(rs)); + typeUcharY.setX(yRowStride).setY(height); + Allocation yAlloc = Allocation.createTyped(rs, typeUcharY.create()); + yAlloc.copyFrom(y); + mYuv420.set_ypsIn(yAlloc); + + Type.Builder typeUcharUV = new Type.Builder(rs, Element.U8(rs)); + // note that the size of the u's and v's are as follows: + // ( (width/2)*PixelStride + padding ) * (height/2) + // = (RowStride ) * (height/2) + typeUcharUV.setX(u.length); + Allocation uAlloc = Allocation.createTyped(rs, typeUcharUV.create()); + uAlloc.copyFrom(u); + mYuv420.set_uIn(uAlloc); + + Allocation vAlloc = Allocation.createTyped(rs, typeUcharUV.create()); + vAlloc.copyFrom(v); + mYuv420.set_vIn(vAlloc); + + // handover parameters + mYuv420.set_picWidth(width); + mYuv420.set_uvRowStride(uvRowStride); + mYuv420.set_uvPixelStride(uvPixelStride); + + Bitmap outBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + Allocation outAlloc = Allocation.createFromBitmap(rs, outBitmap, Allocation.MipmapControl.MIPMAP_NONE, Allocation.USAGE_SCRIPT); + + Script.LaunchOptions lo = new Script.LaunchOptions(); + lo.setX(0, width); // by this we ignore the y’s padding zone, i.e. the right side of x between width and yRowStride + lo.setY(0, height); + + mYuv420.forEach_doConvert(outAlloc, lo); + outAlloc.copyTo(outBitmap); + + return outBitmap; + } + + private float[] getFrame(ImageProxy imageProxy) { + @SuppressLint("UnsafeExperimentalUsageError") + Image image = imageProxy.getImage(); + // extract the jpeg content + if (image == null) { + return null; + } + Bitmap imageBitmap = YUV_420_888_toRGB(image, image.getWidth(), image.getHeight()); + + imageProxy.close(); + // crop input image at centre to model input size + Bitmap cropImageBitmap = Bitmap.createBitmap(MODEL_INPUT_SIZE, MODEL_INPUT_SIZE, Bitmap.Config.ARGB_8888); + Matrix frameToCropTransform = getTransformationMatrix(imageBitmap.getWidth(), imageBitmap.getHeight(), + MODEL_INPUT_SIZE, MODEL_INPUT_SIZE, 0, true); + Canvas canvas = new Canvas(cropImageBitmap); + canvas.drawBitmap(imageBitmap, frameToCropTransform, null); + + // image pixel int values + int[] pixelValues = new int[MODEL_INPUT_SIZE * MODEL_INPUT_SIZE]; + // image RGB float values + + // pre-process the image data from 0-255 int to normalized float based on the + // provided parameters. + cropImageBitmap.getPixels(pixelValues, 0, MODEL_INPUT_SIZE, 0, 0, MODEL_INPUT_SIZE, MODEL_INPUT_SIZE); + for (int j = 0; j < pixelValues.length; ++j) { + mCHW2[j * 3 + 0] = ((pixelValues[j] >> 16) & 0xFF) / 255.0f; + mCHW2[j * 3 + 1] = ((pixelValues[j] >> 8) & 0xFF) / 255.0f; + mCHW2[j * 3 + 2] = (pixelValues[j] & 0xFF) / 255.0f; + } + + // pre-process the image rgb data transpose based on the provided parameters. + for (int k = 0; k < IMG_CHANNEL; ++k) { + for (int l = 0; l < MODEL_INPUT_SIZE; ++l) { + for (int m = 0; m < MODEL_INPUT_SIZE; ++m) { + int dst_index = m + MODEL_INPUT_SIZE * l + MODEL_INPUT_SIZE * MODEL_INPUT_SIZE * k; + int src_index = k + IMG_CHANNEL * m + IMG_CHANNEL * MODEL_INPUT_SIZE * l; + mCHW[dst_index] = mCHW2[src_index]; + } + } + } + + return mCHW; + } + + @Override + public void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + cameraProviderFuture = ProcessCameraProvider.getInstance(getActivity()); + } + + @SuppressLint({"RestrictedApi", "UnsafeExperimentalUsageError"}) + @Nullable + @Override + public View onCreateView(@NonNull LayoutInflater inflater, @Nullable ViewGroup container, @Nullable Bundle savedInstanceState) { + View v = inflater.inflate(R.layout.fragment_camera2_basic, container, false); + previewView = v.findViewById(R.id.textureView); + rs = RenderScript.create(getActivity()); + mYuv420 = new ScriptC_yuv420888(rs); + cameraProviderFuture.addListener(() -> { + try { + ProcessCameraProvider cameraProvider = cameraProviderFuture.get(); + bindPreview(cameraProvider); + } catch (ExecutionException | InterruptedException e) { + // No errors need to be handled for this Future. This should never be reached + } + }, ContextCompat.getMainExecutor(getActivity())); + + imageAnalysis = new ImageAnalysis.Builder() + .setTargetResolution(new Size(224, 224)) + .setMaxResolution(new Size(300, 300)) + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build(); + + imageAnalysis.setAnalyzer(threadPoolExecutor, image -> { + Log.e(TAG, "w: " + image.getWidth() + " h: " + image.getHeight()); + if (mRunClassifier && isProcessingDone.tryAcquire()) { + long t1 = SystemClock.uptimeMillis(); + //float[] chw = getFrame(image); + //float[] chw = YUV_420_888_toRGBPixels(image); + float[] chw = getFrame(image); + if (chw != null) { + long t2 = SystemClock.uptimeMillis(); + String[] results = inference(chw); + long t3 = SystemClock.uptimeMillis(); + StringBuilder msgBuilder = new StringBuilder(); + for (int l = 1; l < 5; l++) { + msgBuilder.append(results[l]).append("\n"); + } + String msg = msgBuilder.toString(); + msg += "getFrame(): " + (t2 - t1) + "ms" + "\n"; + msg += "inference(): " + (t3 - t2) + "ms" + "\n"; + String finalMsg = msg; + this.getActivity().runOnUiThread(() -> { + mResultView.setText(String.format("model: %s \n %s", mCurModel, results[0])); + mInfoView.setText(finalMsg); + }); + } + isProcessingDone.release(); + } + image.close(); + }); + return v; + } + + private void bindPreview(@NonNull ProcessCameraProvider cameraProvider) { + @SuppressLint("RestrictedApi") Preview preview = new Preview.Builder() + .setMaxResolution(new Size(800, 800)) + .setTargetName("Preview") + .build(); + + preview.setSurfaceProvider(previewView.getPreviewSurfaceProvider()); + CameraSelector cameraSelector = + new CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_BACK) + .build(); + Camera camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageAnalysis); + } + + @Override + public void onDestroyView() { + threadPoolExecutor.shutdownNow(); + super.onDestroyView(); + } + + private void setInputName(String modelName) { + if (modelName.equals("mobilenet_v2")) { + INPUT_NAME = "input_1"; + } else if (modelName.equals("resnet18_v1")) { + INPUT_NAME = "data"; + } else { + throw new RuntimeException("Model input may not be right. Please set INPUT_NAME here explicitly."); + } + } + + /* + Load precompiled model on TVM graph runtime and init the system. + */ + private class LoadModelAsyncTask extends AsyncTask { + + @Override + protected Integer doInBackground(Void... args) { + mRunClassifier = false; + // load synset name + int modelIndex = mModelView.getCheckedItemPosition(); + setInputName(models[modelIndex]); + String model = MODELS + "/" + models[modelIndex]; + String labelFilename = MODEL_LABEL_FILE; + Log.i(TAG, "Reading labels from: " + model + "/" + labelFilename); + try { + labels = new JSONObject(new String(getBytesFromFile(assetManager, model + "/" + labelFilename))); + } catch (IOException | JSONException e) { + Log.e(TAG, "Problem reading labels name file!", e); + return -1;//failure + } + + // load json graph + String modelGraph; + String graphFilename = MODEL_GRAPH_FILE; + Log.i(TAG, "Reading json graph from: " + model + "/" + graphFilename); + try { + modelGraph = new String(getBytesFromFile(assetManager, model + "/" + graphFilename)); + } catch (IOException e) { + Log.e(TAG, "Problem reading json graph file!", e); + return -1;//failure + } + + // upload tvm compiled function on application cache folder + String libCacheFilePath; + String libFilename = EXE_GPU ? MODEL_CL_LIB_FILE : MODEL_CPU_LIB_FILE; + Log.i(TAG, "Uploading compiled function to cache folder"); + try { + libCacheFilePath = getTempLibFilePath(libFilename); + byte[] modelLibByte = getBytesFromFile(assetManager, model + "/" + libFilename); + FileOutputStream fos = new FileOutputStream(libCacheFilePath); + fos.write(modelLibByte); + fos.close(); + } catch (IOException e) { + Log.e(TAG, "Problem uploading compiled function!", e); + return -1;//failure + } + + // load parameters + byte[] modelParams; + try { + modelParams = getBytesFromFile(assetManager, model + "/" + MODEL_PARAM_FILE); + } catch (IOException e) { + Log.e(TAG, "Problem reading params file!", e); + return -1;//failure + } + + Log.i(TAG, "creating java tvm context..."); + // create java tvm context + TVMContext tvmCtx = EXE_GPU ? TVMContext.opencl() : TVMContext.cpu(); + + Log.i(TAG, "loading compiled functions..."); + Log.i(TAG, libCacheFilePath); + // tvm module for compiled functions + Module modelLib = Module.load(libCacheFilePath); + + + // get global function module for graph runtime + Log.i(TAG, "getting graph runtime create handle..."); + + Function runtimeCreFun = Function.getFunction("tvm.graph_runtime.create"); + Log.i(TAG, "creating graph runtime..."); + + Log.i(TAG, "ctx type: " + tvmCtx.deviceType); + Log.i(TAG, "ctx id: " + tvmCtx.deviceId); + + TVMValue runtimeCreFunRes = runtimeCreFun.pushArg(modelGraph) + .pushArg(modelLib) + .pushArg(tvmCtx.deviceType) + .pushArg(tvmCtx.deviceId) + .invoke(); + + Log.i(TAG, "as module..."); + graphRuntimeModule = runtimeCreFunRes.asModule(); + Log.i(TAG, "getting graph runtime load params handle..."); + // get the function from the module(load parameters) + Function loadParamFunc = graphRuntimeModule.getFunction("load_params"); + Log.i(TAG, "loading params..."); + loadParamFunc.pushArg(modelParams).invoke(); + // release tvm local variables + modelLib.release(); + loadParamFunc.release(); + runtimeCreFun.release(); + mCurModel = model; + mRunClassifier = true; + return 0;//success + } + } +} diff --git a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java new file mode 100644 index 000000000000..f9c573a5d1fe --- /dev/null +++ b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tvm.android.androidcamerademo; + +import android.content.pm.PackageInfo; +import android.content.pm.PackageManager; +import android.os.Bundle; +import android.widget.Toast; + +import androidx.annotation.NonNull; +import androidx.appcompat.app.AppCompatActivity; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +public class MainActivity extends AppCompatActivity implements + ActivityCompat.OnRequestPermissionsResultCallback { + + private static final int PERMISSIONS_REQUEST_CODE = 1; + + private String[] getRequiredPermissions() { + try { + PackageInfo info = getPackageManager() + .getPackageInfo(getPackageName(), PackageManager.GET_PERMISSIONS); + String[] ps = info.requestedPermissions; + if (ps != null && ps.length > 0) { + return ps; + } else { + return new String[0]; + } + } catch (Exception e) { + return new String[0]; + } + } + + private boolean allPermissionsGranted() { + for (String permission : getRequiredPermissions()) { + if (ContextCompat.checkSelfPermission(this, permission) + != PackageManager.PERMISSION_GRANTED) { + return false; + } + } + return true; + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + if (!allPermissionsGranted()) { + requestPermissions(getRequiredPermissions(), PERMISSIONS_REQUEST_CODE); + return; + } + startFragment(); + } + + private void startFragment() { + getSupportFragmentManager() + .beginTransaction() + .replace(R.id.container, Camera2BasicFragment.newInstance()) + .commit(); + } + + @Override + public void onRequestPermissionsResult( + int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + if (allPermissionsGranted()) { + startFragment(); + } else { + Toast.makeText(this, "Required permissions are not granted. App may not run", Toast.LENGTH_SHORT).show(); + finish(); + } + } +} diff --git a/apps/android_camera/app/src/main/jni/Android.mk b/apps/android_camera/app/src/main/jni/Android.mk new file mode 100644 index 000000000000..f135c97602ee --- /dev/null +++ b/apps/android_camera/app/src/main/jni/Android.mk @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +LOCAL_PATH := $(call my-dir) +MY_PATH := $(LOCAL_PATH) + +include $(CLEAR_VARS) + +LOCAL_PATH := $(MY_PATH) +ROOT_PATH := $(MY_PATH)/../../../../../.. + +ifndef config + ifneq ("$(wildcard ./config.mk)","") + config ?= config.mk + else + config ?= make/config.mk + endif +endif + +include $(config) + +LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc +LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog + +LOCAL_C_INCLUDES := $(ROOT_PATH)/include \ + $(ROOT_PATH)/3rdparty/dlpack/include \ + $(ROOT_PATH)/3rdparty/dmlc-core/include \ + $(ROOT_PATH)/3rdparty/HalideIR/src \ + $(ROOT_PATH)/topi/include + +LOCAL_MODULE = tvm4j_runtime_packed + +LOCAL_CPP_FEATURES += exceptions +LOCAL_LDLIBS += -latomic +LOCAL_ARM_MODE := arm + +ifdef ADD_C_INCLUDES + LOCAL_C_INCLUDES += $(ADD_C_INCLUDES) +endif + +ifdef ADD_LDLIBS + LOCAL_LDLIBS += $(ADD_LDLIBS) +endif + +include $(BUILD_SHARED_LIBRARY) \ No newline at end of file diff --git a/apps/android_camera/app/src/main/jni/Application.mk b/apps/android_camera/app/src/main/jni/Application.mk new file mode 100644 index 000000000000..95a5a9697bcc --- /dev/null +++ b/apps/android_camera/app/src/main/jni/Application.mk @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ifndef config + ifneq ("$(wildcard ./config.mk)","") + config ?= config.mk + else + config ?= make/config.mk + endif +endif + +include $(config) + +# We target every architecture except armeabi here, for two reasons: +# 1) armeabi is deprecated in NDK r16 and removed in r17 +# 2) vulkan is not supported in armeabi +APP_ABI ?= all +APP_STL := c++_shared + +APP_CPPFLAGS += -DDMLC_LOG_STACK_TRACE=0 -DTVM4J_ANDROID=1 -std=c++11 -Oz -frtti +ifeq ($(USE_OPENCL), 1) + APP_CPPFLAGS += -DTVM_OPENCL_RUNTIME=1 +endif + +ifeq ($(USE_VULKAN), 1) + APP_CPPFLAGS += -DTVM_VULKAN_RUNTIME=1 + APP_LDFLAGS += -lvulkan +endif + +ifeq ($(USE_SORT), 1) + APP_CPPFLAGS += -DUSE_SORT=1 +endif \ No newline at end of file diff --git a/apps/android_camera/app/src/main/jni/build.sh b/apps/android_camera/app/src/main/jni/build.sh new file mode 100755 index 000000000000..eb6aa2b2a748 --- /dev/null +++ b/apps/android_camera/app/src/main/jni/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -x +PATH="$PATH:/usr/local/bin" +CURR_DIR=$(cd `dirname $0`; pwd) +ROOT_DIR="$CURR_DIR/../../../../../.." +javah -o $CURR_DIR/org_apache_tvm_native_c_api.h -cp "$ROOT_DIR/jvm/core/target/*" org.apache.tvm.LibInfo || exit -1 +cp -f $ROOT_DIR/jvm/native/src/main/native/org_apache_tvm_native_c_api.cc $CURR_DIR/ || exit -1 +cp -f $ROOT_DIR/jvm/native/src/main/native/jni_helper_func.h $CURR_DIR/ || exit -1 +rm -rf $CURR_DIR/../libs +ndk-build --directory=$CURR_DIR diff --git a/apps/android_camera/app/src/main/jni/make/config.mk b/apps/android_camera/app/src/main/jni/make/config.mk new file mode 100644 index 000000000000..49e332665ad9 --- /dev/null +++ b/apps/android_camera/app/src/main/jni/make/config.mk @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#------------------------------------------------------------------------------- +# Template configuration for compiling +# +# If you want to change the configuration, please use the following +# steps. Assume you are on the root directory. First copy the this +# file so that any local changes will be ignored by git +# +# cp make/config.mk . +# +# Next modify the according entries, and then compile by +# +# ./build.sh +# +#------------------------------------------------------------------------------- +APP_ABI = all + +APP_PLATFORM = android-24 + +# whether enable OpenCL during compile +USE_OPENCL = 0 + +# whether to enable Vulkan during compile +USE_VULKAN = 0 + +# whether to enable contrib sort functions during compile +USE_SORT = 1 + +ifeq ($(USE_VULKAN), 1) + # Statically linking vulkan requires API Level 24 or higher + APP_PLATFORM = android-24 +endif + +# the additional include headers you want to add, e.g., SDK_PATH/adrenosdk/Development/Inc +ADD_C_INCLUDES = + +# the additional link libs you want to add, e.g., ANDROID_LIB_PATH/libOpenCL.so +ADD_LDLIBS = diff --git a/apps/android_camera/app/src/main/jni/tvm_runtime.h b/apps/android_camera/app/src/main/jni/tvm_runtime.h new file mode 100644 index 000000000000..a58252e780fe --- /dev/null +++ b/apps/android_camera/app/src/main/jni/tvm_runtime.h @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * \file tvm_runtime.h + * \brief Pack all tvm runtime source files + */ +#include +#include + +/* Enable custom logging - this will cause TVM to pass every log message + * through CustomLogMessage instead of LogMessage. By enabling this, we must + * implement dmlc::CustomLogMessage::Log. We use this to pass TVM log + * messages to Android logcat. + */ +#define DMLC_LOG_CUSTOMIZE 1 + +/* Ensure that fatal errors are passed to the logger before throwing + * in LogMessageFatal + */ +#define DMLC_LOG_BEFORE_THROW 1 + +#include "../src/runtime/c_runtime_api.cc" +#include "../src/runtime/cpu_device_api.cc" +#include "../src/runtime/workspace_pool.cc" +#include "../src/runtime/library_module.cc" +#include "../src/runtime/system_library.cc" +#include "../src/runtime/module.cc" +#include "../src/runtime/registry.cc" +#include "../src/runtime/file_util.cc" +#include "../src/runtime/dso_library.cc" +#include "../src/runtime/rpc/rpc_session.cc" +#include "../src/runtime/rpc/rpc_event_impl.cc" +#include "../src/runtime/rpc/rpc_server_env.cc" +#include "../src/runtime/rpc/rpc_module.cc" +#include "../src/runtime/rpc/rpc_socket_impl.cc" +#include "../src/runtime/thread_pool.cc" +#include "../src/runtime/threading_backend.cc" +#include "../src/runtime/graph/graph_runtime.cc" +#include "../src/runtime/ndarray.cc" +#include "../src/runtime/object.cc" + +#ifdef TVM_OPENCL_RUNTIME +#include "../src/runtime/opencl/opencl_device_api.cc" +#include "../src/runtime/opencl/opencl_module.cc" +#endif + +#ifdef TVM_VULKAN_RUNTIME +#include "../src/runtime/vulkan/vulkan.cc" +#endif + +#ifdef USE_SORT +#include "../src/runtime/contrib/sort/sort.cc" +#endif + + +#include + +void dmlc::CustomLogMessage::Log(const std::string& msg) { + // This is called for every message logged by TVM. + // We pass the message to logcat. + __android_log_write(ANDROID_LOG_DEBUG, "TVM_RUNTIME", msg.c_str()); +} \ No newline at end of file diff --git a/apps/android_camera/app/src/main/res/drawable/item_selector.xml b/apps/android_camera/app/src/main/res/drawable/item_selector.xml new file mode 100644 index 000000000000..bd87d8bc06b2 --- /dev/null +++ b/apps/android_camera/app/src/main/res/drawable/item_selector.xml @@ -0,0 +1,23 @@ + + + + + + + + + + diff --git a/apps/android_camera/app/src/main/res/layout/activity_main.xml b/apps/android_camera/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000000..80f9ac6902d6 --- /dev/null +++ b/apps/android_camera/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,23 @@ + + + \ No newline at end of file diff --git a/apps/android_camera/app/src/main/res/layout/fragment_camera2_basic.xml b/apps/android_camera/app/src/main/res/layout/fragment_camera2_basic.xml new file mode 100644 index 000000000000..12dc53b46db3 --- /dev/null +++ b/apps/android_camera/app/src/main/res/layout/fragment_camera2_basic.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + diff --git a/apps/android_camera/app/src/main/res/layout/listview_row.xml b/apps/android_camera/app/src/main/res/layout/listview_row.xml new file mode 100644 index 000000000000..4c233dc40379 --- /dev/null +++ b/apps/android_camera/app/src/main/res/layout/listview_row.xml @@ -0,0 +1,28 @@ + + + + + diff --git a/apps/android_camera/app/src/main/res/values/colors.xml b/apps/android_camera/app/src/main/res/values/colors.xml new file mode 100644 index 000000000000..8cb461a91903 --- /dev/null +++ b/apps/android_camera/app/src/main/res/values/colors.xml @@ -0,0 +1,20 @@ + + + + #DA6A84 + #F6F6F6 + #72BF3A + diff --git a/apps/android_camera/app/src/main/res/values/strings.xml b/apps/android_camera/app/src/main/res/values/strings.xml new file mode 100644 index 000000000000..cc69189a2ab6 --- /dev/null +++ b/apps/android_camera/app/src/main/res/values/strings.xml @@ -0,0 +1,18 @@ + + + + TVM Android Demo + diff --git a/apps/android_camera/app/src/main/res/values/styles.xml b/apps/android_camera/app/src/main/res/values/styles.xml new file mode 100644 index 000000000000..545a6eeed59e --- /dev/null +++ b/apps/android_camera/app/src/main/res/values/styles.xml @@ -0,0 +1,31 @@ + + + + + + + + + + diff --git a/apps/android_camera/app/src/main/res/xml/provider_paths.xml b/apps/android_camera/app/src/main/res/xml/provider_paths.xml new file mode 100644 index 000000000000..cef8fc075a53 --- /dev/null +++ b/apps/android_camera/app/src/main/res/xml/provider_paths.xml @@ -0,0 +1,20 @@ + + + + + diff --git a/apps/android_camera/app/src/main/rs/yuv420888.rs b/apps/android_camera/app/src/main/rs/yuv420888.rs new file mode 100644 index 000000000000..05229a261024 --- /dev/null +++ b/apps/android_camera/app/src/main/rs/yuv420888.rs @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Source: https://stackoverflow.com/questions/36212904/yuv-420-888-interpretation-on-samsung-galaxy-s7-camera2 +#pragma version(1) +#pragma rs java_package_name(org.apache.tvm.android.androidcamerademo); +#pragma rs_fp_relaxed + +int32_t width; +int32_t height; + +uint picWidth, uvPixelStride, uvRowStride ; +rs_allocation ypsIn,uIn,vIn; + +// The LaunchOptions ensure that the Kernel does not enter the padding zone of Y, so yRowStride can be ignored WITHIN the Kernel. +uchar4 __attribute__((kernel)) doConvert(uint32_t x, uint32_t y) { + + // index for accessing the uIn's and vIn's + uint uvIndex= uvPixelStride * (x/2) + uvRowStride*(y/2); + + // get the y,u,v values + uchar yps= rsGetElementAt_uchar(ypsIn, x, y); + uchar u= rsGetElementAt_uchar(uIn, uvIndex); + uchar v= rsGetElementAt_uchar(vIn, uvIndex); + + // calc argb + int4 argb; + argb.r = yps + v * 1436 / 1024 - 179; + argb.g = yps -u * 46549 / 131072 + 44 -v * 93604 / 131072 + 91; + argb.b = yps +u * 1814 / 1024 - 227; + argb.a = 255; + + uchar4 out = convert_uchar4(clamp(argb, 0, 255)); + return out; +} diff --git a/apps/android_camera/build.gradle b/apps/android_camera/build.gradle new file mode 100644 index 000000000000..a58bc631dac5 --- /dev/null +++ b/apps/android_camera/build.gradle @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + repositories { + jcenter() + maven { + url 'https://maven.google.com' + } + google() + } + dependencies { + classpath 'com.android.tools.build:gradle:3.6.1' + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + jcenter() + maven { + url 'https://maven.google.com' + } + mavenLocal() + mavenCentral() + google() + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} \ No newline at end of file diff --git a/apps/android_camera/dev_tools/gen_keystore.sh b/apps/android_camera/dev_tools/gen_keystore.sh new file mode 100755 index 000000000000..ff20597b7aab --- /dev/null +++ b/apps/android_camera/dev_tools/gen_keystore.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CURR_DIR=$(cd `dirname $0`; pwd) +keytool -genkey -keystore $CURR_DIR/tvmdemo.keystore -alias tvmdemo -keyalg RSA -validity 10000 diff --git a/apps/android_camera/dev_tools/sign_apk.sh b/apps/android_camera/dev_tools/sign_apk.sh new file mode 100755 index 000000000000..29e7777562cf --- /dev/null +++ b/apps/android_camera/dev_tools/sign_apk.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CURR_DIR=$(cd `dirname $0`; pwd) +APK_DIR=$CURR_DIR/../app/build/outputs/apk/release +UNSIGNED_APK=$APK_DIR/app-release-unsigned.apk +SIGNED_APK=$APK_DIR/tv8mdemo-release.apk +jarsigner -verbose -keystore $CURR_DIR/tv8mdemo.keystore -signedjar $SIGNED_APK $UNSIGNED_APK 'tv8mdemo' +echo $SIGNED_APK diff --git a/apps/android_camera/gradle.properties b/apps/android_camera/gradle.properties new file mode 100644 index 000000000000..f1328bf5fd43 --- /dev/null +++ b/apps/android_camera/gradle.properties @@ -0,0 +1,21 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +org.gradle.jvmargs=-Xmx4608M +android.useAndroidX=true +android.enableJetifier=true diff --git a/apps/android_camera/models/prepare_model.py b/apps/android_camera/models/prepare_model.py new file mode 100644 index 000000000000..36674d273bd1 --- /dev/null +++ b/apps/android_camera/models/prepare_model.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import pathlib +from pathlib import Path +from typing import Union +import os +from os import environ +import json + +import tvm +import tvm.relay as relay +from tvm.contrib import util, ndk, graph_runtime as runtime +from tvm.contrib.download import download_testdata, download + +target = 'llvm -target=arm64-linux-android' +target_host = None + +def del_dir(target: Union[Path, str], only_if_empty: bool = False): + target = Path(target).expanduser() + assert target.is_dir() + for p in sorted(target.glob('**/*'), reverse=True): + if not p.exists(): + continue + p.chmod(0o666) + if p.is_dir(): + p.rmdir() + else: + if only_if_empty: + raise RuntimeError(f'{p.parent} is not empty!') + p.unlink() + target.rmdir() + +def get_model(model_name, batch_size=1): + if model_name == 'resnet18_v1': + import mxnet as mx + from mxnet import gluon + from mxnet.gluon.model_zoo import vision + gluon_model = vision.get_model(model_name, pretrained=True) + img_size = 224 + data_shape = (batch_size, 3, img_size, img_size) + net, params = relay.frontend.from_mxnet(gluon_model, {"data": data_shape}) + return (net, params) + elif model_name == 'mobilenet_v2': + import keras + from keras.applications.mobilenet_v2 import MobileNetV2 + keras.backend.clear_session() # Destroys the current TF graph and creates a new one. + weights_url = ''.join(['https://github.com/JonathanCMitchell/', + 'mobilenet_v2_keras/releases/download/v1.1/', + 'mobilenet_v2_weights_tf_dim_ordering_tf_kernels_0.5_224.h5']) + weights_file = 'mobilenet_v2_weights.h5' + weights_path = download_testdata(weights_url, weights_file, module='keras') + keras_mobilenet_v2 = MobileNetV2(alpha=0.5, include_top=True, weights=None, + input_shape=(224, 224, 3), classes=1000) + keras_mobilenet_v2.load_weights(weights_path) + + img_size = 224 + data_shape = (batch_size, 3, img_size, img_size) + mod, params = relay.frontend.from_keras(keras_mobilenet_v2, {'input_1': data_shape}) + return (mod, params) + +def main(model_str, output_path): + if output_path.exists(): + del_dir(output_path) + output_path.mkdir() + output_path_str = os.fspath(output_path) + print(model_str) + print("getting model...") + net, params = get_model(model_str) + try: + os.mkdir(model_str) + except FileExistsError: + pass + print("building...") + with relay.build_config(opt_level=3): + graph, lib, params = relay.build(net, target, target_host=target_host, params=params) + print("dumping lib...") + lib.export_library(output_path_str + '/' + 'deploy_lib_cpu.so', ndk.create_shared) + print("dumping graph...") + with open(output_path_str + '/' + 'deploy_graph.json', 'w') as f: + f.write(graph) + print("dumping params...") + with open(output_path_str + '/' + 'deploy_param.params', 'wb') as f: + f.write(relay.save_param_dict(params)) + print("dumping labels...") + synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/', + '4d0b62f3d01426887599d4f7ede23ee5/raw/', + '596b27d23537e5a1b5751d2b0481ef172f58b539/', + 'imagenet1000_clsid_to_human.txt']) + synset_path = output_path_str + '/image_net_labels' + download(synset_url, output_path_str + '/image_net_labels') + with open(synset_path) as fi: + synset = eval(fi.read()) + with open(output_path_str + '/image_net_labels.json', "w") as fo: + json.dump(synset, fo, indent=4) + os.remove(synset_path) + +if __name__ == '__main__': + if environ.get('TVM_NDK_CC') is None: + raise RuntimeError("Require environment variable TVM_NDK_CC") + models_path = Path().absolute().parent.joinpath('app/src/main/assets/models/') + if not models_path.exists(): + models_path.mkdir() + models = {'mobilenet_v2': models_path.joinpath('mobilenet_v2'), + 'resnet18_v1': models_path.joinpath('resnet18_v1') + } + for model, output_path in models.items(): + main(model, output_path) diff --git a/apps/android_camera/settings.gradle b/apps/android_camera/settings.gradle new file mode 100644 index 000000000000..ee503c22bf30 --- /dev/null +++ b/apps/android_camera/settings.gradle @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +include ':app' diff --git a/tests/lint/add_asf_header.py b/tests/lint/add_asf_header.py index cdfbb9a71dd3..a44fbd3df1b5 100644 --- a/tests/lint/add_asf_header.py +++ b/tests/lint/add_asf_header.py @@ -177,7 +177,6 @@ def add_header(fname, header): print("Skip file %s ..." % fname) return - with open(fname, "w") as outfile: skipline = False ext = os.path.splitext(fname)[1][1:]