Added format tools to docker image

Added buildifier and google-java-format to docker image
Reved the docker image version to 1.0
Added script (docker_run.sh) to Run formatters with Docker image
Added script to format all code
Update to java 11 to support google-java-format

Makefile

@@ -17,9 +17,6 @@
 #SAMSUNG_BACKEND=--//android/java/org/mlperf/inference:with_samsung="1"
 SAMSUNG_BACKEND=
 
-#MEDIATEK_BACKEND=--//android/java/org/mlperf/inference:with_mediatek="1"
-MEDIATEK_BACKEND=
-
 .PHONY: docker_image app bazel-version rundocker clean
 
 all: app
@@ -28,12 +25,12 @@ include mobile_back_tflite/tflite_backend.mk
 include mobile_back_qti/qti_backend.mk
 
 
-output/mlperf_mobile_docker_0_2.stamp:
+output/mlperf_mobile_docker_1_0.stamp: android/docker/mlperf_mobile/Dockerfile
 	@mkdir -p output/docker
-	@docker image build -t mlcommons/mlperf_mobile:0.2 android/docker/mlperf_mobile
+	@docker image build -t mlcommons/mlperf_mobile:1.0 android/docker/mlperf_mobile
 	@touch $@
 
-docker_image: output/mlperf_mobile_docker_0_2.stamp
+docker_image: output/mlperf_mobile_docker_1_0.stamp
 
 COMMON_DOCKER_FLAGS1= \
                 -e USER=mlperf \
@@ -43,7 +40,7 @@ COMMON_DOCKER_FLAGS1= \
                 ${QTI_VOLUMES} \
 		-w /home/mlperf/mobile_app \
                 -u `id -u`:`id -g` \
-		mlcommons/mlperf_mobile:0.2 bazel-3.7.2
+		mlcommons/mlperf_mobile:1.0 bazel-3.7.2
 
 COMMON_DOCKER_FLAGS2= \
                 ${PROXY_WORKAROUND2} \
@@ -61,7 +58,7 @@ NATIVE_DOCKER_FLAGS= \
                 ${COMMON_DOCKER_FLAGS1} --bazelrc=/dev/null \
                 ${COMMON_DOCKER_FLAGS2}
 
-proto_test: output/mlperf_mobile_docker_0_2.stamp
+proto_test: output/mlperf_mobile_docker_1_0.stamp
 	@echo "Building proto_test"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -70,7 +67,7 @@ proto_test: output/mlperf_mobile_docker_0_2.stamp
 	@cp output/`readlink bazel-bin`/android/cpp/proto/proto_test output/proto_test
 	@chmod 777 output/proto_test
 
-main: output/mlperf_mobile_docker_0_2.stamp ${QTI_DEPS}
+main: output/mlperf_mobile_docker_1_0.stamp ${QTI_DEPS}
 	@echo "Building main"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -82,7 +79,7 @@ main: output/mlperf_mobile_docker_0_2.stamp ${QTI_DEPS}
 	@${QTI_LIB_COPY}
 	@chmod 777 output/binary/main output/binary/libtflitebackend.so
 
-libtflite: output/mlperf_mobile_docker_0_2.stamp
+libtflite: output/mlperf_mobile_docker_1_0.stamp
 	@echo "Building libtflite"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -93,7 +90,7 @@ libtflite: output/mlperf_mobile_docker_0_2.stamp
 	@chmod 777 output/binary/libtflitebackend.so
 
 
-app: output/mlperf_mobile_docker_0_2.stamp ${QTI_DEPS}
+app: output/mlperf_mobile_docker_1_0.stamp ${QTI_DEPS}
 	@echo "Building mlperf_app.apk"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -104,7 +101,7 @@ app: output/mlperf_mobile_docker_0_2.stamp ${QTI_DEPS}
 	@cp output/`readlink bazel-bin`/android/java/org/mlperf/inference/mlperf_app.apk output/mlperf_app.apk
 	@chmod 777 output/mlperf_app.apk
 
-app_x86_64: output/mlperf_mobile_docker_0_2.stamp
+app_x86_64: output/mlperf_mobile_docker_1_0.stamp
 	@echo "Building mlperf_app.apk"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -115,7 +112,7 @@ app_x86_64: output/mlperf_mobile_docker_0_2.stamp
 	@cp output/`readlink bazel-bin`/android/java/org/mlperf/inference/mlperf_app.apk output/mlperf_app_x86_64.apk
 	@chmod 777 output/mlperf_app.apk
 
-test_app: output/mlperf_mobile_docker_0_2.stamp
+test_app: output/mlperf_mobile_docker_1_0.stamp
 	@echo "Building mlperf_app.apk"
 	@mkdir -p output/home/mlperf/cache && chmod 777 output/home/mlperf/cache
 	@docker run \
@@ -126,22 +123,22 @@ test_app: output/mlperf_mobile_docker_0_2.stamp
 	@cp output/`readlink bazel-bin`/android/androidTest/mlperf_test_app.apk output/mlperf_test_app.apk
 	@chmod 777 output/mlperf_test_app.apk
 
-rundocker: output/mlperf_mobile_docker_0_2.stamp
+rundocker: output/mlperf_mobile_docker_1_0.stamp
 	@docker run -it \
                 -e USER=mlperf \
 		-v $(CURDIR):/home/mlperf/mobile_app \
 		-v $(CURDIR)/output/home/mlperf/cache:/home/mlperf/cache \
 		-w /home/mlperf/mobile_app \
                 -u `id -u`:`id -g` \
-		mlcommons/mlperf_mobile:0.2
+		mlcommons/mlperf_mobile:1.0
 
-rundocker_root: output/mlperf_mobile_docker_0_2.stamp
+rundocker_root: output/mlperf_mobile_docker_1_0.stamp
 	@docker run -it \
                 -e USER=mlperf \
 		-v $(CURDIR):/home/mlperf/mobile_app \
 		-v $(CURDIR)/output/home/mlperf/cache:/home/mlperf/cache \
 		-w /home/mlperf/mobile_app \
-		mlcommons/mlperf_mobile:0.2
+		mlcommons/mlperf_mobile:1.0
 
 clean:
 	@([ -d output/home/mlperf/cache ] && chmod -R +w output/home/mlperf/cache) || true

android/docker/mlperf_mobile/Dockerfile

@@ -1,3 +1,4 @@
+# Docker image name: mlcommons/mlperf_mobile:1.0
 FROM ubuntu:bionic
 LABEL maintainer="quic_mcharleb@quicinc.com"
 
@@ -10,7 +11,7 @@ RUN apt-get update && apt-get upgrade -y && apt-get autoremove -y && \
     apt-get install -y --no-install-recommends \
        python3 python3-pip libpython3.6-dev python3.6-venv libgl1-mesa-glx libglib2.0 cython3 gcc make curl unzip libc++1-8 \
        git locales openssh-client ca-certificates tar gzip parallel \
-       zip bzip2 gnupg wget python3-six python3-pip libncurses5 openjdk-8-jdk-headless clang-format-10
+       zip bzip2 gnupg wget python3-six python3-pip libncurses5 openjdk-11-jdk-headless clang-format-10 golang-1.13-go
 
 RUN pip3 install pip==19.3.1 setuptools==31.0.1
 RUN pip3 install tensorflow-cpu==1.15
@@ -45,15 +46,16 @@ ARG android_home=/opt/android/sdk
 
 # Install Android SDK and NDK
 RUN mkdir -p ${android_home} && \
-    curl --silent --show-error --location --fail --retry 3 --output /tmp/sdk_tools.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip && \
-    unzip -q /tmp/sdk_tools.zip -d ${android_home} && \
+    wget -O /tmp/sdk_tools.zip https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip && \
+    unzip -q /tmp/sdk_tools.zip -d ${android_home}/cmdline-tools/ && \
+    mv ${android_home}/cmdline-tools/cmdline-tools ${android_home}/cmdline-tools/tools && \
     rm /tmp/sdk_tools.zip
 
 # Set environment variables
 ENV ANDROID_HOME ${android_home}
 ENV ANDROID_NDK_HOME ${android_ndk_home}
 ENV ADB_INSTALL_TIMEOUT 120
-ENV PATH=${ANDROID_HOME}/emulator:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools:${PATH}
+ENV PATH=${ANDROID_HOME}/emulator:${ANDROID_HOME}/cmdline-tools/tools/bin:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/build-tools/29.0.3:${PATH}
 
 RUN mkdir ~/.android && echo '### User Sources for Android SDK Manager' > ~/.android/repositories.cfg
 
@@ -81,6 +83,13 @@ RUN cd /tmp &&  wget -nv https://dl.google.com/android/repository/${ndk_version}
 
 ENV ANDROID_NDK_HOME ${android_ndk_home}
 
+# Add Java format checker and buildifier
+ENV GOBIN /opt/formatters/bin
+RUN mkdir -p ${GOBIN} && curl --output `dirname ${GOBIN}`/google-java-format-1.9-all-deps.jar -L https://github.com/google/google-java-format/releases/download/v1.9/google-java-format-1.9-all-deps.jar
+RUN /usr/lib/go-1.13/bin/go get github.com/bazelbuild/buildtools/buildifier
+ENV PATH=${PATH}:${GOBIN} 
+
 RUN mkdir -p /home/mlperf && chmod 777 /home/mlperf
 ENV HOME /home/mlperf
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 

android/format_all_code.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2020-2021 The MLPerf Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+
+# This script formats all staged C++ and build files.
+
+# Formatting cpp files using clang-format.
+cpp_files=$(find android -name "*.h" && find android -name "*.cc")
+if [ "$cpp_files" ]; then
+  clang-format-10 -i -style=google $cpp_files
+fi
+
+
+# Formatting build files using buildifier.
+buildifier_is_present=$(which buildifier)
+if [ ! -x "$buildifier_is_present" ] ; then
+  echo "*"
+  echo "* Bazel config files can't be formated because 'buildifier' is not in \$PATH"
+  echo "*\n"
+else
+  build_files=$(echo WORKSPACE && find android -name BUILD && find android -name BUILD.bazel && find android -name "*.bzl")
+  if [ "$build_files" ]; then
+    buildifier -v $build_files
+  fi
+fi 
+
+# Formatting Java files.
+java_is_present=$(which java)
+if [ ! -x "$java_is_present" ] ; then
+  echo "*"
+  echo "* Java files can't be formated because 'java' is not in \$PATH"
+  echo "*\n"
+else
+  java_files=$(find android -name "*.java")
+  if [ "$java_files" ]; then
+    java -jar /opt/formatters/google-java-format-1.9-all-deps.jar --replace  $java_files
+  fi
+fi

android/format_code.sh

@@ -38,7 +38,7 @@ if [ "$cpp_files" ]; then
     -it \
     -v `pwd`:/home/mlperf/mobile_app \
     -w /home/mlperf/mobile_app \
-    mlcommons/mlperf_mobile:0.2 clang-format-10 -i -style=google $cpp_files
+    mlcommons/mlperf_mobile:1.0 clang-format-10 -i -style=google $cpp_files
 fi
 
 
@@ -64,6 +64,6 @@ if [ ! -x "$java_is_present" ] ; then
 else
   java_files=$(ls_staged_files "\.java")
   if [ "$java_files" ]; then
-    java -jar formatters/google-java-format/google-java-format-1.9-all-deps.jar --replace  $java_files
+    java -jar /opt/formatters/google-java-format-1.9-all-deps.jar --replace  $java_files
   fi
 fi

android/formatters/README.md

@@ -12,16 +12,18 @@ You should use script `format_code.sh` in project root directory for usual devel
 
 Example:
 ```
-$ sh format_code.sh
+$ sh android/formatters/docker_run.sh android/format_code.sh
 ```
 
 
 ## CI
 The scripts in the current directory are intended to be used primarily in the CI process. "CI" should be used as the only parameter in that case.
 
-Example for bazel:
+To run the scripts in the docker image use:
 ```
-$ sh formatters/run-bazel-format.sh CI
+$ sh android/formatters/docker_run.sh android/formatters/run-bazel-format.sh CI
+$ sh android/formatters/docker_run.sh android/formatters/run-clang-format.sh CI
+$ sh android/formatters/docker_run.sh android/formatters/run-google-java-format.sh CI
 ```
 
 The scripts format all files found in scan paths. Scan paths are different for the scripts

android/formatters/docker_run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+docker run -it \
+                -e USER=mlperf \
+                -v `pwd`:/home/mlperf/mobile_app \
+                -w /home/mlperf/mobile_app \
+                -u `id -u`:`id -g` \
+                mlcommons/mlperf_mobile:1.0 $*
+

android/formatters/run-bazel-format.sh

@@ -16,9 +16,9 @@
 ##########################################################################
 
 buildifier WORKSPACE
-find . -name BUILD | xargs buildifier
-find . -name BUILD.bazel | xargs buildifier
-find . -iname "*.bzl" | xargs buildifier
+find android -name BUILD | xargs buildifier
+find android -name BUILD.bazel | xargs buildifier
+find android -iname "*.bzl" | xargs buildifier
 
 if [ "$1" = "CI" ]; then
     git diff >bazel-codeformat-${GIT_COMMIT}.patch

android/formatters/run-clang-format.sh

@@ -20,17 +20,17 @@ CPP_DIRS=('android/cpp' 'mobile_back_qti' 'mobile_back_tflite')
 for directory in "${CPP_DIRS[@]}"
 do
     find $directory -iname "*.h" | \
-        xargs docker run --rm -it -v `pwd`:/home/mlperf/mobile_app -w /home/mlperf/mobile_app mlcommons/mlperf_mobile:0.2 clang-format-10 -i -style=google
+        xrags clang-format-10 -i -style=google
 
     find $directory -iname "*.cc" | \
-        xargs docker run --rm -it -v `pwd`:/home/mlperf/mobile_app -w /home/mlperf/mobile_app mlcommons/mlperf_mobile:0.2 clang-format-10 -i -style=google
+        xargs clang-format-10 -i -style=google
 
     find $directory -iname "*.cpp" | \
-        xargs docker run --rm -it -v `pwd`:/home/mlperf/mobile_app -w /home/mlperf/mobile_app mlcommons/mlperf_mobile:0.2 clang-format-10 -i -style=google
+        xargs clang-format-10 -i -style=google
 done
 
 find java/org/mlperf/inference/jni -iname "*.cc" | \
-    xargs docker run --rm -it -v `pwd`:/home/mlperf/mobile_app -w /home/mlperf/mobile_app mlcommons/mlperf_mobile:0.2 clang-format-10 -i -style=google
+    xargs clang-format-10 -i -style=google
 
 if [ "$1" = "CI" ]; then
     git diff >cpp-codeformat-${GIT_COMMIT}.patch

android/formatters/run-format-code.sh

@@ -33,7 +33,7 @@ ls_staged_files () {
 cpp_files=$(ls_staged_files "\.h|\.cc|\.cpp")
 if [ "$cpp_files" ]; then
   echo "Updated/new C++ files: $cpp_files"
-  clang-format -i  -style=google $cpp_files
+  clang-format-10 -i  -style=google $cpp_files
 else
   echo "There is no updated/new C++ files"
 fi
@@ -51,7 +51,7 @@ fi
 java_files=$(ls_staged_files "\.java")
 if [ "$java_files" ]; then
   echo "Updated/new Java files: $java_files"
-  java -jar formatters/google-java-format/google-java-format-1.9-all-deps.jar --replace  $java_files
+  java -jar /opt/formatters/google-java-format-1.9-all-deps.jar --replace  $java_files
 else
   echo "There is no updated/new Java files"
 fi

android/formatters/run-google-java-format.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 ##########################################################################
 
-FORMATTER=formatters/google-java-format/google-java-format-1.9-all-deps.jar
+FORMATTER=/opt/formatters/google-java-format-1.9-all-deps.jar
 
 find java -iname "*.java" | \
     xargs java -jar $FORMATTER --set-exit-if-changed --replace