Enable Exynosbackend e2e test (#15731)

### Summary
Enable E2E test for the Exynos backend.
Modify it to run the testing using a device from devicefarm that can be
used directly in CI.

### Test plan
Each test case was verified on a real device.

cc @SS-JIA @digantdesai @kimishpatel

---------

Signed-off-by: jiseong.oh <jiseong.oh@samsung.com>
Co-authored-by: chong-chen <chong.chen@samsung.com>
Co-authored-by: Jonghun Cha <jhbb.cha@samsung.com>

Author: 3 people

.ci/scripts/setup-samsung-linux-deps.sh

@@ -8,61 +8,199 @@
 
 set -ex
 
+API_KEY=$SAMSUNG_AI_LITECORE_KEY
+if [[ -z "${API_KEY}" ]]; then
+  echo "ERROR: It didn't set up SAMSUNG_AI_LITECORE_KEY." >&2
+  exit 1
+fi
+
+OS_NAME="Ubuntu 22.04"
+LITECORE_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/ai-litecore/download"
+DEVICEFARM_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/remotelab/download"
+
+parse_url() {
+  local json="$1"
+  if command -v jq >/dev/null 2>&1; then
+    jq -r '.data // empty' <<<"$json"
+  else
+    sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$json"
+  fi
+}
 
-download_ai_lite_core() {
-  API_BASE="https://soc-developer.semiconductor.samsung.com/api/v1/resource/ai-litecore/download"
-  API_KEY=$SAMSUNG_AI_LITECORE_KEY
-
-  VERSION="0.7"
-  OS_NAME="Ubuntu 22.04"
-  OUT_FILE="/tmp/exynos-ai-litecore-v${VERSION}.tar.gz"
-  TARGET_PATH="/tmp/exynos_ai_lite_core"
-
-  mkdir -p ${TARGET_PATH}
-  # Presigned issue URL
-  JSON_RESP=$(curl -sS -G \
-    --location --fail --retry 3 \
+download_and_extract() {
+  local base_url="$1"
+  local version="$2"
+  local out_dir="$3"
+  local out_file="$4"
+
+  local resp
+  resp=$(curl -fsSL -G \
     -H "apikey: ${API_KEY}" \
-    --data-urlencode "version=${VERSION}" \
+    --data-urlencode "version=${version}" \
     --data-urlencode "os=${OS_NAME}" \
-    "${API_BASE}")
+    "${base_url}")
+
+  local download_url
+  download_url=$(parse_url "$resp")
+  if [[ -z "${download_url}" ]]; then
+    echo "ERROR: It failed to download from ${base_url} ."
+    echo "Response: $resp" >&2
+    exit 1
+  fi
+
+  curl -fsSL -L --retry 3 -o "${out_file}" "${download_url}"
+  echo "Download completed: ${out_file}"
 
-  DOWNLOAD_URL=$(echo "$JSON_RESP" | sed -n 's/.*"data":[[:space:]]*"\([^"]*\)".*/\1/p')
+  mkdir -p "${out_dir}"
+  case "${out_file##*.}" in
+  tar|tgz|gz)
+    echo "Extracting TAR.GZ..."
+    tar -C "${out_dir}" --strip-components=1 -xzvf "${out_file}"
+    ;;
 
-  if [[ -z "$DOWNLOAD_URL" ]]; then
-    echo "Failed to extract download URL"
-    echo "$JSON_RESP"
+  zip)
+    echo "Extracting ZIP..."
+    unzip -q -d "${out_dir}" "${out_file}"
+    ;;
+
+  *)
     exit 1
+    ;;
+  esac
+  echo "Extracted to: ${out_dir}"
+}
+
+download_ai_lite_core() {
+  local litecore_version="${1:-1.0}"
+  local litecore_out="/tmp/exynos-ai-litecore-v${litecore_version}.tar.gz"
+  local litecore_dir="/tmp/exynos_ai_lite_core"
+
+  download_and_extract \
+    "${LITECORE_BASE}" \
+    "${litecore_version}" \
+    "${litecore_dir}" \
+    "${litecore_out}"
+
+  export EXYNOS_AI_LITECORE_ROOT="${litecore_dir}"
+  export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:${EXYNOS_AI_LITECORE_ROOT}/lib/x86_64-linux"
+}
+
+install_devicefarm_cli() {
+  local cli_version="${1:-beta-1.0.8}"
+  local cli_out="/tmp/devicefarm-cli-v${cli_version}.zip"
+  local cli_dir="/tmp/devicefarm_cli"
+
+  download_and_extract \
+    "${DEVICEFARM_BASE}" \
+    "${cli_version}" \
+    "${cli_dir}" \
+    "${cli_out}"
+
+  export PATH="${PATH%:}:${cli_dir}"
+  chmod +x "${cli_dir}/devicefarm-cli"
+}
+
+reserve_if_needed() {
+  if ! command -v devicefarm-cli >/dev/null 2>&1; then
+    echo "[WARN] devicefarm-cli is not installed." >&2
+    return 1
   fi
 
-  # Download LiteCore
-  curl -sS -L --fail --retry 3 \
-    --output "$OUT_FILE" \
-    "$DOWNLOAD_URL"
+  local raw_info info_lines
+  raw_info="$(devicefarm-cli -I)"
+
+  info_lines="$(printf '%s\n' "$raw_info" | grep -v '^\\[INFO\\]')"
 
-  echo "Download done: $OUT_FILE"
+  local found_count
 
+  found_count=$(printf '%s\n' "$info_lines" \
+    | grep -Eo 'Found available reservations *: *[0-9]+' \
+    | grep -Eo '[0-9]+')
+  [[ -z "$found_count" ]] && found_count=0
 
-  tar -C "${TARGET_PATH}" --strip-components=1 -xzvf "${OUT_FILE}"
+  echo "[INFO] Current Reserved Count: $found_count"
+
+  local THRESHOLD_SECONDS=12600
+  local any_below_threshold=0
+
+  if (( found_count > 0 )); then
+    local table_body
+    table_body=$(printf '%s\n' "$info_lines" | sed -n '2,$p')
+
+    while IFS= read -r line; do
+      if [[ "$line" =~ ^[0-9]+[[:space:]]+([0-9]{1,2}:[0-9]{2}:[0-9]{2}) ]]; then
+        local time_str="${BASH_REMATCH[1]}"
+        IFS=: read -r hh mm ss <<<"$time_str"
+        (( seconds = 10#$hh * 3600 + 10#$mm * 60 + 10#$ss ))
+        if (( seconds <= THRESHOLD_SECONDS )); then
+          any_below_threshold=1
+          break
+        fi
+      fi
+    done <<<"$table_body"
+  else
+    any_below_threshold=1
+  fi
 
-  export EXYNOS_AI_LITECORE_ROOT=${TARGET_PATH}
-  export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${EXYNOS_AI_LITECORE_ROOT}/lib/x86_64-linux
+  if (( any_below_threshold )); then
+    echo "[INFO] Reserving now."
+    devicefarm-cli -R
+  else
+    echo "[INFO] Don't need to be reserved."
+  fi
+
+  local info_after reservation_id max_seconds=0 max_id
+
+  info_after="$(devicefarm-cli -I)"
+
+  local body_after
+  body_after=$(printf '%s\n' "$info_after" | grep -v '^\\[INFO\\]' | sed -n '2,$p')
+
+  while IFS= read -r line; do
+    if [[ "$line" =~ ^[0-9]+[[:space:]]+([0-9]{1,2}:[0-9]{2}:[0-9]{2})[[:space:]].*([0-9a-f-]{36})$ ]]; then
+      local time_str="${BASH_REMATCH[1]}"
+      local id="${BASH_REMATCH[2]}"
+      IFS=: read -r hh mm ss <<<"$time_str"
+      (( seconds = 10#$hh * 3600 + 10#$mm * 60 + 10#$ss ))
+      if (( seconds > max_seconds )); then
+        max_seconds=$seconds
+        max_id=$id
+      fi
+    fi
+  done <<<"$body_after"
+
+  reservation_id=$max_id
+
+  if [[ -n "$reservation_id" ]]; then
+    devicefarm-cli -C "$reservation_id"
+    devicefarm-cli -E "ls /"
+  else
+    echo "[WARN] There is no available devices."
+  fi
 }
 
 install_enn_backend() {
-  NDK_INSTALLATION_DIR=/opt/ndk
-  rm -rf "${NDK_INSTALLATION_DIR}" && sudo mkdir -p "${NDK_INSTALLATION_DIR}"
-  ANDROID_NDK_VERSION=r28c
+  local ndk_dir="/opt/ndk"
+  local ndk_version="r28c"
+
+  if [[ ! -d "${ndk_dir}" ]]; then
+    sudo mkdir -p "${ndk_dir}"
+    sudo chown "$(whoami)":"$(whoami)" "${ndk_dir}"
+  fi
+
+  export ANDROID_NDK_ROOT="${ndk_dir}"
+  echo "NDK will be installed/used at: ${ANDROID_NDK_ROOT}"
 
-  # build Exynos backend
-  export ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT:-/opt/ndk}
   bash backends/samsung/build.sh --build all
-  # set env variable
-  export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
-  export PYTHONPATH=${PYTHONPATH:-}:${EXECUTORCH_ROOT}/..
+
+  export EXECUTORCH_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+  export PYTHONPATH="${PYTHONPATH:-}:${EXECUTORCH_ROOT}/.."
 }
 
-AI_LITE_CORE_VERSION=0.7.0
+litecore_ver="1.0"
+devicefarm_ver="beta-1.0.8"
 
-download_ai_lite_core ${AI_LITE_CORE_VERSION}
+download_ai_lite_core ${litecore_ver}
+install_devicefarm_cli "${devicefarm_ver}"
 install_enn_backend
+reserve_if_needed

.github/workflows/pull.yml

@@ -915,9 +915,8 @@ jobs:
         PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh cifar10
         PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh mobilenetv2
 
-
-  test-samsung-models-linux:
-    name: test-samsung-models-linux
+  test-samsung-quantmodels-linux:
+    name: test-samsung-quantmodels-linux
     # Skip this job if the pull request is from a fork (secrets are not available)
     if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
@@ -931,7 +930,7 @@ jobs:
       docker-image: ci-image:executorch-ubuntu-22.04-clang12-android
       submodules: 'recursive'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout: 90
+      timeout: 180
       script: |
         set -ex
 
@@ -946,21 +945,44 @@ jobs:
         export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY
         source .ci/scripts/setup-samsung-linux-deps.sh
 
-        # Test models serially
-        models="mv2 ic3 resnet18 resnet50 mv3 ic4 dl3 edsr vit w2l"
-        for model in $models; do
-          python -m executorch.examples.samsung.aot_compiler --model_name=$model -c E9955
-        done
-
         # Test quant models
         model_scripts="deeplab_v3 edsr inception_v3 inception_v4 mobilenet_v2 mobilenet_v3 resnet18 resnet50 vit wav2letter"
         for m_script in $model_scripts; do
           python -m executorch.examples.samsung.scripts.${m_script} -c e9955 -p A8W8
         done
 
-        # Test ops
-        python -m unittest discover -s backends/samsung/test/ops -p "test_*.py"
+  test-samsung-models-linux:
+    name: test-samsung-models-linux
+    # Skip this job if the pull request is from a fork (secrets are not available)
+    if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name != 'pull_request'
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    secrets: inherit
+    with:
+      secrets-env: SAMSUNG_AI_LITECORE_KEY
+      runner: linux.2xlarge
+      docker-image: ci-image:executorch-ubuntu-22.04-clang12-android
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 360
+      script: |
+        set -ex
+
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Setup python
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
+
+        # Setup Samsung SDK (AI Lite Core) and install enn backend
+        export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY
+        source .ci/scripts/setup-samsung-linux-deps.sh
 
+        # Test models
+        python -m unittest discover -s backends/samsung/test/models -p "test_*.py"
 
   test-vulkan-models-linux:
     name: test-vulkan-models-linux

backends/samsung/test/models/test_deeplab_v3.py

@@ -0,0 +1,28 @@
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file in the root
+# directory of this source tree for more details.
+import unittest
+
+from executorch.backends.samsung.serialization.compile_options import (
+    gen_samsung_backend_compile_spec,
+)
+from executorch.backends.samsung.test.tester import SamsungTester
+from executorch.examples.models.deeplab_v3 import DeepLabV3ResNet50Model
+
+
+class TestMilestoneDeepLabV3(unittest.TestCase):
+    def test_dl3_fp16(self):
+        model = DeepLabV3ResNet50Model().get_eager_model()
+        example_input = DeepLabV3ResNet50Model().get_example_inputs()
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
+        (
+            tester.export()
+            .to_edge_transform_and_lower()
+            .to_executorch()
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.009)
+        )

backends/samsung/test/models/test_edsr.py

@@ -0,0 +1,30 @@
+# Copyright (c) Samsung Electronics Co. LTD
+# All rights reserved
+#
+# Licensed under the BSD License (the "License"); you may not use this file
+# except in compliance with the License. See the license file in the root
+# directory of this source tree for more details.
+
+
+import unittest
+
+from executorch.backends.samsung.serialization.compile_options import (
+    gen_samsung_backend_compile_spec,
+)
+from executorch.backends.samsung.test.tester import SamsungTester
+from executorch.examples.models.edsr import EdsrModel
+
+
+class TestMilestoneEdsr(unittest.TestCase):
+    def test_edsr_fp16(self):
+        model = EdsrModel().get_eager_model()
+        example_input = EdsrModel().get_example_inputs()
+        tester = SamsungTester(
+            model, example_input, [gen_samsung_backend_compile_spec("E9955")]
+        )
+        (
+            tester.export()
+            .to_edge_transform_and_lower()
+            .to_executorch()
+            .run_method_and_compare_outputs(inputs=example_input, atol=0.02)
+        )