Skip to content

Commit a6922e4

Browse files
swolchokswolchok
committed
rebase
[ghstack-poisoned]
2 parents 04b5196 + 2d4533a commit a6922e4

File tree

811 files changed

+42824
-15552
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

811 files changed

+42824
-15552
lines changed

‎.ci/docker/ci_commit_pins/optimum-executorch.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
a3942627f5ac048e06b4b1d703b0a6a53bf6da5b
1+
36e3dd54effb3f6d13d792029609292fdd5502bb

‎.ci/docker/ci_commit_pins/pytorch.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2dccff7dcf56b0d168ebfd7ca08bdeca37273c56
1+
6fc0ad22f0a07b6f38d138861c56a765d5a9bb02

‎.ci/docker/common/install_zephyr.sh‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88

99
set -ex
1010

11+
# shellcheck source=/dev/null
12+
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
13+
1114
# Double check if the NDK version is set
1215
[ -n "${ZEPHYR_SDK}" ]
1316

@@ -77,16 +80,8 @@ install_prerequiresites() {
7780
chmod +x kitware-archive.sh && \
7881
./kitware-archive.sh && \
7982
rm -f kitware-archive.sh
80-
useradd -d /home/zephyruser -m -s /bin/bash zephyruser
81-
}
82-
83-
install_sdk() {
84-
wget https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.0/zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
85-
tar -xf zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
86-
rm -f zephyr-sdk-0.16.0_linux-x86_64.tar.xz && \
87-
cd zephyr-sdk-0.16.0/ && \
88-
./setup.sh -c -t arm-zephyr-eabi
83+
pip_install --no-cache-dir west
84+
pip_install pyelftools
8985
}
9086

9187
install_prerequiresites
92-
install_sdk

‎.ci/docker/ubuntu/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,9 @@ ARG ARM_SDK
8686

8787
ARG ZEPHYR_SDK
8888
COPY ./common/install_zephyr.sh install_zephyr.sh
89+
COPY ./common/utils.sh utils.sh
8990
RUN if [ -n "${ZEPHYR_SDK}" ]; then bash ./install_zephyr.sh; fi
90-
RUN rm install_zephyr.sh
91+
RUN rm install_zephyr.sh utils.sh
9192

9293
ARG QNN_SDK
9394

‎.ci/scripts/build-mediatek-sdk.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ build_neuron_backend() {
1414
export NEURON_BUFFER_ALLOCATOR_LIB=${MEDIATEK_SDK_ROOT}/libneuron_buffer_allocator.so
1515
export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
1616

17-
1817
cd ${EXECUTORCH_ROOT}
1918
./backends/mediatek/scripts/mtk_build.sh
19+
./examples/mediatek/mtk_build_examples.sh
2020
}
2121

2222
build_neuron_backend

‎.ci/scripts/gather_benchmark_configs.py‎

Lines changed: 98 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,65 @@
1616
from examples.models import MODEL_NAME_TO_MODEL
1717

1818

19-
# Device pools for AWS Device Farm
19+
DEVICE_POOLS_REGEX = re.compile(r"(?P<device_name>[^\+]+)\+(?P<variant>[^\+]+)")
20+
# Device pools for AWS Device Farm. Initially, I choose to distribute models to these pool
21+
# round-robin for simplicity. For public pool, only one per device type is needed because
22+
# AWS will scale the number of devices there for us. However, for private pool, we need to
23+
# manually maintain multiple pools of the same device to evenly distribute models there.
24+
# The pool ARNs are extracted from the output of the following command:
25+
# aws devicefarm list-device-pools \
26+
# --arn arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 \
27+
# --region us-west-2
2028
DEVICE_POOLS = {
21-
"apple_iphone_15": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3b5acd2e-92e2-4778-b651-7726bafe129d",
22-
"apple_iphone_15+ios_18": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/12c8b15c-8d03-4e07-950d-0a627e7595b4",
23-
"samsung_galaxy_s22": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa",
24-
"samsung_galaxy_s22_private": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/ea6b049d-1508-4233-9a56-5d9eacbe1078",
25-
"samsung_galaxy_s24": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db",
26-
"google_pixel_8_pro": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a",
27-
"google_pixel_3_private_rooted": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d",
28-
"apple_iphone_15_private": "arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/55929353-2f28-4ee5-bdff-d1a95f58cb28",
29+
"apple_iphone_15": {
30+
"public": [
31+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3b5acd2e-92e2-4778-b651-7726bafe129d",
32+
],
33+
"ios_18_public": [
34+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/12c8b15c-8d03-4e07-950d-0a627e7595b4",
35+
],
36+
"private": [
37+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/55929353-2f28-4ee5-bdff-d1a95f58cb28",
38+
],
39+
"plus_private": [
40+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/767bfb3e-a00e-4d92-998b-4eafdcf7213b",
41+
],
42+
"pro_private": [
43+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/1394f34c-2981-4c55-aaa2-246871ac713b",
44+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/099e8def-4609-4383-8787-76b88e500c1d",
45+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d6707270-b009-479e-a83a-7bdb255f9de5",
46+
],
47+
},
48+
"samsung_galaxy_s22": {
49+
"public": [
50+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e59f866a-30aa-4aa1-87b7-4510e5820dfa",
51+
],
52+
"private": [
53+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/ea6b049d-1508-4233-9a56-5d9eacbe1078",
54+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/1fa924a1-5aff-475b-8f4d-f7c6d8de4fe9",
55+
],
56+
"ultra_private": [
57+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/5f79d72e-e229-4f9c-962f-5d37196fcfe7",
58+
],
59+
},
60+
"samsung_galaxy_s24": {
61+
"public": [
62+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98f8788c-2e25-4a3c-8bb2-0d1e8897c0db",
63+
],
64+
"ultra_private": [
65+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/5f79d72e-e229-4f9c-962f-5d37196fcfe7",
66+
],
67+
},
68+
"google_pixel_8": {
69+
"pro_public": [
70+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/d65096ab-900b-4521-be8b-a3619b69236a",
71+
],
72+
},
73+
"google_pixel_3": {
74+
"rooted_private": [
75+
"arn:aws:devicefarm:us-west-2:308535385114:devicepool:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/98d23ca8-ea9e-4fb7-b725-d402017b198d",
76+
],
77+
},
2978
}
3079

3180
# Predefined benchmark configurations
@@ -318,25 +367,56 @@ def get_benchmark_configs() -> Dict[str, Dict]: # noqa: C901
318367

319368
# Add configurations for each valid device
320369
for device in devices:
370+
# Parse the device name
371+
m = re.match(DEVICE_POOLS_REGEX, device)
372+
if not m:
373+
logging.warning(
374+
f"Invalid device name: {device} is not in DEVICE_NAME+VARIANT format. Skipping."
375+
)
376+
continue
377+
378+
device_name = m.group("device_name")
379+
variant = m.group("variant")
380+
381+
if device_name not in DEVICE_POOLS:
382+
logging.warning(f"Unsupported device '{device}'. Skipping.")
383+
continue
384+
385+
if variant not in DEVICE_POOLS[device_name]:
386+
logging.warning(
387+
f"Unsupported {device}'s variant '{variant}'. Skipping."
388+
)
389+
continue
390+
391+
device_pool_count = len(DEVICE_POOLS[device_name][variant])
392+
if not device_pool_count:
393+
logging.warning(
394+
f"No device pool defined for {device}'s variant '{variant}'. Skipping."
395+
)
396+
continue
397+
398+
device_pool_index = 0
321399
for config in configs:
322-
if config == "llama3_coreml_ane" and not device.endswith("+ios_18"):
323-
device = f"{device}+ios_18"
400+
if config == "llama3_coreml_ane" and "ios_18" not in variant:
401+
variant = "ios_18_public"
324402
logging.info(
325-
f"Benchmark config '{config}' only works on iOS 18+, auto-upgraded device pool to '{device}'"
403+
f"Benchmark config '{config}' only works on iOS 18+, auto-upgraded device variant to '{variant}'"
326404
)
327405

328-
if device not in DEVICE_POOLS:
329-
logging.warning(f"Unsupported device '{device}'. Skipping.")
330-
continue
331-
332406
record = {
333407
"model": model_name,
334408
"config": config,
335-
"device_name": device,
336-
"device_arn": DEVICE_POOLS[device],
409+
"device_name": device_name,
410+
"variant": variant,
411+
"device_arn": DEVICE_POOLS[device_name][variant][
412+
device_pool_index % device_pool_count
413+
],
337414
}
338415
benchmark_configs["include"].append(record)
339416

417+
# Distribute configs to pools of the same device round-robin
418+
device_pool_index += 1
419+
340420
set_output("benchmark_configs", json.dumps(benchmark_configs))
341421

342422

‎.ci/scripts/setup-emscripten.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@
77

88
set -ex
99

10+
# need version >= 17
11+
install_node() {
12+
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh | bash
13+
source "$HOME/.nvm/nvm.sh"
14+
nvm install 22
15+
}
16+
1017
install_emscripten() {
1118
git clone https://github.com/emscripten-core/emsdk.git
1219
pushd emsdk || return
@@ -16,4 +23,5 @@ install_emscripten() {
1623
popd || return
1724
}
1825

26+
install_node
1927
install_emscripten

0 commit comments

Comments
 (0)