From 1fd7ba11d3907d2ea093fa018fb0e97dd02f4342 Mon Sep 17 00:00:00 2001
From: Scott Todd <scott.todd0@gmail.com>
Date: Thu, 26 Sep 2024 15:34:35 -0700
Subject: [PATCH 1/4] Start on script for building linux packages under
 manylinux.

---
 shortfin/build_tools/build_linux_package.sh | 118 ++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100755 shortfin/build_tools/build_linux_package.sh

diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh
new file mode 100755
index 000000000..2b7dbc398
--- /dev/null
+++ b/shortfin/build_tools/build_linux_package.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+# Copyright 2024 Advanced Micro Devices, Inc.
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# build_linux_package.sh
+# One stop build of shortfin Python package for Linux. The Linux build is
+# complicated because it has to be done via a manylinux docker container.
+#
+# Usage:
+# Build everything (all python versions):
+#   sudo ./build_tools/build_linux_package.sh
+#
+# Build specific Python versions to custom directory:
+#   OVERRIDE_PYTHON_VERSIONS="cp312-cp312 cp313-313" \
+#   OUTPUT_DIR="/tmp/wheelhouse" \
+#   sudo ./build_tools/build_linux_package.sh
+#
+# Valid Python versions match a subdirectory under /opt/python in the docker
+# image. Typically:
+#   cp312-cp312 cp313-cp313
+#
+# Note that this script is meant to be run on CI and it will pollute both the
+# output directory and in-tree build/ directories with docker created, root
+# owned builds. Sorry - there is no good way around it.
+#
+# It can be run on a workstation but recommend using a git worktree dedicated
+# to packaging to avoid stomping on development artifacts.
+set -xeu -o errtrace
+
+THIS_DIR="$(cd $(dirname $0) && pwd)"
+REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)"
+SCRIPT_NAME="$(basename $0)"
+ARCH="$(uname -m)"
+
+# TODO: update to manylinux_2_28, 2014 is being used by torch-mlir
+# MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-"ghcr.io/nod-ai/manylinux_x86_64:main@sha256:facedb71df670016e74e646d71e869e6fff70d4cdbaa6634d4d0a10d6e174399" }')}"
+MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux2014_${ARCH}@sha256:2ace4b4f06a726d270821cb4993caeed3aacdaa54801e9f13be8fbe96077f4dc}"
+PYTHON_VERSIONS="${OVERRIDE_PYTHON_VERSIONS:-cp312-cp312}"
+OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}"
+
+function run_on_host() {
+  echo "Running on host"
+  echo "Launching docker image ${MANYLINUX_DOCKER_IMAGE}"
+
+  # Canonicalize paths.
+  mkdir -p "${OUTPUT_DIR}"
+  OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)"
+  echo "Outputting to ${OUTPUT_DIR}"
+  mkdir -p "${OUTPUT_DIR}"
+  docker run --rm \
+    -v "${REPO_ROOT}:${REPO_ROOT}" \
+    -v "${OUTPUT_DIR}:${OUTPUT_DIR}" \
+    -e __MANYLINUX_BUILD_WHEELS_IN_DOCKER=1 \
+    -e "OVERRIDE_PYTHON_VERSIONS=${PYTHON_VERSIONS}" \
+    -e "OUTPUT_DIR=${OUTPUT_DIR}" \
+    "${MANYLINUX_DOCKER_IMAGE}" \
+    -- ${THIS_DIR}/${SCRIPT_NAME}
+
+  echo "******************** BUILD COMPLETE ********************"
+  echo "Generated binaries:"
+  ls -l "${OUTPUT_DIR}"
+}
+
+function run_in_docker() {
+  echo "Running in docker"
+  echo "Marking git safe.directory"
+  git config --global --add safe.directory '*'
+
+  echo "Using python versions: ${PYTHON_VERSIONS}"
+  local orig_path="${PATH}"
+
+  # Build phase.
+  echo "******************** BUILDING PACKAGE ********************"
+  for python_version in ${PYTHON_VERSIONS}; do
+    python_dir="/opt/python/${python_version}"
+    if ! [ -x "${python_dir}/bin/python" ]; then
+      echo "ERROR: Could not find python: ${python_dir} (skipping)"
+      continue
+    fi
+    export PATH="${python_dir}/bin:${orig_path}"
+    echo ":::: Python version $(python --version)"
+    clean_wheels "shortfin" "${python_version}"
+    build_shortfin
+    run_audit_wheel "shortfin" "${python_version}"
+  done
+}
+
+function build_shortfin() {
+  export SHORTFIN_ENABLE_TRACING=ON
+  python -m pip wheel --disable-pip-version-check -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/libshortfin"
+}
+
+function run_audit_wheel() {
+  local wheel_basename="$1"
+  local python_version="$2"
+  generic_wheel="${OUTPUT_DIR}/${wheel_basename}-${python_version}-linux_$(uname -m).whl"
+  ls "${generic_wheel}"
+  echo ":::: Auditwheel ${generic_wheel}"
+  auditwheel repair -w "${OUTPUT_DIR}" "${generic_wheel}"
+  rm -v "${generic_wheel}"
+}
+
+function clean_wheels() {
+  local wheel_basename="$1"
+  local python_version="$2"
+  echo ":::: Clean wheels ${wheel_basename} ${python_version}"
+  rm -f -v "${OUTPUT_DIR}/${wheel_basename}-"*"-${python_version}-"*".whl"
+}
+
+# Trampoline to the docker container if running on the host.
+if [ -z "${__MANYLINUX_BUILD_WHEELS_IN_DOCKER-}" ]; then
+  run_on_host "$@"
+else
+  run_in_docker "$@"
+fi

From 1809c03274d21815eca74af9716ceecfafb437f6 Mon Sep 17 00:00:00 2001
From: Scott Todd <scott.todd0@gmail.com>
Date: Fri, 27 Sep 2024 10:29:24 -0700
Subject: [PATCH 2/4] Change path to libshortfin, fix output path for
 auditwheel.

---
 shortfin/build_tools/build_linux_package.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh
index 2b7dbc398..a124b1a6b 100755
--- a/shortfin/build_tools/build_linux_package.sh
+++ b/shortfin/build_tools/build_linux_package.sh
@@ -90,13 +90,14 @@ function run_in_docker() {
 
 function build_shortfin() {
   export SHORTFIN_ENABLE_TRACING=ON
-  python -m pip wheel --disable-pip-version-check -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/libshortfin"
+  python -m pip wheel --disable-pip-version-check -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/shortfin"
 }
 
 function run_audit_wheel() {
   local wheel_basename="$1"
   local python_version="$2"
-  generic_wheel="${OUTPUT_DIR}/${wheel_basename}-${python_version}-linux_$(uname -m).whl"
+  # Force wildcard expansion here
+  generic_wheel="$(echo "${OUTPUT_DIR}/${wheel_basename}-"*"-${python_version}-linux_${ARCH}.whl")"
   ls "${generic_wheel}"
   echo ":::: Auditwheel ${generic_wheel}"
   auditwheel repair -w "${OUTPUT_DIR}" "${generic_wheel}"

From 92238117d120a258739eef4b59f04b8f85cfa331 Mon Sep 17 00:00:00 2001
From: Scott Todd <scott.todd0@gmail.com>
Date: Fri, 27 Sep 2024 11:12:51 -0700
Subject: [PATCH 3/4] Rework TODOs for manylinux version, fix example for
 `sudo` and env vars.

---
 shortfin/README.md                          | 22 +++++++++++++--------
 shortfin/build_tools/build_linux_package.sh | 14 +++++++------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/shortfin/README.md b/shortfin/README.md
index 5cabe0304..957658400 100644
--- a/shortfin/README.md
+++ b/shortfin/README.md
@@ -33,16 +33,22 @@ pip install -v -e .
 
 ## Package Python Release Builds
 
-```bash
-# Build shortfin.*.whl into the dist/ directory
-#   e.g. `shortfin-0.9-cp312-cp312-linux_x86_64.whl`
-python3 -m pip wheel -v -w dist .
+* To build wheels for Linux using a manylinux Docker container:
 
-# Install the built wheel.
-python3 -m pip install dist/*.whl
-```
+    ```bash
+    sudo ./build_tools/build_linux_package.sh
+    ```
+
+* To build a wheel for your host OS/arch manually:
+
+    ```bash
+    # Build shortfin.*.whl into the dist/ directory
+    #   e.g. `shortfin-0.9-cp312-cp312-linux_x86_64.whl`
+    python3 -m pip wheel -v -w dist .
 
-TODO: helper script to build under manylinux using Docker
+    # Install the built wheel.
+    python3 -m pip install dist/*.whl
+    ```
 
 ## Python Dev Builds
 
diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh
index a124b1a6b..851e5db83 100755
--- a/shortfin/build_tools/build_linux_package.sh
+++ b/shortfin/build_tools/build_linux_package.sh
@@ -14,9 +14,9 @@
 #   sudo ./build_tools/build_linux_package.sh
 #
 # Build specific Python versions to custom directory:
-#   OVERRIDE_PYTHON_VERSIONS="cp312-cp312 cp313-313" \
+#   OVERRIDE_PYTHON_VERSIONS="cp312-cp312 cp313-cp313" \
 #   OUTPUT_DIR="/tmp/wheelhouse" \
-#   sudo ./build_tools/build_linux_package.sh
+#   sudo -E ./build_tools/build_linux_package.sh
 #
 # Valid Python versions match a subdirectory under /opt/python in the docker
 # image. Typically:
@@ -35,10 +35,12 @@ REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)"
 SCRIPT_NAME="$(basename $0)"
 ARCH="$(uname -m)"
 
-# TODO: update to manylinux_2_28, 2014 is being used by torch-mlir
-# MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-"ghcr.io/nod-ai/manylinux_x86_64:main@sha256:facedb71df670016e74e646d71e869e6fff70d4cdbaa6634d4d0a10d6e174399" }')}"
-MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux2014_${ARCH}@sha256:2ace4b4f06a726d270821cb4993caeed3aacdaa54801e9f13be8fbe96077f4dc}"
-PYTHON_VERSIONS="${OVERRIDE_PYTHON_VERSIONS:-cp312-cp312}"
+# TODO(#130): Update to manylinux_2_28, upstream or a fork
+#   * upstream uses a version of gcc that has build warnings/errors
+#   * https://github.com/nod-ai/base-docker-images is a bit out of date but can include a recent clang
+# MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux_2_28_${ARCH}:latest}"
+MANYLINUX_DOCKER_IMAGE="${MANYLINUX_DOCKER_IMAGE:-quay.io/pypa/manylinux2014_${ARCH}:latest}"
+PYTHON_VERSIONS="${OVERRIDE_PYTHON_VERSIONS:-cp312-cp312 cp313-cp313}"
 OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}"
 
 function run_on_host() {

From b5017894185d96aec9f72b2b24963d87fd8f6a1d Mon Sep 17 00:00:00 2001
From: Scott Todd <scott.todd0@gmail.com>
Date: Mon, 30 Sep 2024 10:12:27 -0700
Subject: [PATCH 4/4] Add extra copyright from forked file.

---
 shortfin/build_tools/build_linux_package.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/shortfin/build_tools/build_linux_package.sh b/shortfin/build_tools/build_linux_package.sh
index 851e5db83..6f5c67d46 100755
--- a/shortfin/build_tools/build_linux_package.sh
+++ b/shortfin/build_tools/build_linux_package.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 # Copyright 2024 Advanced Micro Devices, Inc.
+# Copyright 2022 The IREE Authors
 #
 # Licensed under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.