diff --git a/.ci/build.sh b/.ci/build.sh index 28f79a7ab..aa5eb54e0 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,128 +1,36 @@ #!/usr/bin/env bash -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +function setup { + export BENCHMARK_HOME=$GITHUB_WORKSPACE -# fail this script immediately if any command fails with a non-zero exit code -set -e -# fail on pipeline errors, e.g. when grepping -set -o pipefail -# fail on any unset environment variables -set -u + export THESPLOG_FILE=$BENCHMARK_HOME/.benchmark/logs/actor-system-internal.log + export THESPLOG_FILE_MAXSIZE=204800 # default is 50 KiB + export THESPLOG_THRESHOLD=INFO # default log level is WARNING -function update_pyenv { - # need to have the latest pyenv version to ensure latest patch releases are installable - cd $HOME/.pyenv/plugins/python-build/../.. && git pull origin master --rebase && cd - -} - -function build { - export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}" - # this value is in bytes, the default is 50kB. We increase it to 200kiB. - export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800} - # adjust the default log level from WARNING - export THESPLOG_THRESHOLD="INFO" - - # turn nounset off because some of the following commands fail if nounset is turned on - set +u - - export PATH="$HOME/.pyenv/bin:$PATH" export TERM=dumb export LC_ALL=en_US.UTF-8 - update_pyenv - eval "$(pyenv init -)" - # ensure pyenv shims are added to PATH, see https://github.com/pyenv/pyenv/issues/1906 - eval "$(pyenv init --path)" - eval "$(pyenv virtualenv-init -)" - make prereq - make install - make precommit - make test + # Init pyenv. + PATH=$HOME/.pyenv/shims:$PATH:$HOME/.pyenv/bin } -function build_it { - export THESPLOG_FILE="${THESPLOG_FILE:-${BENCHMARK_HOME}/.benchmark/logs/actor-system-internal.log}" - # this value is in bytes, the default is 50kB. We increase it to 200kiB. - export THESPLOG_FILE_MAXSIZE=${THESPLOG_FILE_MAXSIZE:-204800} - # adjust the default log level from WARNING - export THESPLOG_THRESHOLD="INFO" - - # turn nounset off because some of the following commands fail if nounset is turned on - set +u - - export PATH="$HOME/.pyenv/bin:$PATH" - export TERM=dumb - export LC_ALL=en_US.UTF-8 - export BENCHMARK_HOME="$GITHUB_WORKSPACE" +function build_and_unit_test { + setup - update_pyenv - eval "$(pyenv init -)" - # ensure pyenv shims are added to PATH, see https://github.com/pyenv/pyenv/issues/1906 - eval "$(pyenv init --path)" - eval "$(pyenv virtualenv-init -)" + set -e + make develop + make lint + make test +} - python3_version=`python3 --version` - echo "Python3 version is ... $python3_version" +function run_it { + setup - python3 -m pip install opensearch-benchmark docker pull ubuntu/squid:latest - make prereq - make install - make precommit - - # make it38, it39, etc. + # make it38, it39, etc. so they run as concurrent GHA jobs make "it${1//./}" } -function license-scan { - # turn nounset off because some of the following commands fail if nounset is turned on - set +u - - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - # ensure pyenv shims are added to PATH, see https://github.com/pyenv/pyenv/issues/1906 - eval "$(pyenv init --path)" - eval "$(pyenv virtualenv-init -)" - - make prereq - # only install depdencies that are needed by end users - make install-user - fossa analyze -} - -function archive { - # Treat unset env variables as an error, but only in this function as there are other functions that allow unset variables - set -u - - # this will only be done if the build number variable is present - BENCHMARK_DIR=${BENCHMARK_HOME}/.benchmark - if [[ -d ${BENCHMARK_DIR} ]]; then - find ${BENCHMARK_DIR} -name "*.log" -printf "%P\\0" | tar -cvjf ${BENCHMARK_DIR}/${BUILD_NUMBER}.tar.bz2 -C ${BENCHMARK_DIR} --transform "s,^,ci-${BUILD_NUMBER}/," --null -T - - else - echo "Benchmark directory [${BENCHMARK_DIR}] not present. Ensure the BENCHMARK_DIR environment variable is correct" - exit 1 - fi -} +$@ -if declare -F "$1" > /dev/null; then - "$@" - exit -else - echo "Please specify a function to run" - exit 1 -fi diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-build.yml similarity index 60% rename from .github/workflows/docker-test.yml rename to .github/workflows/docker-build.yml index 039733521..da87a35d3 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-build.yml @@ -1,4 +1,4 @@ -name: Docker Build and Test +name: Docker Build on: pull_request: workflow_dispatch: @@ -28,14 +28,12 @@ jobs: with: version: 'v0.9.1' - uses: actions/checkout@v4 - with: - path: 'opensearch-benchmark-git' + - name: Docker Build ${{ matrix.platform }} run: | docker buildx version - cp -a opensearch-benchmark-git/* ./ - echo "Disable VERSION arg to enter docker build test mode" - PLATFORM=${{ matrix.platform }} - PLATFORM=`echo $PLATFORM | tr '/' '-'` - docker buildx build --platform ${{ matrix.platform }} --build-arg BUILD_ENV=testing --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` -f "docker/Dockerfile" -t "osb/osb-$PLATFORM" -o type=docker . - docker images | grep "osb/osb-$PLATFORM" + tag=osb/osb-`echo ${{ matrix.platform }} | tr '/' '-'` + set -x + docker buildx build --platform ${{ matrix.platform }} --build-arg VERSION=`cat version.txt` --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` -f docker/Dockerfile -t "$tag" -o type=docker . + set +x + docker images | grep "$tag" diff --git a/.github/workflows/manual-integ.yml b/.github/workflows/integ-test.yml similarity index 51% rename from .github/workflows/manual-integ.yml rename to .github/workflows/integ-test.yml index 66906fae1..9195f0eaa 100644 --- a/.github/workflows/manual-integ.yml +++ b/.github/workflows/integ-test.yml @@ -1,4 +1,4 @@ -name: Integ Actions +name: Run Integration Tests on: [workflow_dispatch, pull_request] jobs: Integration-Tests: @@ -12,11 +12,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} + - uses: KengoTODA/actions-setup-docker-compose@v1 with: version: '1.29.2' + # - name: Enforce docker-compose v1 # run: | # echo "GitHub starts to switch runners to include docker-compose v2" @@ -27,43 +27,19 @@ jobs: # sudo pip install docker-compose==1.29.2 # docker --version # docker-compose --version + - name: Check out repository code uses: actions/checkout@v2 - - name: Clone pyenv + + - name: Install pyenv run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv - - name: Clone pyenv-virtualenv - run: git clone https://github.com/pyenv/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv - - name: Install JDK 8 - uses: actions/setup-java@v3 - with: - distribution: 'adopt' - java-version: '8' - - run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV - - name: Install JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'adopt' - java-version: '11' - - run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV - - name: Install JDK 15 - uses: actions/setup-java@v3 - with: - distribution: 'adopt' - java-version: '15' - - run: echo "JAVA15_HOME=$JAVA_HOME" >> $GITHUB_ENV - - name: Install JDK 16 - uses: actions/setup-java@v3 - with: - distribution: 'adopt' - java-version: '16' - - run: echo "JAVA16_HOME=$JAVA_HOME" >> $GITHUB_ENV + - name: Install JDK 17 uses: actions/setup-java@v3 with: distribution: 'adopt' java-version: '17' - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV - - name: Run the CI build_it script - run: bash .ci/build.sh build_it ${{ matrix.python-version }} - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + + - name: Run the CI build script + run: bash .ci/build.sh run_it ${{ matrix.python-version }} diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/publish-release.yml similarity index 79% rename from .github/workflows/release-drafter.yml rename to .github/workflows/publish-release.yml index 54e12cf82..ab07aadd5 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/publish-release.yml @@ -1,4 +1,4 @@ -name: Release drafter +name: Publish Release to GitHub on: push: @@ -6,7 +6,7 @@ on: - "*" jobs: - draft-a-release: + publish-release: runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -22,21 +22,20 @@ jobs: issue-title: 'Release opensearch-benchmark' issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }} **Commit**: ${{ github.sha }}" exclude-workflow-initiator-as-approver: true + - name: Set up Python 3 uses: actions/setup-python@v3 with: python-version: '3.x' - - name: Install build tools - run: | - python -m pip install --upgrade build + - name: Build project for distribution run: | - python -m build - tar -zvcf artifacts.tar.gz dist - - name: Release + make build + tar zcvf artifacts.tar.gz dist + + - name: Publish release uses: softprops/action-gh-release@v1 with: draft: true generate_release_notes: true - files: | - artifacts.tar.gz \ No newline at end of file + files: artifacts.tar.gz diff --git a/.github/workflows/main.yml b/.github/workflows/unit-test.yml similarity index 72% rename from .github/workflows/main.yml rename to .github/workflows/unit-test.yml index 42e872782..5ae43cbf7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/unit-test.yml @@ -1,5 +1,5 @@ -name: CI Actions -on: [pull_request] +name: Run Unit Tests +on: [workflow_dispatch, pull_request] jobs: Unit-Tests: strategy: @@ -8,12 +8,13 @@ jobs: - ubuntu-latest - macos-latest runs-on: ${{ matrix.os }} + steps: - name: Check out repository code uses: actions/checkout@v2 + - name: Clone pyenv run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv + - name: Run the CI build script - run: bash .ci/build.sh build - env: - BENCHMARK_HOME: env.GITHUB_WORKSPACE + run: bash .ci/build.sh build_and_unit_test diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index bd7767d9d..e9088d9a2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -23,16 +23,29 @@ This document will walk you through on what's needed to start contributing code ### Prerequisites - - **Pyenv** : Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. - For more details please refer to the [PyEnv installation instructions](https://github.com/pyenv/pyenv#installation). + - **pyenv**: Install `pyenv` and follow the instructions in the output of `pyenv init` to set up your shell and restart it before proceeding. + For more details please refer to the [pyenv installation instructions](https://github.com/pyenv/pyenv#installation). - **Optional Step:** For Debian-based systems, install the following modules to continue with the next steps: + `pyenv` requires that the C compiler and development libraries be installed, so that the specified Python versions can be build from source. The installation instructions vary from platform to platform. + + For Debian-based systems, install the following modules to continue with the next steps: ``` sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ xz-utils tk-dev libffi-dev liblzma-dev git ``` + For Amazon Linux 2023, run the following command: + ``` + sudo yum -y install gcc openssl-devel bzip2-devel libffi-devel ncurses-devel sqlite-devel readline-devel zlib-devel xz-devel + ``` + + On the Mac platform, XCode needs to be installed as well as some additional required libraries: + ``` + xcode-select --install + brew install pyenv jq zlib xz + ``` + - **JDK**: Although OSB is a Python application, it optionally builds and provisions OpenSearch clusters. JDK version 17 is used to build the current version of OpenSearch. Please refer to the [build setup requirements](https://github.com/opensearch-project/OpenSearch/blob/ca564fd04f5059cf9e3ce8aba442575afb3d99f1/DEVELOPER_GUIDE.md#install-prerequisites). Note that the `javadoc` executable should be available in the JDK installation. An earlier version of the JDK can be used, but not all the integration tests will pass. @@ -54,15 +67,8 @@ For those working on WSL2, it is recommended to clone the repository and set up After you git cloned the forked copy of OpenSearch Benchmark, use the following command-line instructions to set up OpenSearch Benchmark for development: ``` cd opensearch-benchmark -make prereq -make install -``` - -NOTE: `make prereq` produces the following message. -``` -IMPORTANT: please add `eval "$(pyenv init -)"` to your bash profile and restart your terminal before proceeding any further. +make develop ``` -This line is commonly thought of as an error message but rather it's just a warning. Unless you haven't already added `eval "$(pyenv init -)"` to your bash profile and restarted your terminal, then feel free to proceed forward. This eval statement is necessary in the startup configuration as it allows Pyenv to manage python versions by adding python shims to your path. If you experience any issues, please see https://github.com/pyenv/pyenv. Depending on the platform and shell you have, use the following command to activate the virtual environment: diff --git a/Makefile b/Makefile index dcf66d8fa..a1d1ae7c6 100644 --- a/Makefile +++ b/Makefile @@ -16,38 +16,27 @@ # under the License. SHELL = /bin/bash -# We assume an active virtualenv for development -PYENV_REGEX = .pyenv/shims -PY_BIN = python3 -PY_PREFIX = python -# https://github.com/pypa/pip/issues/5599 -PIP_WRAPPER = $(PY_BIN) -m pip -export PY38 = $(shell jq -r '.python_versions.PY38' .ci/variables.json) -export PY38_BIN = $(PY_PREFIX)$(shell cut -d '.' -f 1,2 <<< $(PY38)) -export PY39 = $(shell jq -r '.python_versions.PY39' .ci/variables.json) -export PY39_BIN = $(PY_PREFIX)$(shell cut -d '.' -f 1,2 <<< $(PY39)) -export PY310 = $(shell jq -r '.python_versions.PY310' .ci/variables.json) -export PY310_BIN = $(PY_PREFIX)$(shell cut -d '.' -f 1,2 <<< $(PY310)) -export PY311 = $(shell jq -r '.python_versions.PY311' .ci/variables.json) -export PY311_BIN = $(PY_PREFIX)$(shell cut -d '.' -f 1,2 <<< $(PY311)) -VENV_NAME ?= .venv -VENV_ACTIVATE_FILE = $(VENV_NAME)/bin/activate -VENV_ACTIVATE = . $(VENV_ACTIVATE_FILE) -VEPYTHON = $(VENV_NAME)/bin/$(PY_BIN) -VEPYLINT = $(VENV_NAME)/bin/pylint -PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv.\n" -PYENV_PREREQ_HELP = "\033[0;31mIMPORTANT\033[0m: If you haven't already, please add \033[0;31meval \"\$$(pyenv init -)\"\033[0m to your bash profile and restart your terminal before proceeding any further.\n" -VE_MISSING_HELP = "\033[0;31mIMPORTANT\033[0m: Couldn't find $(PWD)/$(VENV_NAME); have you executed make venv-create?\033[0m\n" - -prereq: - pyenv install --skip-existing $(PY38) - pyenv install --skip-existing $(PY39) - pyenv install --skip-existing $(PY310) - pyenv install --skip-existing $(PY311) - pyenv local $(PY38) - @# Ensure all Python versions are registered for this project - @ jq -r '.python_versions | [.[] | tostring] | join("\n")' .ci/variables.json > .python-version - -@ printf $(PYENV_PREREQ_HELP) +PYTHON = python3 +PIP = pip3 +VERSIONS = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d') +VERSION38 = $(shell jq -r '.python_versions | .[]' .ci/variables.json | sed '$$d' | grep 3\.8) +PYENV_ERROR = "\033[0;31mIMPORTANT\033[0m: Please install pyenv and run \033[0;31meval \"\$$(pyenv init -)\"\033[0m.\n" + +all: develop + +pyinst: + @which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; } + @for i in $(VERSIONS); do pyenv install --skip-existing $$i; done + pyenv local $(VERSIONS) + +pyinst38: + @which pyenv > /dev/null 2>&1 || { printf $(PYENV_ERROR); exit 1; } + pyenv install --skip-existing $(VERSION38) + pyenv local $(VERSION38) + +check-pip: + # Install pyenv if the Python environment is externally managed. + @if ! $(PIP) > /dev/null 2>&1 || ! $(PIP) install pip > /dev/null 2>&1; then make pyinst38; fi check-java: @if ! test "$(JAVA_HOME)" || ! java --version > /dev/null 2>&1 || ! javadoc --help > /dev/null 2>&1; then \ @@ -58,87 +47,63 @@ check-java: echo "NOTE: Java version 17 required to have all integration tests pass" >&2; \ fi -venv-create: - @if [[ ! -x $$(command -v pyenv) ]]; then \ - printf $(PYENV_ERROR); \ - exit 1; \ - fi; - @if [[ ! -f $(VENV_ACTIVATE_FILE) ]]; then \ - eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" && $(PY38_BIN) -mvenv $(VENV_NAME); \ - eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" && $(PY39_BIN) -mvenv $(VENV_NAME); \ - eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" && $(PY310_BIN) -mvenv $(VENV_NAME); \ - eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" && $(PY311_BIN) -mvenv $(VENV_NAME); \ - printf "Created python3 venv under $(PWD)/$(VENV_NAME).\n"; \ - fi; - -check-venv: - @if [[ ! -f $(VENV_ACTIVATE_FILE) ]]; then \ - printf $(VE_MISSING_HELP); \ - fi - -install-user: venv-create - . $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install --upgrade pip setuptools wheel - . $(VENV_ACTIVATE_FILE); PIP_ONLY_BINARY=h5py $(PIP_WRAPPER) install -e . +install-deps: check-pip + $(PIP) install --upgrade pip setuptools wheel -install: install-user - # Also install development dependencies - . $(VENV_ACTIVATE_FILE); $(PIP_WRAPPER) install -e .[develop] +# pylint does not work with Python versions >3.8: +# Value 'Optional' is unsubscriptable (unsubscriptable-object) +develop: pyinst38 install-deps + PIP_ONLY_BINARY=h5py $(PIP) install -e .[develop] -clean: nondocs-clean docs-clean +build: install-deps + $(PIP) install --upgrade build + $(PYTHON) -m build -nondocs-clean: - rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist osbenchmark.egg-info logs junit-py*.xml NOTICE.txt +# Builds a wheel from source, then installs it. +install: build + PIP_ONLY_BINARY=h5py $(PIP) install dist/opensearch_benchmark-*.whl + rm -rf dist -docs-clean: - cd docs && $(MAKE) clean +clean: + rm -rf .benchmarks .eggs .tox .benchmark_it .cache build dist *.egg-info logs junit-py*.xml *.whl NOTICE.txt # Avoid conflicts between .pyc/pycache related files created by local Python interpreters and other interpreters in Docker python-caches-clean: -@find . -name "__pycache__" -prune -exec rm -rf -- \{\} \; -@find . -name ".pyc" -prune -exec rm -rf -- \{\} \; -# Force recreation of the virtual environment used by tox. -# -# See https://github.com/opensearch-project/OpenSearch-Benchmark/blob/main/DEVELOPER_GUIDE.md: -# -# > Note pip will not update project dependencies (specified either in the install_requires or the extras -# > section of the setup.py) if any version already exists in the virtual environment; therefore we recommend -# > to recreate your environments whenever your project dependencies change. +# Note: pip will not update project dependencies (specified either in the install_requires or the extras +# section of the setup.py) if any version is already installed; therefore we recommend +# recreating your environments whenever your project dependencies change. tox-env-clean: rm -rf .tox -lint: check-venv - @find osbenchmark benchmarks scripts tests it -name "*.py" -exec $(VEPYLINT) -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + - -docs: check-venv - @. $(VENV_ACTIVATE_FILE); cd docs && $(MAKE) html - -serve-docs: check-venv - @. $(VENV_ACTIVATE_FILE); cd docs && $(MAKE) serve - -test: check-venv - . $(VENV_ACTIVATE_FILE); pytest tests/ +lint: + @find osbenchmark benchmarks scripts tests it -name "*.py" -exec pylint -j0 -rn --load-plugins pylint_quotes --rcfile=$(CURDIR)/.pylintrc \{\} + -precommit: lint +test: develop + pytest tests/ -it: check-java check-venv python-caches-clean tox-env-clean - . $(VENV_ACTIVATE_FILE); tox +it: pyinst check-java python-caches-clean tox-env-clean + @which tox || $(PIP) install tox + tox -it38 it39 it310 it311: check-java check-venv python-caches-clean tox-env-clean - . $(VENV_ACTIVATE_FILE); tox -e $(@:it%=py%) +it38 it39 it310 it311: pyinst check-java python-caches-clean tox-env-clean + @which tox || $(PIP) install tox + tox -e $(@:it%=py%) -benchmark: check-venv - . $(VENV_ACTIVATE_FILE); pytest benchmarks/ +benchmark: + pytest benchmarks/ -coverage: check-venv - . $(VENV_ACTIVATE_FILE); coverage run setup.py test - . $(VENV_ACTIVATE_FILE); coverage html +coverage: + coverage run setup.py test + coverage html -release-checks: check-venv - . $(VENV_ACTIVATE_FILE); ./release-checks.sh $(release_version) $(next_version) +release-checks: + ./release-checks.sh $(release_version) $(next_version) # usage: e.g. make release release_version=0.9.2 next_version=0.9.3 -release: check-venv release-checks clean docs it - . $(VENV_ACTIVATE_FILE); ./release.sh $(release_version) $(next_version) +release: release-checks clean it + ./release.sh $(release_version) $(next_version) -.PHONY: install clean nondocs-clean docs-clean python-caches-clean tox-env-clean docs serve-docs test it it38 benchmark coverage release release-checks prereq venv-create check-env +.PHONY: install clean python-caches-clean tox-env-clean test it it38 benchmark coverage release release-checks pyinst diff --git a/docker/Dockerfile b/docker/Dockerfile index 147b5b861..0a777952d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,36 +2,50 @@ # Install OpenSearch Benchmark to build a Docker image # ######################################################## -ARG VERSION -ARG BUILD_ENV=production +# +# Stage 1: build packages and compile where needed +# +FROM python:3.11.2-slim AS build-stage -FROM python:3.11.2-slim as build_env_testing -ONBUILD COPY opensearch-benchmark-git/ ./ +RUN apt-get -y update && \ + apt-get install -y curl git gcc pbzip2 pigz make jq && \ + apt-get -y upgrade -FROM python:3.11.2-slim as build_env_production -ONBUILD RUN echo Production Environment +COPY . opensearch-benchmark -FROM build_env_${BUILD_ENV} -WORKDIR /opensearch-benchmark -ENV BENCHMARK_RUNNING_IN_DOCKER=True +# There is no binary package currently available for yappi on ARM. +RUN cd opensearch-benchmark; \ + make build; \ + if test "`uname -m`" = aarch64; then pip wheel yappi && cp yappi-*.whl dist; fi -RUN apt-get -y update && \ - apt-get install -y curl git gcc pbzip2 pigz && \ - apt-get -y upgrade && \ - rm -rf /var/lib/apt/lists/* + +# +# Stage 2: create image +# +FROM python:3.11.2-slim AS image-stage +ENV BENCHMARK_RUNNING_IN_DOCKER=True RUN groupadd --gid 1000 opensearch-benchmark && \ useradd -d /opensearch-benchmark -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash benchmark -ENV PIP_ONLY_BINARY=h5py -RUN if [ "$BUILD_ENV" = "testing" ] ; then echo Testing; ls -l; python3 -m pip install -e . ; \ - else echo Production; if [ -z "$VERSION" ] ; then python3 -m pip install opensearch-benchmark ; else python3 -m pip install opensearch-benchmark==$VERSION ; fi; fi +WORKDIR /opensearch-benchmark RUN mkdir -p /opensearch-benchmark/.benchmark && \ chown -R 1000:0 /opensearch-benchmark/.benchmark +COPY --from=build-stage /opensearch-benchmark/dist/*.whl ./ + +RUN set -ex; \ + apt-get -y update; \ + apt-get install -y git pbzip2; \ + apt-get -y upgrade; \ + rm -rf /var/lib/apt/lists/*; \ + PIP_ONLY_BINARY=h5py pip install *.whl; \ + rm *.whl + USER 1000 +ARG VERSION ARG BUILD_DATE LABEL org.label-schema.schema-version="1.0" \ diff --git a/setup.py b/setup.py index 5e0cc4c7f..55ba5b85a 100644 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ def str_from_file(name): "tox==3.14.0", "coverage==5.5", "twine==1.15.0", - "wheel==0.38.4", + "wheel>=0.38.4", "github3.py==1.3.0", "pylint==2.6.0", "pylint-quotes==0.2.1"