From 0bbabc559c9010fadc7d3185d3d700c7fee22758 Mon Sep 17 00:00:00 2001 From: Grieve Date: Thu, 23 May 2024 16:31:37 +0800 Subject: [PATCH 01/17] Add Dockerfile --- ibis-server/Dockerfile | 38 ++++++++++++++++++++++++++++++++++++++ ibis-server/Makefile | 6 ++++++ ibis-server/README.md | 4 ++++ ibis-server/pyproject.toml | 2 +- 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 ibis-server/Dockerfile diff --git a/ibis-server/Dockerfile b/ibis-server/Dockerfile new file mode 100644 index 000000000..898a8d95a --- /dev/null +++ b/ibis-server/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.11.9-slim as builder + +# libpq-dev is required for psycopg2 +RUN apt-get update && apt-get -y install libpq-dev gcc + + # python +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + \ + # pip + PIP_NO_CACHE_DIR=off \ + PIP_DISABLE_PIP_VERSION_CHECK=on \ + PIP_DEFAULT_TIMEOUT=100 \ + \ + # poetry + POETRY_VIRTUALENVS_IN_PROJECT=true \ + POETRY_CACHE_DIR=/tmp/poetry_cache + +RUN pip install poetry==1.7.1 + +WORKDIR /app + +COPY pyproject.toml ./ + +RUN poetry install --without dev && rm -rf $POETRY_CACHE_DIR + +FROM python:3.11.9-slim as runtime + +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" + +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +COPY app app + +EXPOSE 8000 + +ENTRYPOINT fastapi run \ No newline at end of file diff --git a/ibis-server/Makefile b/ibis-server/Makefile index 0fa3d9e90..a72a5857f 100644 --- a/ibis-server/Makefile +++ b/ibis-server/Makefile @@ -13,3 +13,9 @@ dev: test: poetry run pytest + +docker-build: + docker image build . -t ibis-server + +docker-run: + docker run -it --rm -p 8000:8000 --env-file .env ibis-server \ No newline at end of file diff --git a/ibis-server/README.md b/ibis-server/README.md index c90118641..0235c3f5c 100644 --- a/ibis-server/README.md +++ b/ibis-server/README.md @@ -14,3 +14,7 @@ - Execute `make run` to start the server - Execute `make dev` to start the server in development mode. It will auto-reload after the code is edited. - Default port is `8000`, you can change it by `make run PORT=8001` or `make dev PORT=8001` + +## Docker +- Build the image: `make docker-build` +- Run the container: `make docker-run` \ No newline at end of file diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index 9391caab2..4b8c65931 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -15,7 +15,7 @@ ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0 google-auth = "2.29.0" httpx = "0.27.0" python-dotenv = "1.0.1" -orjson = "2.0.1" +orjson = "3.10.3" [tool.poetry.group.dev.dependencies] pytest = "8.2.0" From 71b77e70ec98ca5392425f857d7b3811adb94a6e Mon Sep 17 00:00:00 2001 From: Grieve Date: Thu, 23 May 2024 17:53:48 +0800 Subject: [PATCH 02/17] Make ibis version same as java engine --- ibis-server/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index 4b8c65931..21ddc9eb0 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ibis-server" -version = "0.1.0" +version = "0.4.5" description = "" authors = ["Canner "] readme = "README.md" From 18a0750fa1526eadf3e655aa058fda70a6ec6e18 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 11:03:37 +0800 Subject: [PATCH 03/17] Add CI Install python dependencies in the CI --- .github/workflows/build-dev-image.yml | 41 ++++++++++++++ .github/workflows/build-image.yml | 71 +++++++++++++++++++---- .github/workflows/stable-release.yml | 82 ++++++++++++++++++++++++++- ibis-server/Dockerfile | 24 ++++---- ibis-server/Makefile | 4 +- ibis-server/pyproject.toml | 4 +- 6 files changed, 201 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 595e24223..90d72a9a7 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -47,3 +47,44 @@ jobs: --tag ghcr.io/canner/wren-engine:nightly \ --push -f ./Dockerfile \ --build-arg "WREN_VERSION=${WREN_VERSION}" . + build-ibis-image: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./ibis-server + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - uses: abatilo/actions-poetry@v3 + with: + poetry-version: 1.7.1 + - name: Setup a local virtual environment + run: | + poetry config virtualenvs.in-project true --local + - uses: actions/cache@v4 + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + - name: Install the project dependencies + run: poetry install --without ibis,dev + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and Push Docker image + run: | + LATEST_COMMIT=$(git log -1 --pretty=%h) + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --tag ghcr.io/canner/wren-engine-ibis:${LATEST_COMMIT} \ + --tag ghcr.io/canner/wren-engine-ibis:nightly \ + --file Dockerfile \ + --push . diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 5d7d05b50..e6fbd88ef 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -8,7 +8,23 @@ on: description: Docker image tag name (Optional) jobs: + prepare-tag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Prepare tag name + id: prepare_tag + run: | + if [ -n "${{ github.event.inputs.docker_image_tag_name }}" ]; then + tag_name=${{ github.event.inputs.docker_image_tag_name }} + else + tag_name=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')-$(git log -1 --pretty=%h) + fi + echo "tag_name=$tag_name" >> "$GITHUB_OUTPUT" + outputs: + tag_name: ${{ steps.prepare_tag.outputs.tag_name }} build-image: + needs: prepare-tag runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,16 +47,9 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Prepare tag name - id: prepare_tag - run: | - if [ -n "${{ github.event.inputs.docker_image_tag_name }}" ]; then - tag_name=${{ github.event.inputs.docker_image_tag_name }} - else - tag_name=$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')-$(git log -1 --pretty=%h) - fi - echo ::set-output name=tag_name::$tag_name - name: Build Docker image + env: + TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }} run: | WREN_VERSION=$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout) cd ./docker @@ -48,6 +57,48 @@ jobs: cp -r ../wren-sqlglot-server/ ./ docker buildx build \ --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine:${{ steps.prepare_tag.outputs.tag_name }} \ + --tag ghcr.io/canner/wren-engine:$TAG_NAME \ --push -f ./Dockerfile \ --build-arg "WREN_VERSION=${WREN_VERSION}" . + build-ibis-image: + needs: prepare-tag + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./ibis-server + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - uses: abatilo/actions-poetry@v3 + with: + poetry-version: 1.7.1 + - name: Setup a local virtual environment + run: | + poetry config virtualenvs.in-project true --local + - uses: actions/cache@v4 + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + - name: Install the project dependencies + run: poetry install --without ibis,dev + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and Push Docker image + env: + TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --tag ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ + --file Dockerfile \ + --push . diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index e61a03ece..4dca75c83 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -8,7 +8,7 @@ on: description: Specific version number (Optional). Default will be the current version plus 0.0.1. jobs: - prepare-release: + stable-release-wren-engine: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -62,3 +62,83 @@ jobs: --tag ghcr.io/canner/wren-engine:latest \ --push -f ./Dockerfile \ --build-arg "WREN_VERSION=${WREN_VERSION}" . + prepare-ibis-version: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./ibis-server + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GHCR_TOKEN }} + - name: Set up Git + run: | + git config --global user.email "dev@cannerdata.com" + git config --global user.name "stable-release-bot" + - uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - uses: abatilo/actions-poetry@v3 + with: + poetry-version: 1.7.1 + - name: Prepare next version + id: next_version + run: | + if [ -n "${{ github.event.inputs.specific_version }}" ]; then + version=${{ github.event.inputs.specific_version }} + else + version=$(grep -m 1 version pyproject.toml | grep -e '\d.\d.\d' -o) + IFS=. read major minor micro <<< "${version}" + micro=$((micro + 1)) + version=$major.$minor.$micro + fi + poetry version --next-phase $version + git add pyproject.toml + git commit -m "Upgrade ibis version to $version" + git push + echo "value=$version" >> $GITHUB_OUTPUT + outputs: + next_version: ${{ steps.next_version.outputs.value }} + stable-release-ibis: + needs: prepare-ibis-version + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./ibis-server + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version-file: pyproject.toml + - uses: abatilo/actions-poetry@v3 + with: + poetry-version: 1.7.1 + - name: Setup a local virtual environment + run: | + poetry config virtualenvs.in-project true --local + - uses: actions/cache@v4 + with: + path: ./.venv + key: venv-${{ hashFiles('poetry.lock') }} + - name: Install the project dependencies + run: poetry install --without ibis,dev + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build and Push Docker image + env: + TAG_NAME: ${{ steps.prepare-ibis-version.outputs.next_version }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --tag ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ + --tag ghcr.io/canner/wren-engine-ibis:latest \ + --file Dockerfile \ + --push . diff --git a/ibis-server/Dockerfile b/ibis-server/Dockerfile index 898a8d95a..d047f603b 100644 --- a/ibis-server/Dockerfile +++ b/ibis-server/Dockerfile @@ -1,38 +1,40 @@ FROM python:3.11.9-slim as builder # libpq-dev is required for psycopg2 -RUN apt-get update && apt-get -y install libpq-dev gcc +RUN apt-get update && apt-get -y install libpq-dev # python ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ - \ # pip PIP_NO_CACHE_DIR=off \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 \ - \ # poetry - POETRY_VIRTUALENVS_IN_PROJECT=true \ - POETRY_CACHE_DIR=/tmp/poetry_cache + POETRY_VIRTUALENVS_IN_PROJECT=true RUN pip install poetry==1.7.1 WORKDIR /app COPY pyproject.toml ./ +COPY poetry.lock ./ +COPY .venv ./ -RUN poetry install --without dev && rm -rf $POETRY_CACHE_DIR +RUN poetry install --with ibis FROM python:3.11.9-slim as runtime -ENV VIRTUAL_ENV=/app/.venv \ - PATH="/app/.venv/bin:$PATH" - -COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} +# libpq-dev is required for psycopg2 +RUN apt-get update \ + && apt-get -y install libpq-dev \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/.venv /app/.venv COPY app app +ENV PATH="/app/.venv/bin:$PATH" + EXPOSE 8000 -ENTRYPOINT fastapi run \ No newline at end of file +ENTRYPOINT fastapi run diff --git a/ibis-server/Makefile b/ibis-server/Makefile index a72a5857f..f11f3de8f 100644 --- a/ibis-server/Makefile +++ b/ibis-server/Makefile @@ -15,7 +15,7 @@ test: poetry run pytest docker-build: - docker image build . -t ibis-server + docker image build . -t wren-engine-ibis docker-run: - docker run -it --rm -p 8000:8000 --env-file .env ibis-server \ No newline at end of file + docker run -it --rm -p 8000:8000 --env-file .env wren-engine-ibis diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index 21ddc9eb0..e4af87d0d 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -11,12 +11,14 @@ python = ">=3.11,<3.12" fastapi = "0.111.0" pydantic = "2.6.4" uvicorn = {extras = ["standard"], version = "0.29.0"} -ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0.0"} google-auth = "2.29.0" httpx = "0.27.0" python-dotenv = "1.0.1" orjson = "3.10.3" +[tool.poetry.group.ibis.dependencies] +ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0.0"} + [tool.poetry.group.dev.dependencies] pytest = "8.2.0" From 9c92ea7b8a4ef671d9e35dec2edc48ab5174fb12 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 11:50:51 +0800 Subject: [PATCH 04/17] Make python dependencies install in the Docker If we install the dependencies in the CI, may cause platform issue --- .github/workflows/build-dev-image.yml | 15 --------------- .github/workflows/build-image.yml | 15 --------------- .github/workflows/stable-release.yml | 15 --------------- ibis-server/Dockerfile | 19 ++++++++++--------- ibis-server/pyproject.toml | 4 +--- 5 files changed, 11 insertions(+), 57 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 90d72a9a7..772d623b3 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -54,21 +54,6 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - uses: abatilo/actions-poetry@v3 - with: - poetry-version: 1.7.1 - - name: Setup a local virtual environment - run: | - poetry config virtualenvs.in-project true --local - - uses: actions/cache@v4 - with: - path: ./.venv - key: venv-${{ hashFiles('poetry.lock') }} - - name: Install the project dependencies - run: poetry install --without ibis,dev - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index e6fbd88ef..3b315aa4d 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -68,21 +68,6 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - uses: abatilo/actions-poetry@v3 - with: - poetry-version: 1.7.1 - - name: Setup a local virtual environment - run: | - poetry config virtualenvs.in-project true --local - - uses: actions/cache@v4 - with: - path: ./.venv - key: venv-${{ hashFiles('poetry.lock') }} - - name: Install the project dependencies - run: poetry install --without ibis,dev - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 4dca75c83..eea7a80f4 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -107,21 +107,6 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version-file: pyproject.toml - - uses: abatilo/actions-poetry@v3 - with: - poetry-version: 1.7.1 - - name: Setup a local virtual environment - run: | - poetry config virtualenvs.in-project true --local - - uses: actions/cache@v4 - with: - path: ./.venv - key: venv-${{ hashFiles('poetry.lock') }} - - name: Install the project dependencies - run: poetry install --without ibis,dev - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: diff --git a/ibis-server/Dockerfile b/ibis-server/Dockerfile index d047f603b..0138bbeea 100644 --- a/ibis-server/Dockerfile +++ b/ibis-server/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.9-slim as builder +FROM python:3.11-buster as builder # libpq-dev is required for psycopg2 RUN apt-get update && apt-get -y install libpq-dev @@ -11,29 +11,30 @@ ENV PYTHONUNBUFFERED=1 \ PIP_DISABLE_PIP_VERSION_CHECK=on \ PIP_DEFAULT_TIMEOUT=100 \ # poetry - POETRY_VIRTUALENVS_IN_PROJECT=true + POETRY_NO_INTERACTION=1 \ + POETRY_VIRTUALENVS_IN_PROJECT=1 \ + POETRY_VIRTUALENVS_CREATE=1 RUN pip install poetry==1.7.1 WORKDIR /app COPY pyproject.toml ./ -COPY poetry.lock ./ -COPY .venv ./ -RUN poetry install --with ibis +RUN poetry install --without dev -FROM python:3.11.9-slim as runtime +FROM python:3.11-slim-buster as runtime # libpq-dev is required for psycopg2 RUN apt-get update \ && apt-get -y install libpq-dev \ && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/.venv /app/.venv -COPY app app +ENV VIRTUAL_ENV=/app/.venv \ + PATH="/app/.venv/bin:$PATH" -ENV PATH="/app/.venv/bin:$PATH" +COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV} +COPY app app EXPOSE 8000 diff --git a/ibis-server/pyproject.toml b/ibis-server/pyproject.toml index e4af87d0d..21ddc9eb0 100644 --- a/ibis-server/pyproject.toml +++ b/ibis-server/pyproject.toml @@ -11,14 +11,12 @@ python = ">=3.11,<3.12" fastapi = "0.111.0" pydantic = "2.6.4" uvicorn = {extras = ["standard"], version = "0.29.0"} +ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0.0"} google-auth = "2.29.0" httpx = "0.27.0" python-dotenv = "1.0.1" orjson = "3.10.3" -[tool.poetry.group.ibis.dependencies] -ibis-framework = {extras = ["bigquery", "postgres", "snowflake"], version = "9.0.0"} - [tool.poetry.group.dev.dependencies] pytest = "8.2.0" From 822b242bbf60ec12789f6c3669399befed36c8a0 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 12:07:14 +0800 Subject: [PATCH 05/17] Use docker/build-push-action --- .github/workflows/build-dev-image.yml | 24 +++++++++++++++--------- .github/workflows/build-image.yml | 15 ++++++++------- .github/workflows/stable-release.yml | 19 ++++++++++--------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 772d623b3..1aecde3fb 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -54,6 +54,10 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 + - id: prepare_tag + run: | + LATEST_COMMIT=$(git log -1 --pretty=%h) + echo "tag_name=main-${LATEST_COMMIT}" >> $GITHUB_ENV - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -64,12 +68,14 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and Push Docker image - run: | - LATEST_COMMIT=$(git log -1 --pretty=%h) - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine-ibis:${LATEST_COMMIT} \ - --tag ghcr.io/canner/wren-engine-ibis:nightly \ - --file Dockerfile \ - --push . + - name: Build and push + uses: docker/build-push-action@v5 + env: + TAG_NAME: ${{ steps.prepare_tag.outputs.tag_name }} + with: + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/canner/wren-engine-ibis:${TAG_NAME} + ghcr.io/canner/wren-engine-ibis:nightly diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 3b315aa4d..3061a1563 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -78,12 +78,13 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and Push Docker image + - name: Build and push + uses: docker/build-push-action@v5 env: TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ - --file Dockerfile \ - --push . + with: + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ No newline at end of file diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index eea7a80f4..5faa4c332 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -117,13 +117,14 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and Push Docker image + - name: Build and push + uses: docker/build-push-action@v5 env: - TAG_NAME: ${{ steps.prepare-ibis-version.outputs.next_version }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - --tag ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ - --tag ghcr.io/canner/wren-engine-ibis:latest \ - --file Dockerfile \ - --push . + TAG_NAME: ${{ needs.prepare-ibis-version.outputs.next_version }} + with: + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/canner/wren-engine-ibis:${TAG_NAME} + ghcr.io/canner/wren-engine-ibis:latest \ No newline at end of file From 7fff6a6d27d93e7cb93c71ab7c06c8216e85af8f Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 14:48:40 +0800 Subject: [PATCH 06/17] Fix tag name --- .github/workflows/build-dev-image.yml | 4 ++-- .github/workflows/build-image.yml | 2 +- .github/workflows/stable-release.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 1aecde3fb..8b23e50da 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -57,7 +57,7 @@ jobs: - id: prepare_tag run: | LATEST_COMMIT=$(git log -1 --pretty=%h) - echo "tag_name=main-${LATEST_COMMIT}" >> $GITHUB_ENV + echo "tag_name=${LATEST_COMMIT}" >> $GITHUB_ENV - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -77,5 +77,5 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/canner/wren-engine-ibis:${TAG_NAME} + ghcr.io/canner/wren-engine-ibis:$TAG_NAME ghcr.io/canner/wren-engine-ibis:nightly diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 3061a1563..312a9f3fe 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -87,4 +87,4 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/canner/wren-engine-ibis:${TAG_NAME} \ No newline at end of file + ghcr.io/canner/wren-engine-ibis:$TAG_NAME \ No newline at end of file diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 5faa4c332..72c15eba0 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -126,5 +126,5 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/canner/wren-engine-ibis:${TAG_NAME} + ghcr.io/canner/wren-engine-ibis:$TAG_NAME ghcr.io/canner/wren-engine-ibis:latest \ No newline at end of file From 59ecc1bd5fe3992824ca23e27cb15b3d36cc965b Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 15:23:56 +0800 Subject: [PATCH 07/17] Add docker/metadata-action --- .github/workflows/build-dev-image.yml | 18 +++++++++--------- .github/workflows/build-image.yml | 12 ++++++++---- .github/workflows/stable-release.yml | 14 +++++++++----- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 8b23e50da..47f13144e 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -54,10 +54,14 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - - id: prepare_tag - run: | - LATEST_COMMIT=$(git log -1 --pretty=%h) - echo "tag_name=${LATEST_COMMIT}" >> $GITHUB_ENV + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine-ibis + tags: | + type=sha + type=schedule,pattern=nightly - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -70,12 +74,8 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build and push uses: docker/build-push-action@v5 - env: - TAG_NAME: ${{ steps.prepare_tag.outputs.tag_name }} with: file: ./Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ghcr.io/canner/wren-engine-ibis:$TAG_NAME - ghcr.io/canner/wren-engine-ibis:nightly + tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 312a9f3fe..bedfe0954 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -68,6 +68,13 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine-ibis + tags: | + type=raw,value=${{ needs.prepare-tag.outputs.tag_name }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -80,11 +87,8 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build and push uses: docker/build-push-action@v5 - env: - TAG_NAME: ${{ needs.prepare-tag.outputs.tag_name }} with: file: ./Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ghcr.io/canner/wren-engine-ibis:$TAG_NAME \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 72c15eba0..640107a1b 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -107,6 +107,14 @@ jobs: working-directory: ./ibis-server steps: - uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/canner/wren-engine-ibis + tags: | + type=raw,value=${{ needs.prepare-ibis-version.outputs.next_version }} + type=raw,value=latest - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -119,12 +127,8 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Build and push uses: docker/build-push-action@v5 - env: - TAG_NAME: ${{ needs.prepare-ibis-version.outputs.next_version }} with: file: ./Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ghcr.io/canner/wren-engine-ibis:$TAG_NAME - ghcr.io/canner/wren-engine-ibis:latest \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file From 6c9f8beceade02d081beed9abfe0a1627c0f64c5 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 15:26:38 +0800 Subject: [PATCH 08/17] Fix Dockerfile path --- .github/workflows/build-dev-image.yml | 5 +---- .github/workflows/build-image.yml | 5 +---- .github/workflows/stable-release.yml | 5 +---- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 47f13144e..00b3916e5 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -49,9 +49,6 @@ jobs: --build-arg "WREN_VERSION=${WREN_VERSION}" . build-ibis-image: runs-on: ubuntu-latest - defaults: - run: - working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - name: Docker meta @@ -75,7 +72,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./Dockerfile + file: ./ibis-server/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index bedfe0954..49bc6a006 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -63,9 +63,6 @@ jobs: build-ibis-image: needs: prepare-tag runs-on: ubuntu-latest - defaults: - run: - working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - name: Docker meta @@ -88,7 +85,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./Dockerfile + file: ./ibis-server/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 640107a1b..5092fed3a 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -102,9 +102,6 @@ jobs: stable-release-ibis: needs: prepare-ibis-version runs-on: ubuntu-latest - defaults: - run: - working-directory: ./ibis-server steps: - uses: actions/checkout@v4 - name: Docker meta @@ -128,7 +125,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./Dockerfile + file: ./ibis-server/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file From 7b8a6b2def2c058cd8803fe1d46dcfca07c05d70 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 15:28:02 +0800 Subject: [PATCH 09/17] Fix context --- .github/workflows/build-dev-image.yml | 2 +- .github/workflows/build-image.yml | 2 +- .github/workflows/stable-release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 00b3916e5..c1551c891 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -72,7 +72,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./ibis-server/Dockerfile + context: ./ibis-server platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 49bc6a006..cdab76eb1 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -85,7 +85,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./ibis-server/Dockerfile + context: ./ibis-server platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 5092fed3a..09840fdec 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -125,7 +125,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - file: ./ibis-server/Dockerfile + context: ./ibis-server platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file From 55a33d0d6b9a8ca7e34e28335d5c2c429ab45dec Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 15:55:39 +0800 Subject: [PATCH 10/17] Fix tag --- .github/workflows/build-dev-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index c1551c891..bdaa6e97b 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -58,7 +58,7 @@ jobs: images: ghcr.io/canner/wren-engine-ibis tags: | type=sha - type=schedule,pattern=nightly + type=raw,value=nightly - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: From 639b7e3869664f00bc0505e2563ae1e7606b2a09 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 17:27:56 +0800 Subject: [PATCH 11/17] Update README --- ibis-server/README.md | 47 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/ibis-server/README.md b/ibis-server/README.md index 0235c3f5c..f9784b268 100644 --- a/ibis-server/README.md +++ b/ibis-server/README.md @@ -1,20 +1,59 @@ # Ibis server -## Environment Setup +## Quick Start +### Running on Docker +Pull the image from the GitHub Container Registry +```bash +docker pull ghcr.io/canner/wren-engine-ibis:latest +``` +Create `.env` file and fill in the environment variables (see [Environment Variables](#Environment-Variables)) \ +```bash +vim .env +``` +Run the container +```bash +docker run --env-file .env -p 8000:8000 ghcr.io/canner/wren-engine-ibis:latest +``` +### Running on Local +Requirements: +- Python 3.11 +- [poetry](https://github.com/python-poetry/poetry) 1.7.1 + +Clone the repository and navigate to the ibis directory +```bash +git clone git@github.com:Canner/wren-engine.git +cd ibis-server +``` +Create `.env` file and fill in the environment variables (see [Environment Variables](#Environment-Variables)) +```bash +vim .env +``` +Install the dependencies +```bash +make install +``` +Run the server +```bash +make run +``` + +## Contributing + +### Environment Setup - Python 3.11 - Install `poetry` with version 1.7.1: `curl -sSL https://install.python-poetry.org | python3 - --version 1.7.1` - Execute `make install` to install the dependencies - Execute `make test` to run the tests - Create `.env` file and fill in the environment variables -## Environment Variables +### Environment Variables - `WREN_ENGINE_ENDPOINT`: The endpoint of the Wren engine -## Start the server +### Start the server - Execute `make run` to start the server - Execute `make dev` to start the server in development mode. It will auto-reload after the code is edited. - Default port is `8000`, you can change it by `make run PORT=8001` or `make dev PORT=8001` -## Docker +### Docker - Build the image: `make docker-build` - Run the container: `make docker-run` \ No newline at end of file From 2ddf7d1863ee71e0cbe4e94d8f4f3d93147f3add Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 17:33:25 +0800 Subject: [PATCH 12/17] Remove invalid default working-directory --- .github/workflows/stable-release.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 09840fdec..0a7efdd05 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -64,9 +64,6 @@ jobs: --build-arg "WREN_VERSION=${WREN_VERSION}" . prepare-ibis-version: runs-on: ubuntu-latest - defaults: - run: - working-directory: ./ibis-server steps: - uses: actions/checkout@v4 with: @@ -77,7 +74,7 @@ jobs: git config --global user.name "stable-release-bot" - uses: actions/setup-python@v5 with: - python-version-file: pyproject.toml + python-version-file: ./ibis-server/pyproject.toml - uses: abatilo/actions-poetry@v3 with: poetry-version: 1.7.1 From 003da24166e13b9195f8cb09f38ba6f065def921 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 17:39:54 +0800 Subject: [PATCH 13/17] Fix pyproject.toml path --- .github/workflows/stable-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 0a7efdd05..e22a40ef8 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -84,13 +84,13 @@ jobs: if [ -n "${{ github.event.inputs.specific_version }}" ]; then version=${{ github.event.inputs.specific_version }} else - version=$(grep -m 1 version pyproject.toml | grep -e '\d.\d.\d' -o) + version=$(grep -m 1 version ./ibis-server/pyproject.toml | grep -e '\d.\d.\d' -o) IFS=. read major minor micro <<< "${version}" micro=$((micro + 1)) version=$major.$minor.$micro fi poetry version --next-phase $version - git add pyproject.toml + git add ./ibis-server/pyproject.toml git commit -m "Upgrade ibis version to $version" git push echo "value=$version" >> $GITHUB_OUTPUT From 34eed7ef9ae116285c3ebd251d2eecdc5a37c8af Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 17:47:22 +0800 Subject: [PATCH 14/17] Use step.working-directory --- .github/workflows/stable-release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index e22a40ef8..1a1e1043e 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -80,17 +80,18 @@ jobs: poetry-version: 1.7.1 - name: Prepare next version id: next_version + working-directory: ibis-server run: | if [ -n "${{ github.event.inputs.specific_version }}" ]; then version=${{ github.event.inputs.specific_version }} else - version=$(grep -m 1 version ./ibis-server/pyproject.toml | grep -e '\d.\d.\d' -o) + version=$(grep -m 1 version pyproject.toml | grep -e '\d.\d.\d' -o) IFS=. read major minor micro <<< "${version}" micro=$((micro + 1)) version=$major.$minor.$micro fi poetry version --next-phase $version - git add ./ibis-server/pyproject.toml + git add pyproject.toml git commit -m "Upgrade ibis version to $version" git push echo "value=$version" >> $GITHUB_OUTPUT From 4bbbda0e5604570a9ebd5c908250699fded8e6b7 Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 18:23:44 +0800 Subject: [PATCH 15/17] Use poetry version --- .github/workflows/stable-release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index 1a1e1043e..ebc7beb07 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -83,17 +83,14 @@ jobs: working-directory: ibis-server run: | if [ -n "${{ github.event.inputs.specific_version }}" ]; then - version=${{ github.event.inputs.specific_version }} + poetry version --next-phase ${{ github.event.inputs.specific_version }} else - version=$(grep -m 1 version pyproject.toml | grep -e '\d.\d.\d' -o) - IFS=. read major minor micro <<< "${version}" - micro=$((micro + 1)) - version=$major.$minor.$micro + poetry version patch fi - poetry version --next-phase $version git add pyproject.toml git commit -m "Upgrade ibis version to $version" git push + version=$(poetry version | awk '{print $2}') echo "value=$version" >> $GITHUB_OUTPUT outputs: next_version: ${{ steps.next_version.outputs.value }} From a88548128316ca0a014e633e8f2f541eea2e026c Mon Sep 17 00:00:00 2001 From: Grieve Date: Fri, 24 May 2024 18:34:41 +0800 Subject: [PATCH 16/17] Add empty line in the end of file --- .github/workflows/build-image.yml | 2 +- .github/workflows/stable-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index cdab76eb1..6a50b1220 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -88,4 +88,4 @@ jobs: context: ./ibis-server platforms: linux/amd64,linux/arm64 push: true - tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/stable-release.yml b/.github/workflows/stable-release.yml index ebc7beb07..d47769743 100644 --- a/.github/workflows/stable-release.yml +++ b/.github/workflows/stable-release.yml @@ -123,4 +123,4 @@ jobs: context: ./ibis-server platforms: linux/amd64,linux/arm64 push: true - tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file + tags: ${{ steps.meta.outputs.tags }} From 88081c27e39e694b5af51cdd3d7455767b033745 Mon Sep 17 00:00:00 2001 From: Grieve Date: Tue, 28 May 2024 13:57:01 +0800 Subject: [PATCH 17/17] Edit title --- ibis-server/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibis-server/README.md b/ibis-server/README.md index f9784b268..65023fb08 100644 --- a/ibis-server/README.md +++ b/ibis-server/README.md @@ -37,7 +37,7 @@ Run the server make run ``` -## Contributing +## Developer Guide ### Environment Setup - Python 3.11