ci: native binaries (#382)

* add pyinstaller * pyinstaller compilation smoke test * pyinstaller: move to dev deps, add py version * add --hidden-import jinja2_ansible_filters * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * docs: wip adr * chore: test * docs: added some points about Nuitka * chore: testing ci * chore: testing ci * chore: testing ci * chore: adding multiple os runs for build * chore: improving build * chore: fixing tests; removing setting from vscode clashing with ruff * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * chore: testing ci * cI: refining ci/cd pipelines; dumping binaries to (pre)release * chore: adding new poetry action to cd * chore: removing old adr draft * chore: addressing pr comments * chore: patching tealer --------- Co-authored-by: Joe Polny <joepolny@gmail.com> Co-authored-by: Negar Abbasi <negar.abbasi@makerx.com.au>
algorandfoundation · Jan 29, 2024 · daa9fc5 · daa9fc5 · github-actions · Jan 29, 2024
1 parent 3b0f258
commit daa9fc5
Show file tree

Hide file tree

Showing 22 changed files with 4,412 additions and 191 deletions.
diff --git a/.github/actions/build-binaries/action.yaml b/.github/actions/build-binaries/action.yaml
@@ -0,0 +1,67 @@
+name: "Setup, Build, and Test"
+description: "Set up Python with Poetry, build and test binaries"
+inputs:
+  package_name:
+    description: "The name of the package to build and test"
+    required: true
+  upload_binaries:
+    description: "Flag to determine if this is a production release"
+    required: true
+  operating_system:
+    description: "Operating system to set the correct binary path and extension"
+    required: true
+  build_command:
+    description: "Command to build the binaries"
+    required: true
+  python_version:
+    description: "Python version to use"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build Executable
+      run: ${{ inputs.build_command }}
+      shell: bash
+
+    - name: Test Executable
+      run: |
+        ls -l dist
+        ./dist/${{ inputs.package_name }}/${{ inputs.package_name }}${{ inputs.operating_system == 'windows-latest' && '.exe' || '' }} --help
+      shell: bash
+
+    - name: Set release version
+      shell: bash
+      continue-on-error: true
+      if: ${{ inputs.upload_binaries == 'true' }}
+      run: |
+        echo "RELEASE_VERSION=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_ENV
+        git describe --tags $(git rev-list --tags --max-count=1)
+
+    - name: Zip binaries
+      shell: bash
+      continue-on-error: true
+      if: ${{ inputs.upload_binaries == 'true' }}
+      run: |
+        ls -l dist
+        cd dist/algokit/
+        tar -zcvf ../../algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz *
+        cd ../..
+        ls -l
+
+    - name: Upload binary as artifact
+      if: ${{ inputs.upload_binaries == 'true' }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: algokit-cli-${{ inputs.operating_system }}-py${{ inputs.python_version }}
+        path: algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz
+
+    - name: Append binary to release
+      continue-on-error: true
+      if: ${{ inputs.upload_binaries == 'true' }}
+      uses: softprops/action-gh-release@v1
+      with:
+        files: |
+          algokit-${{ env.RELEASE_VERSION }}-${{ inputs.operating_system }}-py${{ inputs.python_version }}.tar.gz
+        tag_name: ${{ env.RELEASE_VERSION }}
+        prerelease: ${{ contains(env.RELEASE_VERSION, 'beta') }}
diff --git a/.github/actions/setup-poetry/action.yaml b/.github/actions/setup-poetry/action.yaml
@@ -0,0 +1,22 @@
+name: "Python Poetry Action"
+description: "An action to setup Poetry"
+inputs:
+  poetry-version:
+    description: "The version of poetry to install"
+    required: false
+    default: "latest"
+runs:
+  using: "composite"
+  steps:
+    - run: |
+        pip install --user pipx
+        pipx ensurepath
+      shell: bash
+    - if: ${{ inputs.poetry-version == 'latest' }}
+      run: |
+        pipx install poetry
+      shell: bash
+    - if: ${{ inputs.poetry-version != 'latest' }}
+      run: |
+        pipx install poetry==${{ inputs.poetry-version }}
+      shell: bash
diff --git a/.github/workflows/build-binaries.yaml b/.github/workflows/build-binaries.yaml
@@ -0,0 +1,135 @@
+name: Build, Test and Publish Pyinstaller Binaries
+
+on:
+  workflow_call:
+    inputs:
+      upload_binaries:
+        required: true
+        type: string
+      python_version:
+        required: true
+        type: string
+
+jobs:
+  build-binaries-ubuntu:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source code (for release)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries == 'true' }}
+        with:
+          fetch-depth: 0
+
+      - name: Checkout source code (for build)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries != 'true' }}
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python_version }}
+
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Build linux binary
+        uses: ./.github/actions/build-binaries
+        with:
+          python_version: ${{ inputs.python_version }}
+          package_name: "algokit"
+          upload_binaries: ${{ inputs.upload_binaries }}
+          operating_system: ${{ runner.os }}
+          build_command: "poetry run poe package_unix"
+
+  build-binaries-windows:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout source code (for release)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries == 'true' }}
+        with:
+          fetch-depth: 0
+
+      - name: Checkout source code (for build)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries != 'true' }}
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python_version }}
+
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Build windows binary
+        uses: ./.github/actions/build-binaries
+        with:
+          python_version: ${{ inputs.python_version }}
+          package_name: "algokit"
+          upload_binaries: ${{ inputs.upload_binaries }}
+          operating_system: ${{ runner.os }}
+          build_command: "poetry run poe package_windows"
+
+  build-binaries-macos:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout source code (for release)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries == 'true' }}
+        with:
+          fetch-depth: 0
+
+      - name: Checkout source code (for build)
+        uses: actions/checkout@v4
+        if: ${{ inputs.upload_binaries != 'true' }}
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python_version }}
+
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-${{ inputs.python_version }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Build macos binary
+        uses: ./.github/actions/build-binaries
+        with:
+          python_version: ${{ inputs.python_version }}
+          package_name: "algokit"
+          upload_binaries: ${{ inputs.upload_binaries }}
+          operating_system: ${{ runner.os }}
+          build_command: "poetry run poe package_unix"
diff --git a/.github/workflows/build-python.yaml b/.github/workflows/build-python.yaml
@@ -6,35 +6,49 @@ jobs:
   build-python:
     strategy:
       matrix:
-        #os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Mac and Windows chew through build minutes - waiting until repo is public to enable
-        os: ["ubuntu-latest", "windows-latest"]
-        python: ["3.10"]
+        python: ["3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry
-
-      - name: Set up Python ${{ matrix.python }}
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-          cache: "poetry"
+
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ matrix.os }}-${{ matrix.python }}
 
       - name: Install dependencies
-        run: poetry install --no-interaction && pipx install tealer==0.1.1
+        # TODO: remove fixed pipx dependency once 3.12 compatibility is addressed
+        # track here -> https://github.com/crytic/tealer/pull/209
+        run: poetry install --no-interaction && pipx install git+https://github.com/algorandfoundation/tealer@3-12
 
       - name: pytest + coverage
         shell: bash
         run: |
           set -o pipefail
           poetry run pytest -n auto --junitxml=pytest-junit.xml --cov-report=term-missing:skip-covered --cov=src | tee pytest-coverage.txt
+        id: pytest
+
+      - name: Upload received snapshots (in case of failure)
+        if: failure() && steps.pytest.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-artifacts-${{ matrix.os }}-python${{ matrix.python }}
+          path: tests/**/*.received.txt
 
-      - name: pytest coverage comment - using Python 3.10 on ubuntu-latest
-        if: matrix.python == '3.10' && matrix.os == 'ubuntu-latest'
+      - name: pytest coverage comment - using Python 3.12 on ubuntu-latest
+        if: matrix.python == '3.12' && matrix.os == 'ubuntu-latest'
         continue-on-error: true # forks fail to add a comment, so continue any way
         uses: MishaKav/pytest-coverage-comment@main
         with:

diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
@@ -38,17 +38,19 @@ jobs:
           # Fetch entire repository history so we can determine version number from it
           fetch-depth: 0
 
-      - name: Install poetry
-        run: pipx install poetry
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.10"
-          cache: "poetry"
 
-      - name: Install dependencies
-        run: poetry install --no-interaction --no-root
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-3.10
 
       - name: Get branch name
         shell: bash
@@ -112,6 +114,15 @@ jobs:
           path: dist/algokit*-py3-none-any.whl
           if-no-files-found: error
 
+  upload-binaries:
+    name: Build and Upload Binaries
+    if: ${{ github.ref_name == 'main' }}
+    uses: ./.github/workflows/build-binaries.yaml
+    needs: release
+    with:
+      upload_binaries: "true"
+      python_version: "3.12"
+
   cd-publish-release-packages:
     name: Publish Release Packages
     needs: release

diff --git a/.github/workflows/check-python.yaml b/.github/workflows/check-python.yaml
@@ -8,16 +8,21 @@ jobs:
     runs-on: "ubuntu-latest"
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry
-
-      - name: Set up Python 3.10
+      - name: Set up Python 3.12
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
-          cache: "poetry"
+          python-version: "3.12"
+
+      - name: Set up Poetry
+        uses: ./.github/actions/setup-poetry
+
+      - uses: actions/cache@v4
+        name: Setup poetry cache
+        with:
+          path: ./.venv
+          key: venv-${{ hashFiles('poetry.lock') }}-${{ runner.os }}-3.12
 
       - name: Install dependencies
         run: poetry install --no-interaction

diff --git a/.github/workflows/issue_closed.yml b/.github/workflows/issue_closed.yml
diff --git a/.github/workflows/issue_commented.yml b/.github/workflows/issue_commented.yml
diff --git a/.github/workflows/issue_labelled.yml b/.github/workflows/issue_labelled.yml
File	Stmts	Miss	Cover	Missing
src/algokit
__init__.py	15	7	53%	6–13, 17–24, 32–34
__main__.py	2	2	0%	1–3
src/algokit/cli
completions.py	108	2	98%	83, 98
deploy.py	72	7	90%	44, 46, 92–94, 158, 182
dispenser.py	121	1	99%	77
doctor.py	48	3	94%	142–144
explore.py	50	12	76%	34–39, 41–46
generate.py	67	3	96%	74–75, 140
goal.py	44	1	98%	71
init.py	190	16	92%	277–278, 328, 331–333, 344, 388, 414, 454, 463–465, 468–473, 486
localnet.py	119	15	87%	74–78, 111, 123, 138–148, 161, 206, 227–228
task.py	34	3	91%	25–28
src/algokit/cli/common
utils.py	26	2	92%	120, 123
src/algokit/cli/tasks
analyze.py	81	1	99%	81
assets.py	82	13	84%	65–66, 72, 74–75, 105, 119, 125–126, 132, 134, 136–137
ipfs.py	51	8	84%	52, 80, 92, 94–95, 105–107
mint.py	66	4	94%	48, 70, 91, 250
send_transaction.py	65	10	85%	52–53, 57, 89, 158, 170–174
sign_transaction.py	59	8	86%	21, 28–30, 71–72, 109, 123
transfer.py	39	3	92%	26, 90, 117
utils.py	99	45	55%	26–34, 40–43, 75–76, 100–101, 125–133, 152–162, 209, 258–259, 279–290, 297–299
vanity_address.py	56	10	82%	41, 45–48, 112, 114, 121–123
wallet.py	79	4	95%	21, 66, 136, 162
src/algokit/core
bootstrap.py	117	10	91%	101–102, 124, 151, 180–185
conf.py	54	8	85%	12, 24, 28, 36, 38, 71–73
deploy.py	69	11	84%	61–64, 73–75, 79, 84, 91–93
dispenser.py	202	26	87%	91, 123–124, 141–149, 191–192, 198–200, 218–219, 259–260, 318, 332–334, 345–346, 356, 369, 384
doctor.py	65	7	89%	67–69, 92–94, 134
generate.py	41	2	95%	69, 87
goal.py	60	3	95%	30–31, 41
log_handlers.py	68	7	90%	50–51, 63, 112–116, 125
proc.py	45	1	98%	98
sandbox.py	218	18	92%	62, 73–75, 96, 142–149, 160, 456, 472, 497, 505
typed_client_generation.py	80	5	94%	55–57, 70, 75
utils.py	104	31	70%	44–45, 49–68, 129, 132, 138–152
version_prompt.py	72	8	89%	26–27, 39, 58–61, 79, 108
src/algokit/core/tasks
analyze.py	93	3	97%	105–112, 187
ipfs.py	63	7	89%	58–64, 140, 144, 146, 152
nfd.py	49	13	73%	25, 31, 34–41, 70–72, 99–101
vanity_address.py	90	34	62%	49–50, 54, 59–75, 92–108, 128–131
wallet.py	71	5	93%	37, 129, 155–157
src/algokit/core/tasks/mint
mint.py	78	10	87%	123–133, 187
models.py	90	11	88%	50, 52, 57, 71–74, 85–88
TOTAL	3472	400	88%