Skip to content

Commit

Permalink
Move fennel_data_lib into a seperate pypi
Browse files Browse the repository at this point in the history
  • Loading branch information
@aditya-nambiar
aditya-nambiar committed Aug 7, 2024
1 parent f6d0fd3 commit bbe720d
Show file tree
Hide file tree
Showing 5 changed files with 1,299 additions and 1,045 deletions.
99 changes: 99 additions & 0 deletions .github/workflows/fennel-lib-publish.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
name: CI

on:
# These tests are quite cheap, run them on every PR
pull_request:
branches: [ main ]

# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
permissions:
contents: read

jobs:
linux:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: ubuntu-latest
target: x86_64
- runner: ubuntu-latest
target: x86
- runner: ubuntu-latest
target: aarch64
- runner: ubuntu-latest
target: armv7
- runner: ubuntu-latest
target: s390x
- runner: ubuntu-latest
target: ppc64le
steps:
- uses: actions/checkout@v4
with:
repository: 'fennel-ai/server'
token: ${{ secrets.SERVER_CHECKOUT }}
path: 'server'
ref: 'main'
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.platform.target }}
args: --release --out dist --manifest-path server/fennel_data_lib/Cargo.toml --find-interpreter
sccache: 'true'
manylinux: auto
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
name: wheels-linux-${{ matrix.platform.target }}
path: dist


macos:
runs-on: ${{ matrix.platform.runner }}
strategy:
matrix:
platform:
- runner: macos-latest
target: x86_64
- runner: macos-14
target: aarch64
steps:
- uses: actions/checkout@v4
with:
repository: 'fennel-ai/server'
token: ${{ secrets.SERVER_CHECKOUT }}
path: 'server'
ref: 'main'
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.platform.target }}
args: --release --out dist --manifest-path server/fennel_data_lib/Cargo.toml --find-interpreter
sccache: 'true'
- name: Upload wheels
uses: actions/upload-artifact@v4
with:
name: wheels-macos-${{ matrix.platform.target }}
path: dist

release:
name: Release
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
needs: [linux, macos]
steps:
- uses: actions/download-artifact@v4
- name: Publish to PyPI
uses: PyO3/maturin-action@v1
env:
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PACKAGR_ACCESS_TOKEN }}
with:
command: upload
args: --non-interactive --skip-existing wheels-*/*
95 changes: 46 additions & 49 deletions .github/workflows/packagr_publish.yml
View file
Original file line number Diff line number Diff line change
@@ -1,89 +1,86 @@
name: Build, Test, and Publish
name: Packagr Publish

# Controls when the workflow will run
on:
# Triggers the workflow on push but only for the main branch
push:
branches: [ main ]

# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
build-and-publish:
# This workflow contains a single job called "build"
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
name: Build, Test, and Publish Package

# Steps represent a sequence of tasks that will be executed as part of the job
name: Run tests and publish latest version if succcessful
steps:
# Checkout the Python client repository
- name: Check out client repository (Python)
uses: actions/checkout@v2

# Checkout the Rust server repository
- name: Check out server repository (Rust)
#----------------------------------------------
# check-out repo and set-up python
#----------------------------------------------
- name: Check out repository
uses: actions/checkout@v2
with:
repository: 'fennel-ai/server'
token: ${{ secrets.SERVER_CHECKOUT }}
path: 'server'
ref: 'main'

# Set up Python environment
- name: Set up Python
- name: Set up python
id: setup-python
uses: actions/setup-python@v2
with:
python-version: 3.12

# Install Poetry
#----------------------------------------------
# ----- install & configure poetry -----
#----------------------------------------------
- name: Install Poetry
uses: snok/install-poetry@v1
with:
virtualenvs-create: true
virtualenvs-in-project: true

# Load cached Python virtual environment if it exists
#----------------------------------------------
# load cached venv if cache exists
#----------------------------------------------
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v2
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

# Install Python dependencies if the cache does not exist

#----------------------------------------------
# install dependencies if cache does not exist
#----------------------------------------------
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root

# Set up Rust toolchain
- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
default: true

# Install protobuf compiler (protoc)
- name: Install protobuf compiler (protoc)
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

# Install Maturin to build the package
- name: Install Maturin
run: python -m pip install maturin

# Build the package using Maturin
- name: Build the package using Maturin
run: maturin build --release --features "extension-module" --manifest-path ./server/fennel_data_lib/Cargo.toml --out dist

# Install the built package
- name: Install built package
run: |
source .venv/bin/activate
python -m pip install dist/*.whl
python -c "import fennel_data_lib"
#----------------------------------------------
# install your root project, if required
#----------------------------------------------
- name: Install library
run: poetry install --no-interaction

# Run Python tests
# Run test suite
- name: Run tests
run: |
source .venv/bin/activate
pytest fennel --run_slow
poetry run pytest . --run_slow
poetry run python -m unittest discover fennel
# Publish the package to PyPI if all tests pass
- name: Publish to PyPI
#----------------------------------------------
# publish to pypi - this is run only if the previous steps are successful
#----------------------------------------------
- name: Publish to Pypi
run: |
rm -rf dist/*
pip install setuptools wheel twine
poetry build
twine upload --repository pypi dist/* --verbose -u __token__ -p ${{ secrets.PYPI_PACKAGR_ACCESS_TOKEN }}


46 changes: 2 additions & 44 deletions .github/workflows/test_linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ jobs:
# The type of runner that the job will run on
runs-on: ubuntu-latest



# Steps represent a sequence of tasks that will be executed as part of the job
name: Unit tests with py-version ${{ matrix.python-version }}
steps:
Expand All @@ -38,27 +36,6 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}

#----------------------------------------------
# check-out server repo
#----------------------------------------------
- name: Check out server repository (Rust)
uses: actions/checkout@v2
with:
repository: 'fennel-ai/server'
token: ${{ secrets.SERVER_CHECKOUT }}
path: 'server'
ref: 'main'

- name: Set up Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
default: true

- name: Install protobuf compiler (protoc)
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler

#----------------------------------------------
# ----- install & configure poetry -----
#----------------------------------------------
Expand All @@ -82,24 +59,6 @@ jobs:
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root

#----------------------------------------------
# install maturin
#----------------------------------------------
- name: Install maturin
run: python -m pip install maturin

# Build the package using Maturin
- name: Build the package using Maturin
run: maturin build --release --features "extension-module" --manifest-path ./server/fennel_data_lib/Cargo.toml --out dist

# Install the built package
- name: Install built package
run: |
source .venv/bin/activate
python -m pip install dist/*.whl
python -c "import fennel_data_lib"
#----------------------------------------------
# install your root project, if required
#----------------------------------------------
Expand All @@ -110,6 +69,5 @@ jobs:
- name: Run tests
run: |
source .venv/bin/activate
pytest fennel --run_slow
pytest docs
poetry run python -m unittest discover fennel
poetry run pytest . --run_slow
poetry run python -m unittest discover fennel
Loading

0 comments on commit bbe720d

Please sign in to comment.