feat: Adds multiple builds and update API (#286)

* ci: Updates CI builds * refactor: Removes package docker * chore: Updates API build * feat: Updates API * fix: Fixes API * refactor: Renamed variables * fix: Fixes volume bind * ci: Fixes API docker check * feat: No need to copy with volume bind
frgfm · Oct 20, 2023 · b6a4670 · b6a4670
1 parent 3da8d3d
commit b6a4670
Show file tree

Hide file tree

Showing 19 changed files with 888 additions and 549 deletions.
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -6,11 +6,11 @@
 # {% set urls = pyproject.get('project', {}).get('urls') %}
 package:
   name: pylocron
-  version: "{{ environ.get('BUILD_VERSION') }}"
+  version: "{{ environ.get('BUILD_VERSION', '0.2.2.dev0') }}"
 
 source:
-  fn: pylocron-{{ environ.get('BUILD_VERSION') }}.tar.gz
-  url: ../dist/pylocron-{{ environ.get('BUILD_VERSION') }}.tar.gz
+  fn: pylocron-{{ environ.get('BUILD_VERSION', '0.2.2.dev0') }}.tar.gz
+  url: ../dist/pylocron-{{ environ.get('BUILD_VERSION', '0.2.2.dev0') }}.tar.gz
 
 build:
   number: 0

diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
@@ -7,13 +7,13 @@ on:
     branches: main
 
 jobs:
-  build:
+  package:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: [3.8, 3.9]
+        python: [3.8, 3.9, '3.10', 3.11]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
@@ -32,3 +32,73 @@ jobs:
           pip install -e . --upgrade
       - name: Import package
         run: python -c "import holocron; print(holocron.__version__)"
+
+  pypi:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          architecture: x64
+      - name: Cache python modules
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-python-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}-build
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine --upgrade
+      - run: |
+          python setup.py sdist bdist_wheel
+          twine check dist/*
+
+  conda:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: "3.9"
+      - name: Install dependencies
+        shell: bash -el {0}
+        run: conda install -y conda-build conda-verify
+      - name: Build conda
+        shell: bash -el {0}
+        run: |
+          python setup.py sdist
+          mkdir conda-dist
+          conda env list
+          conda-build .conda/ -c pytorch -c fastai -c conda-forge --output-folder conda-dist
+          ls -l conda-dist/noarch/*tar.bz2
+
+  api:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: 1.6.1
+      - name: Lock the requirements
+        run: |
+          cd api
+          poetry lock
+          poetry export -f requirements.txt --without-hashes --output requirements.txt
+      - name: Build & run docker
+        run: cd api && docker-compose up -d --build
+      - name: Docker sanity check
+        run: sleep 15 && nc -vz localhost 8050
+      - name: Ping server
+        run: curl http://api.localhost:8050/docs
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -4,7 +4,7 @@ on:
     branches: main
 
 jobs:
-  docs-deploy:
+  github-pages:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:

diff --git a/.github/workflows/release.yml → .github/workflows/publish.yml b/.github/workflows/release.yml → .github/workflows/publish.yml
@@ -1,17 +1,16 @@
-name: release
+name: publish
 
 on:
   release:
     types: [published]
 
 jobs:
-  pypi-publish:
+  pypi:
     if: "!github.event.release.prerelease"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
           architecture: x64
@@ -24,16 +23,12 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install setuptools wheel twine --upgrade
-      - name: Get release tag
-        id: release_tag
-        run: |
-          echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//:1}
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-          BUILD_VERSION: ${{ steps.release_tag.outputs.VERSION }}
         run: |
+          echo "BUILD_VERSION=${GITHUB_REF#refs/*/}" | cut -c 2- >> $GITHUB_ENV
           python setup.py sdist bdist_wheel
           twine check dist/*
           twine upload dist/*
@@ -55,7 +50,7 @@ jobs:
           pip install pylocron
           python -c "import holocron; print(holocron.__version__)"
 
-  conda-publish:
+  conda:
     if: "!github.event.release.prerelease"
     runs-on: ubuntu-latest
     steps:
@@ -66,16 +61,14 @@ jobs:
           auto-update-conda: true
           python-version: 3.9
       - name: Install dependencies
+        shell: bash -el {0}
         run: conda install -y conda-build conda-verify anaconda-client
-      - name: Get release tag
-        id: release_tag
-        run: |
-          echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//:1}
       - name: Build and publish
+        shell: bash -el {0}
         env:
           ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
-          BUILD_VERSION: ${{ steps.release_tag.outputs.VERSION }}
         run: |
+          echo "BUILD_VERSION=${GITHUB_REF#refs/*/}" | cut -c 2- >> $GITHUB_ENV
           python setup.py sdist
           mkdir conda-dist
           conda-build .conda/ -c pytorch -c fastai -c conda-forge --output-folder conda-dist

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -0,0 +1,28 @@
+name: push
+on:
+  push:
+    branches: main
+
+jobs:
+  dockerhub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          architecture: x64
+      - uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: "1.6.1"
+      - name: Resolve dependencies
+        run: cd api && poetry export -f requirements.txt --without-hashes --output api/requirements.txt
+      - name: Build docker
+        run: cd api && docker build src/. -t frgfm/holocron:latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: frgfm
+          password: ${{ secrets.DOCKERHUB_PW }}
+      - name: Push to hub
+        run: docker push frgfm/holocron:latest
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -51,6 +51,36 @@ jobs:
         with:
           flags: unittests
           fail_ci_if_error: true
+
+  api:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Install poetry
+        uses: abatilo/actions-poetry@v2
+        with:
+          poetry-version: 1.6.1
+      - name: Lock the requirements
+        run: |
+          cd api
+          poetry lock
+          poetry export -f requirements.txt --without-hashes --with dev --output requirements.txt
+      - name: Build & run docker
+        run: cd api && docker-compose up -d --build
+      - name: Docker sanity check
+        run: sleep 15 && nc -vz localhost 8050
+      - name: Ping server
+        run: curl http://api.localhost:8050/docs
+      - name: Run docker test
+        run: docker compose -f api/docker-compose.yml exec -T backend pytest tests/
 
   headers:
     runs-on: ${{ matrix.os }}

diff --git a/Dockerfile b/Dockerfile
diff --git a/Makefile b/Makefile
@@ -25,7 +25,3 @@ full-docs:
 # Run the Gradio demo
 run-demo:
 	python demo/app.py --port 8080
-
-# Build the docker
-docker:
-	docker build . -t frgfm/holocron:python3.8-slim
diff --git a/api/Makefile b/api/Makefile
@@ -2,22 +2,22 @@
 lock:
 	poetry lock
 
+build:
+	poetry export -f requirements.txt --without-hashes --output requirements.txt
+	docker build . -t frgfm/holocron:python3.9-slim
+
 # Run the docker
 run:
 	poetry export -f requirements.txt --without-hashes --output requirements.txt
-	docker-compose up -d --build
+	docker compose up -d --build
 
 # Run the docker
 stop:
-	docker-compose down
+	docker compose down
 
 # Run tests for the library
 test:
-	poetry export -f requirements.txt --without-hashes --output requirements.txt
-	docker-compose up -d --build
-	poetry export -f requirements.txt --without-hashes --dev --output requirements-dev.txt
-	docker cp requirements-dev.txt api_backend_1:/app/requirements-dev.txt
-	docker-compose exec -T backend pip install -r requirements-dev.txt
-	docker cp tests api_backend_1:/app/tests
-	docker-compose exec -T backend pytest tests/
-	docker-compose down
+	poetry export -f requirements.txt --without-hashes --with dev --output requirements.txt
+	docker compose up -d --build
+	docker compose exec -T backend pytest tests/ --cov=app
+	docker compose down
diff --git a/api/README.md b/api/README.md
@@ -22,7 +22,7 @@ Once completed, your [FastAPI](https://fastapi.tiangolo.com/) server should be r
 
 ### Documentation and swagger
 
-FastAPI comes with many advantages including speed and OpenAPI features. For instance, once your server is running, you can access the automatically built documentation and swagger in your browser at: http://localhost:8080/docs
+FastAPI comes with many advantages including speed and OpenAPI features. For instance, once your server is running, you can access the automatically built documentation and swagger in your browser at: http://api.localhost:8050/docs
 
 
 ### Using the routes
@@ -40,10 +40,10 @@ with this snippet:
 import requests
 with open('/path/to/your/img.jpg', 'rb') as f:
     data = f.read()
-print(requests.post("http://localhost:8080/classification", files={'file': data}).json())
+print(requests.post("http://api.localhost:8050/classification", files={'file': data}).json())
 ```
 
 should yield
 ```
-{'value': 'French horn', 'confidence': 0.9186984300613403}
+{'value': 'French horn', 'confidence': 0.9685316681861877}
 ```