Merge pull request #213 from roboflow/feature/notebook

Built In Jupyter Notebook

Author: paulguerrie

docker/dockerfiles/Dockerfile.onnx.cpu

@@ -34,6 +34,7 @@ RUN pip3 install --upgrade pip  && pip3 install \
     -r requirements.gaze.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    jupyterlab \
     wheel>=0.38.0 \
     setuptools>=65.5.1 \
     --upgrade \
@@ -42,8 +43,15 @@ RUN pip3 install --upgrade pip  && pip3 install \
 FROM scratch
 COPY --from=base / /
 
-WORKDIR /app
+WORKDIR /build
+COPY . .
+RUN make create_wheels
+RUN pip3 install dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl
+
+WORKDIR /notebooks
+COPY examples/notebooks .
 
+WORKDIR /app
 COPY inference inference
 COPY docker/config/cpu_http.py cpu_http.py
 

docker/dockerfiles/Dockerfile.onnx.gpu

@@ -35,12 +35,22 @@ RUN pip3 install --upgrade pip  && pip3 install \
     -r requirements.groundingdino.txt \
     -r requirements.doctr.txt \
     -r requirements.cogvlm.txt \
+    jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip
 
 FROM scratch
 COPY --from=base / /
 
+WORKDIR /build
+COPY . .
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN /bin/make create_wheels
+RUN pip3 install dist/inference_core*.whl dist/inference_gpu*.whl dist/inference_sdk*.whl
+
+WORKDIR /notebooks
+COPY examples/notebooks .
+
 WORKDIR /app/
 COPY inference inference
 COPY docker/config/gpu_http.py gpu_http.py

docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0

@@ -38,6 +38,7 @@ RUN python3.8 -m pip install --upgrade pip  && python3.8 -m pip install \
     -r requirements.http.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip
 

docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1

@@ -38,6 +38,7 @@ RUN python3.8 -m pip install --upgrade pip  && python3.8 -m pip install \
     -r requirements.http.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip
 

docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1

@@ -36,6 +36,7 @@ RUN pip3 install --upgrade pip  && pip3 install \
     -r requirements.http.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip
 

examples/notebooks/inference_pipeline.ipynb

@@ -0,0 +1,124 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "83db9682-cfc4-4cd0-889f-c8747c4033b3",
+   "metadata": {},
+   "source": [
+    "# Inference Pipeline\n",
+    "\n",
+    "Inference Pipelines are a great way to process video streams with Inference. You can configure different sources that include streams from local devices, RTSP streams, and local video files. You can also configure different sinks that include UDP streaming of results, render of results, and custom callbacks to run your own logic after each new set of predictions is available. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ec4136f-53e9-4c8c-9217-a2c533d498ae",
+   "metadata": {},
+   "source": [
+    "### Roboflow API Key\n",
+    "\n",
+    "To load models with `inference`, you'll need a Roboflow API Key. Find instructions for retrieving your API key [here](https://docs.roboflow.com/api-reference/authentication). The utility function below attempts to load your Roboflow API key from your enviornment. If it isn't found, it then prompts you to input it. To avoid needing to input your API key for each example, you can configure your Roboflow API key in your environment via the variable `ROBOFLOW_API_KEY`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af3aad40-d41b-4bc1-ade8-dac052951257",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from utils import get_roboflow_api_key\n",
+    "\n",
+    "api_key = get_roboflow_api_key()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "86f3f805-f628-4e94-91ac-3b2f44bebdc0",
+   "metadata": {},
+   "source": [
+    "### Inference Pipeline Example\n",
+    "\n",
+    "In this example we create a new InferencePipeline. We pass the model ID, the video reference, and a method to render our results. Out pipeline does the rest!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "58dd049c-dcc6-4d0b-85ad-e6d1c0ba805b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from functools import partial\n",
+    "\n",
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt\n",
+    "from IPython import display\n",
+    "\n",
+    "from inference.core.interfaces.stream.inference_pipeline import InferencePipeline\n",
+    "from inference.core.interfaces.stream.sinks import render_boxes\n",
+    "\n",
+    "# Define source video\n",
+    "video_url = \"https://storage.googleapis.com/com-roboflow-marketing/football-video.mp4\"\n",
+    "\n",
+    "# Prepare to plot results\n",
+    "\n",
+    "fig, ax = plt.subplots()\n",
+    "frame_placeholder = np.zeros((480, 640, 3), dtype=np.uint8)  # Adjust the dimensions to match your frame size\n",
+    "image_display = ax.imshow(frame_placeholder)\n",
+    "\n",
+    "# Define our plotting function\n",
+    "def update_plot(new_frame):\n",
+    "    # Update the image displayed\n",
+    "    image_display.set_data(new_frame)\n",
+    "    # Redraw the canvas immediately\n",
+    "    display.display(plt.gcf())\n",
+    "    display.clear_output(wait=True)\n",
+    "\n",
+    "# Define our pipeline's sink\n",
+    "render = partial(render_boxes, on_frame_rendered=update_plot)\n",
+    "\n",
+    "# Instantiate the pipeline\n",
+    "pipeline = InferencePipeline.init(\n",
+    "    model_id=\"soccer-players-5fuqs/1\",\n",
+    "    video_reference=video_url,\n",
+    "    on_prediction=render,\n",
+    "    api_key=api_key,\n",
+    ")\n",
+    "\n",
+    "# Start the pipeline\n",
+    "pipeline.start()\n",
+    "pipeline.join()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07762936-ff33-46c0-a4a2-0a8e729053d1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/notebooks/inference_sdk.ipynb

@@ -0,0 +1,31 @@
+import getpass
+import requests
+
+import cv2
+import numpy as np
+
+from inference.core.env import API_KEY
+
+def get_roboflow_api_key():
+    if API_KEY is None:
+        api_key = getpass.getpass("Roboflow API Key:")
+    else:
+        api_key = API_KEY
+    return api_key
+
+def load_image_from_url(url):
+    # Send a GET request to the URL
+    response = requests.get(url)
+
+    # Ensure that the request was successful
+    if response.status_code == 200:
+        # Convert the response content into a numpy array
+        image_array = np.asarray(bytearray(response.content), dtype=np.uint8)
+
+        # Decode the image array into an OpenCV image
+        image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
+
+        return image
+    else:
+        print(f"Failed to retrieve the image. HTTP status code: {response.status_code}")
+        return None

examples/notebooks/quickstart.ipynb

@@ -0,0 +1,8 @@
+from pydantic import BaseModel, Field, ValidationError
+
+
+class NotebookStartResponse(BaseModel):
+    """Response model for notebook start request"""
+
+    success: str = Field(..., description="Status of the request")
+    message: str = Field(..., description="Message of the request", optional=True)

examples/notebooks/utils.py

@@ -162,7 +162,7 @@
 LICENSE_SERVER = os.getenv("LICENSE_SERVER", None)
 
 # Log level, default is "INFO"
-LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
 
 # Maximum number of active models, default is 8
 MAX_ACTIVE_MODELS = int(os.getenv("MAX_ACTIVE_MODELS", 8))
@@ -204,6 +204,15 @@
 # Model ID, default is None
 MODEL_ID = os.getenv("MODEL_ID")
 
+# Enable jupyter notebook server route, default is False
+NOTEBOOK_ENABLED = str2bool(os.getenv("NOTEBOOK_ENABLED", False))
+
+# Jupyter notebook password, default is "roboflow"
+NOTEBOOK_PASSWORD = os.getenv("NOTEBOOK_PASSWORD", "roboflow")
+
+# Jupyter notebook port, default is 9002
+NOTEBOOK_PORT = int(os.getenv("NOTEBOOK_PORT", 9002))
+
 # Number of workers, default is 1
 NUM_WORKERS = int(os.getenv("NUM_WORKERS", 1))
 

inference/core/entities/responses/notebooks.py

@@ -1,12 +1,13 @@
 import base64
 import traceback
 from functools import partial, wraps
+from time import sleep
 from typing import Any, List, Optional, Union
 
 import uvicorn
 from fastapi import BackgroundTasks, Body, FastAPI, Path, Query, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, Response
+from fastapi.responses import JSONResponse, RedirectResponse, Response
 from fastapi.staticfiles import StaticFiles
 from fastapi_cprofile.profiler import CProfileMiddleware
 
@@ -52,6 +53,7 @@
     ObjectDetectionInferenceResponse,
     StubResponse,
 )
+from inference.core.entities.responses.notebooks import NotebookStartResponse
 from inference.core.entities.responses.sam import (
     SamEmbeddingResponse,
     SamSegmentationResponse,
@@ -73,6 +75,9 @@
     LEGACY_ROUTE_ENABLED,
     METLO_KEY,
     METRICS_ENABLED,
+    NOTEBOOK_ENABLED,
+    NOTEBOOK_PASSWORD,
+    NOTEBOOK_PORT,
     PROFILE,
     ROBOFLOW_SERVICE_SECRET,
 )
@@ -101,6 +106,7 @@
 from inference.core.interfaces.base import BaseInterface
 from inference.core.interfaces.http.orjson_utils import orjson_response
 from inference.core.managers.base import ModelManager
+from inference.core.utils.notebooks import start_notebook
 
 if LAMBDA:
     from inference.core.usage import trackUsage
@@ -1200,6 +1206,45 @@ async def model_add(dataset_id: str, version_id: str, api_key: str = None):
                     }
                 )
 
+        if not LAMBDA:
+
+            @app.get(
+                "/notebook/start",
+                summary="Jupyter Lab Server Start",
+                description="Starts a jupyter lab server for running development code",
+            )
+            @with_route_exceptions
+            async def notebook_start(browserless: bool = False):
+                """Starts a jupyter lab server for running development code.
+
+                Args:
+                    inference_request (NotebookStartRequest): The request containing the necessary details for starting a jupyter lab server.
+                    background_tasks: (BackgroundTasks) pool of fastapi background tasks
+
+                Returns:
+                    NotebookStartResponse: The response containing the URL of the jupyter lab server.
+                """
+                if NOTEBOOK_ENABLED:
+                    start_notebook()
+                    if browserless:
+                        return {
+                            "success": True,
+                            "message": f"Jupyter Lab server started at http://localhost:{NOTEBOOK_PORT}?token={NOTEBOOK_PASSWORD}",
+                        }
+                    else:
+                        sleep(2)
+                        return RedirectResponse(
+                            f"http://localhost:{NOTEBOOK_PORT}/lab/tree/quickstart.ipynb?token={NOTEBOOK_PASSWORD}"
+                        )
+                else:
+                    if browserless:
+                        return {
+                            "success": False,
+                            "message": "Notebook server is not enabled. Enable notebooks via the NOTEBOOK_ENABLED environment variable.",
+                        }
+                    else:
+                        return RedirectResponse(f"/notebook-instructions.html")
+
         app.mount(
             "/",
             StaticFiles(directory="./inference/landing/out", html=True),

inference/core/env.py

@@ -29,7 +29,7 @@ def orjson_response(
     response: Union[List[InferenceResponse], InferenceResponse]
 ) -> ORJSONResponseBytes:
     if isinstance(response, list):
-        content = [r.dict(by_alias=True) for r in response]
+        content = [r.dict(by_alias=True, exclude_none=True) for r in response]
     else:
-        content = response.dict(by_alias=True)
+        content = response.dict(by_alias=True, exclude_none=True)
     return ORJSONResponseBytes(content=content)

inference/core/interfaces/http/http_api.py

@@ -1,5 +1,5 @@
 import logging
-import os
+import warnings
 
 from rich.logging import RichHandler
 
@@ -8,3 +8,6 @@
 logger = logging.getLogger("inference")
 logger.setLevel(LOG_LEVEL)
 logger.addHandler(RichHandler())
+
+if LOG_LEVEL == "ERROR" or LOG_LEVEL == "FATAL":
+    warnings.filterwarnings("ignore", category=UserWarning, module="onnxruntime.*")

inference/core/interfaces/http/orjson_utils.py

@@ -0,0 +1,24 @@
+import os
+import subprocess
+
+import requests
+
+from inference.core.env import NOTEBOOK_PASSWORD, NOTEBOOK_PORT
+
+
+def check_notebook_is_running():
+    try:
+        response = requests.get(f"http://localhost:{NOTEBOOK_PORT}/")
+        return response.status_code == 200
+    except:
+        return False
+
+
+def start_notebook():
+    if not check_notebook_is_running():
+        os.makedirs("/notebooks", exist_ok=True)
+        subprocess.Popen(
+            f"jupyter-lab --allow-root --port={NOTEBOOK_PORT} --ip=0.0.0.0 --notebook-dir=/notebooks --NotebookApp.token='{NOTEBOOK_PASSWORD}' --NotebookApp.password='{NOTEBOOK_PASSWORD}'".split(
+                " "
+            )
+        )