Feat (notebook): add example for dynamic quantization to ONNX export (#…

…877)
Xilinx · Mar 6, 2024 · 4e82c7b · 4e82c7b
1 parent 2fdcb42
commit 4e82c7b
Showing 1 changed file with 102 additions and 10 deletions.
diff --git a/notebooks/ONNX_export_tutorial.ipynb b/notebooks/ONNX_export_tutorial.ipynb
@@ -213,7 +213,7 @@
  " "
  ],
  "text/plain": [
- "<IPython.lib.display.IFrame at 0x7f2a6afb6f50>"
+ "<IPython.lib.display.IFrame at 0x7fb62ae3fe50>"
  ]
  },
  "execution_count": 4,
@@ -331,7 +331,7 @@
  " "
  ],
  "text/plain": [
- "<IPython.lib.display.IFrame at 0x7f2a6afa9e90>"
+ "<IPython.lib.display.IFrame at 0x7fb734383710>"
  ]
  },
  "execution_count": 6,
@@ -460,7 +460,7 @@
  " "
  ],
  "text/plain": [
- "<IPython.lib.display.IFrame at 0x7f2a69d7ff10>"
+ "<IPython.lib.display.IFrame at 0x7fb629e8a010>"
  ]
  },
  "execution_count": 8,
@@ -605,7 +605,7 @@
  " "
  ],
  "text/plain": [
- "<IPython.lib.display.IFrame at 0x7f2a69d2e710>"
+ "<IPython.lib.display.IFrame at 0x7fb62ae37190>"
  ]
  },
  "execution_count": 10,
@@ -704,7 +704,7 @@
  "name": "stderr",
  "output_type": "stream",
  "text": [
- "2024-03-01 03:24:07.215804006 [W:onnxruntime:, graph.cc:1283 Graph] Initializer linear.bias appears in graph inputs and will not be treated as constant value/weight. This may prevent some of the graph optimizations, like const folding. Move it out of graph inputs if there is no need to override it, by either re-generating the model with latest exporter/converter or with the tool onnxruntime/tools/python/remove_initializer_from_input.py.\n"
+ "2024-03-06 02:12:47.492497092 [W:onnxruntime:, graph.cc:1283 Graph] Initializer linear.bias appears in graph inputs and will not be treated as constant value/weight. This may prevent some of the graph optimizations, like const folding. Move it out of graph inputs if there is no need to override it, by either re-generating the model with latest exporter/converter or with the tool onnxruntime/tools/python/remove_initializer_from_input.py.\n"
  ]
  }
  ],
@@ -842,18 +842,18 @@
  },
  "outputs": [
  {
- "name": "stdout",
+ "name": "stderr",
  "output_type": "stream",
  "text": [
- "True\n"
+ "/scratch/fabian/brevitas/src/brevitas/export/onnx/standard/manager.py:26: UserWarning: ONNX opset version set to 13, override with opset_version=\n",
+ " warnings.warn(f\"ONNX opset version set to {DEFAULT_OPSET}, override with {ka}=\")\n"
  ]
  },
  {
- "name": "stderr",
+ "name": "stdout",
  "output_type": "stream",
  "text": [
- "/scratch/fabian/brevitas/src/brevitas/export/onnx/standard/manager.py:26: UserWarning: ONNX opset version set to 13, override with opset_version=\n",
- " warnings.warn(f\"ONNX opset version set to {DEFAULT_OPSET}, override with {ka}=\")\n"
+ "True\n"
  ]
  }
  ],
@@ -912,6 +912,98 @@
  "\n",
  "Due to differences in how the computation is performed between Brevitas and ONNX Runtime, it might happen the two results are slightly different (since Brevitas uses a style closer to QCDQ, rather than operating between integers), thus we added a tolerance for off-by-1 errors."
  ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Export Dynamically Quantized Models to ONNX "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "You can also export dynamically quantized models to ONNX, but there are some limitations. The ONNX DynamicQuantizeLinear requires the following settings:\n",
+ "- Asymmetric quantization (and therefore *unsigned*)\n",
+ "- Min-max scaling\n",
+ "- Rounding to nearest\n",
+ "- Per tensor scaling\n",
+ "- Bit width set to 8\n",
+ "\n",
+ "This is shown in the following example:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from brevitas_examples.common.generative.quantizers import ShiftedUint8DynamicActPerTensorFloat\n",
+ "\n",
+ "IN_CH = 3\n",
+ "IMG_SIZE = 128\n",
+ "OUT_CH = 128\n",
+ "BATCH_SIZE = 1\n",
+ "\n",
+ "class Model(torch.nn.Module):\n",
+ " def __init__(self) -> None:\n",
+ " super().__init__()\n",
+ " self.linear = qnn.QuantLinear(IN_CH, OUT_CH, bias=True, weight_bit_width=8, input_quant=ShiftedUint8DynamicActPerTensorFloat)\n",
+ " self.act = qnn.QuantReLU(input_quant=ShiftedUint8DynamicActPerTensorFloat)\n",
+ " \n",
+ " def forward(self, inp):\n",
+ " inp = self.linear(inp)\n",
+ " inp = self.act(inp)\n",
+ " return inp\n",
+ "\n",
+ "inp = torch.randn(BATCH_SIZE, IN_CH)\n",
+ "model = Model() \n",
+ "model.eval()\n",
+ "path = 'dynamic_quant_model_qcdq.onnx'\n",
+ "\n",
+ "exported_model = export_onnx_qcdq(model, args=inp, export_path=path, opset_version=13)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Serving 'dynamic_quant_model_qcdq.onnx' at http://localhost:8086\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " <iframe\n",
+ " width=\"100%\"\n",
+ " height=\"400\"\n",
+ " src=\"http://localhost:8086/\"\n",
+ " frameborder=\"0\"\n",
+ " allowfullscreen\n",
+ " \n",
+ " ></iframe>\n",
+ " "
+ ],
+ "text/plain": [
+ "<IPython.lib.display.IFrame at 0x7fb62856ccd0>"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "show_netron(\"dynamic_quant_model_qcdq.onnx\", 8086)"
+ ]
  }
  ],
  "metadata": {