Numpy compatible diagflat operator

example/numpy/numpy_semantics.ipynb

@@ -0,0 +1,308 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# How to Use NumPy Semantics in MXNet with `mxnet.numpy` Module\n",
+ "\n",
+ "## NumPy Shape Semantics\n",
+ "\n",
+ "### Example \n",
+ "\n",
+ "| Shape Example | MXNet (before) | MXNet/NumPy |\n",
+ "|:---:|:---:|:---:|\n",
+ "| `()` | unknown | Scalar tensor |\n",
+ "| `(2, 0, 1)` | Second dimension unknown | Zero-size tensor |\n",
+ "| `None`(Python) | N/A | Unknown |\n",
+ "| `(2, -1, 0)`(C++) | N/A | Second dim uknown|\n",
+ "\n",
+ "### Affected modules\n",
+ "- Shape inference: imperative, symbolic, Gluon\n",
+ "- Legacy operators (not recommended to use)\n",
+ "- MXNet/NumPy operators\n",
+ "\n",
+ "## NumPy Array Semantics\n",
+ "**Definition:** The type of created ndarrays is `mxnet.numpy.ndarray`/`mxnet.symbol.numpy._Symbol`, instead of `mxnet.ndarray.NDArray`/`mxnet.symbol.Symbol` (only affects Gluon modules).\n",
+ "- Block/HybridBlock\n",
+ " - Parameter creation and initialization.\n",
+ " - Inputs/outputs (symbol/ndarray) of `__call__`/`forward`/`hybrid_forward`.\n",
+ " - Computational graph construction.\n",
+ "- Dataloader\n",
+ "\n",
+ "## Dependency of Two Types of Semantics\n",
+ "- It is required to keep NumPy shape semantics active while activating NumPy array semantics.\n",
+ "- Deactivating NumPy shape semantics while NumPy array semantics is still active is not allowed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "import mxnet as mx\n",
+ "from mxnet import np, npx, gluon\n",
+ "\n",
+ "logging.basicConfig(level=logging.INFO)\n",
+ "\n",
+ "try:\n",
+ " npx.set_np(shape=False, array=True)\n",
+ "except ValueError as e:\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## How to Enable NumPy Shape semantics"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " a = mx.nd.random.uniform(shape=())\n",
+ "except mx.MXNetError as e:\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " b = mx.nd.random.uniform(shape=(2, 0, 1))\n",
+ "except mx.MXNetError as e:\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " c = np.random.uniform()\n",
+ "except mx.MXNetError as e:\n",
+ " print(e)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " d = np.random.uniform(size=(2, 0, 1))\n",
+ "except mx.MXNetError as e:\n",
+ " print(e) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "npx.set_np(shape=True, array=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "a = mx.nd.random.uniform(shape=())\n",
+ "b = mx.nd.random.uniform(shape=(2, 0, 1))\n",
+ "c = np.random.uniform()\n",
+ "d = np.random.uniform(size=(2, 0, 1))\n",
+ "\n",
+ "print('type(a) =', type(a))\n",
+ "print('a.shape = ', a.shape)\n",
+ "print('a.size = ', a.size)\n",
+ "\n",
+ "print('type(b) =', type(b))\n",
+ "print('b.shape = ', b.shape)\n",
+ "print('b.size = ', b.size)\n",
+ "\n",
+ "print('type(c) =', type(c))\n",
+ "print('c.shape = ', c.shape)\n",
+ "print('c.size = ', c.size)\n",
+ "\n",
+ "print('type(d) =', type(d))\n",
+ "print('d.shape = ', d.shape)\n",
+ "print('d.size = ', d.size)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## How to Enable NumPy Array Semantics\n",
+ "\n",
+ "### Parameters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "npx.reset_np() # reset two types of semantics to the default state, which is False for both of them\n",
+ "\n",
+ "from mxnet.gluon import nn\n",
+ "class Net(gluon.Block):\n",
+ " def __init__(self, in_units=0, **kwargs): # 0 means in_units is unknown and must be inferred at runtime\n",
+ " super(Net, self).__init__(**kwargs)\n",
+ " with self.name_scope():\n",
+ " self.dense0 = nn.Dense(5, in_units=in_units)\n",
+ " self.dense1 = nn.Dense(5, in_units=in_units)\n",
+ " \n",
+ " def forward(self, x):\n",
+ " return self.dense1(self.dense0(x))\n",
+ "\n",
+ "net1 = Net()\n",
+ "net1.initialize()\n",
+ "net1(mx.nd.zeros((3, 10)))\n",
+ "for k, v in net1.collect_params().items():\n",
+ " print('parameter {}, type {}'.format(k, str(type(v.data()))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "npx.set_np()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "net2 = Net()\n",
+ "net2.initialize()\n",
+ "net2(np.zeros((3, 10)))\n",
+ "for k, v in net2.collect_params().items():\n",
+ " print('parameter {}, type {}'.format(k, str(type(v.data()))))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Dataloader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sys\n",
+ "import os\n",
+ "from mxnet.gluon import data as gdata\n",
+ "\n",
+ "\n",
+ "npx.reset_np()\n",
+ "\n",
+ "\n",
+ "def load_data_fashion_mnist(batch_size, resize=None, root=os.path.join(\n",
+ " '~', '.mxnet', 'datasets', 'fashion-mnist')):\n",
+ " \"\"\"Download the Fashion-MNIST dataset and then load into memory.\"\"\"\n",
+ " root = os.path.expanduser(root)\n",
+ " transformer = []\n",
+ " if resize:\n",
+ " transformer += [gdata.vision.transforms.Resize(resize)]\n",
+ " transformer += [gdata.vision.transforms.ToTensor()]\n",
+ " transformer = gdata.vision.transforms.Compose(transformer)\n",
+ "\n",
+ " mnist_train = gdata.vision.FashionMNIST(root=root, train=True)\n",
+ " mnist_test = gdata.vision.FashionMNIST(root=root, train=False)\n",
+ " num_workers = 0 if sys.platform.startswith('win32') else 4\n",
+ "\n",
+ " train_iter = gdata.DataLoader(mnist_train.transform_first(transformer),\n",
+ " batch_size, shuffle=True,\n",
+ " num_workers=num_workers)\n",
+ " test_iter = gdata.DataLoader(mnist_test.transform_first(transformer),\n",
+ " batch_size, shuffle=False,\n",
+ " num_workers=num_workers)\n",
+ " return train_iter, test_iter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_iter, test_iter = load_data_fashion_mnist(16)\n",
+ "\n",
+ "for X, y in train_iter:\n",
+ " print('type(X) = ', type(X))\n",
+ " print('type(y) = ', type(y))\n",
+ " break"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "npx.set_np()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_iter, test_iter = load_data_fashion_mnist(16)\n",
+ "\n",
+ "for X, y in train_iter:\n",
+ " print('type(X) = ', type(X))\n",
+ " print('type(y) = ', type(y))\n",
+ " break"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.0"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

include/mxnet/base.h

@@ -421,7 +421,9 @@ inline int32_t Context::GetGPUCount() {
 #if MXNET_USE_CUDA
  int32_t count;
  cudaError_t e = cudaGetDeviceCount(&count);
- if (e == cudaErrorNoDevice) {
+ // TODO(junwu): Remove e == 35
+ // This is skipped for working around wheel build system with older CUDA driver.
+ if (e == cudaErrorNoDevice || e == 35) {
  return 0;
  }
  CHECK_EQ(e, cudaSuccess) << " CUDA: " << cudaGetErrorString(e);

include/mxnet/c_api.h

@@ -2811,6 +2811,18 @@ MXNET_DLL int MXEnginePushSync(EngineSyncFunc sync_func, void* func_param,
  EngineVarHandle mutable_vars_handle, int num_mutable_vars,
  EngineFnPropertyHandle prop_handle DEFAULT(NULL),
  int priority DEFAULT(0), const char* opr_name DEFAULT(NULL));
+/*!
+ * \brief Create an NDArray from source sharing the same data chunk.
+ * \param src source NDArray
+ * \param out new NDArray sharing the same data chunck with src
+ */
+MXNET_DLL int MXShallowCopyNDArray(NDArrayHandle src, NDArrayHandle* out);
+/*!
+ * \brief Create an Symbol from source sharing the same graph structure.
+ * \param src source Symbol
+ * \param out new Symbol sharing the same graph structure with src
+ */
+MXNET_DLL int MXShallowCopySymbol(SymbolHandle src, SymbolHandle * out);
 
 #ifdef __cplusplus
 }

include/mxnet/tuple.h

@@ -272,6 +272,14 @@ class Tuple {
  is.get();
  if (ch == '(' || ch == '[') break;
  if (!isspace(ch)) {
+ if (ch == 'N') {
+ std::string tmp_val;
+ is >> tmp_val;
+ if (tmp_val == "one") { // is stores "None"
+ t.SetDim(-1);
+ return is;
+ }
+ }
  is.setstate(std::ios::failbit);
  return is;
  }
@@ -653,6 +661,13 @@ inline bool shape_is_known(const TShape& x) {
  return true;
 }
 
+inline bool shape_is_known(const std::vector<TShape>& shapes) {
+ for (const TShape& shape : shapes) {
+ if (!shape_is_known(shape)) return false;
+ }
+ return true;
+}
+
 /*! \brief helper function to cast type of container elements */
 template<typename SrcIter, typename DstIter>
 inline DstIter ShapeTypeCast(const SrcIter begin,

python/mxnet/__init__.py

@@ -25,10 +25,15 @@
 from . import engine
 from .base import MXNetError
 from .util import is_np_shape, set_np_shape, np_shape, use_np_shape
+from .util import is_np_array, np_array, use_np_array, use_np
 from . import base
 from . import contrib
 from . import ndarray
 from . import ndarray as nd
+from . import numpy
+from . import numpy_extension
+from . import numpy as np
+from . import numpy_extension as npx
 from . import name
 # use mx.sym as short for symbol
 from . import symbol as sym