Dynet tutorials.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import dynet as dy\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0.]\n",
      " [ 0.  0.  0.]]\n",
      "[[ 0.  0.  0.]\n",
      " [ 0.  0.  0.]]\n",
      "[1.0, 2.0, 3.0]\n",
      "1.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "## ==== Create a new computation graph\n",
    "# (it is a singleton, we have one at each stage.\n",
    "# renew_cg() clears the current one and starts anew)\n",
    "dy.renew_cg()\n",
    "\n",
    "## ==== Creating Expressions from user input / constants.\n",
    "x = dy.scalarInput(1)\n",
    "\n",
    "v = dy.vecInput(3)\n",
    "v.set([1,2,3])\n",
    "\n",
    "z = dy.matInput(2, 3)\n",
    "\n",
    "# for example:\n",
    "z1 = dy.matInput(2, 2)\n",
    "z1.set([1,2,3,4])\n",
    "\n",
    "## ==== We can take the value of an expression. \n",
    "# For complex expressions, this will run forward propagation.\n",
    "print z.value()    \n",
    "print z.npvalue()      # as numpy array\n",
    "print v.vec_value()    # as vector, if vector\n",
    "print x.scalar_value() # as scalar, if scalar\n",
    "print x.value()        # choose the correct one\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## ==== Parameters\n",
    "# Parameters are things we tune during training.\n",
    "# Usually a matrix or a vector.\n",
    "\n",
    "# First we create a model and add the parameters to it.\n",
    "m = dy.Model() \n",
    "pW = m.add_parameters((8,8)) # an 8x8 matrix\n",
    "pb = m.add_parameters(8)\n",
    "\n",
    "# then we create an Expression out of the model's parameters\n",
    "W = dy.parameter(pW)\n",
    "b = dy.parameter(pb)\n",
    "\n",
    "## ===== Lookup parameters\n",
    "# Similar to parameters, but are representing a \"lookup table\"\n",
    "# that maps numbers to vectors.\n",
    "# These are used for embedding matrices.\n",
    "# for example, this will have VOCAB_SIZE rows, each of DIM dimensions.\n",
    "VOCAB_SIZE=10\n",
    "DIM=5\n",
    "DIMC=3\n",
    "lp  = m.add_lookup_parameters((VOCAB_SIZE, DIM))\n",
    "clp  = m.add_lookup_parameters((VOCAB_SIZE, DIMC))\n",
    "attn = m.add_parameters((DIMC,DIM))\n",
    "\n",
    "# lookup parameters can be initialized from an existing array, i.e:\n",
    "# m[\"lookup\"].init_from_array(wv)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, 5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lp.shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "e = lp[1]\n",
    "A = dy.parameter(attn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3, 5)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A.npvalue().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e.npvalue().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dy.transpose(e).npvalue().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(A*e).npvalue().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.13906434])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dy.transpose(A*e)*clp[1]).npvalue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.13906434])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dy.transpose(clp[1])*(A*e)).npvalue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[exprssion 21/1,\n",
       " exprssion 25/1,\n",
       " exprssion 29/1,\n",
       " exprssion 33/1,\n",
       " exprssion 37/1]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn_w = [dy.transpose(clp[i])*(A*e) for i in xrange(5)]\n",
    "attn_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.08688716,  0.16628774,  0.09851928,  0.52598399,  0.12232184])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dy.softmax(dy.concatenate(attn_w)).npvalue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.08688716,  0.16628774,  0.09851928,  0.52598399,  0.12232184])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn_w = dy.softmax(dy.concatenate(attn_w))\n",
    "attn_w.npvalue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.0369454 , -0.01742968,  0.0734873 ])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(clp[0]*attn_w[0]).npvalue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exprssion 41/1"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attn_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}