speech-to-text/wav2vec-tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import librosa\n",
    "import tensorflow as tf\n",
    "import glob\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# follow hyperparameters from here, https://github.com/pytorch/fairseq/tree/master/examples/wav2vec\n",
    "\n",
    "features = [(512, 10, 5), (512, 8, 4), (512, 4, 2), (512, 4, 2), (512, 4, 2), (512, 1, 1), (512, 1, 1)]\n",
    "aggs = [(512, 2, 1), (512, 3, 1), (512, 4, 1), (512, 5, 1), (512, 6, 1), (512, 7, 1), (512, 8, 1), (512, 9, 1), \n",
    "        (512, 10, 1), (512, 11, 1), (512, 12, 1), (512, 13, 1)]\n",
    "num_negatives = 10\n",
    "prediction_steps = 12\n",
    "learning_rate = 1e-5\n",
    "min_learning_rate = 1e-9\n",
    "max_learning_rate = 0.005\n",
    "learning_scheduler = 'cosine'\n",
    "max_update = 400000\n",
    "residual_scale = 0.5\n",
    "log_compression = True\n",
    "warmup_updates = 50\n",
    "warmup_init_lr = 1e-07\n",
    "batch_size = 32\n",
    "epoch = 10\n",
    "total_steps = batch_size * epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.compat.v1.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 7, 10)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "\n",
    "# 2 batch, 10 dimension, 7 t\n",
    "x = np.transpose(np.random.normal(size = (2, 10, 7)), (0, 2, 1))\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(10), Dimension(2), Dimension(10), Dimension(7)])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def negative_sample(y):\n",
    "    bsz = tf.shape(y)[0]\n",
    "    fsz = tf.shape(y)[1]\n",
    "    tsz = tf.shape(y)[2]\n",
    "    \n",
    "    # b, d, t -> d, b, t\n",
    "    y = tf.transpose(y, [1, 0, 2])\n",
    "    y = tf.reshape(y, (fsz, -1))\n",
    "    # neg_idxs = tf.random_uniform((bsz, num_negatives * tsz), minval=0, maxval=tsz, dtype=tf.int32)\n",
    "    \n",
    "    neg_idxs = np.array([[\n",
    "         1, 2, 3, 1, 4, 0, 5, 6, 1, 2, 0, 4, 2, 1, 0, 5, 4, 5, 4, 6, 6, 4, 1, 6,\n",
    "         6, 3, 4, 4, 5, 0, 1, 5, 4, 4, 1, 1, 0, 2, 0, 6, 2, 6, 3, 4, 5, 6, 2, 4,\n",
    "         0, 2, 1, 2, 6, 4, 2, 4, 0, 2, 4, 2, 1, 0, 4, 6, 6, 4, 4, 2, 3, 4],\n",
    "        [4, 0, 3, 4, 2, 4, 4, 1, 0, 6, 3, 1, 5, 6, 4, 3, 6, 4, 0, 5, 1, 0, 4, 2,\n",
    "         2, 0, 4, 1, 4, 3, 2, 2, 0, 4, 2, 3, 4, 6, 6, 2, 4, 0, 3, 1, 6, 2, 4, 5,\n",
    "         1, 3, 1, 3, 3, 1, 3, 0, 3, 6, 0, 5, 2, 4, 5, 6, 0, 1, 2, 3, 6, 3]])\n",
    "    \n",
    "    ranged = tf.expand_dims(tf.range(1, bsz), axis = 1)\n",
    "    a = tf.add(neg_idxs[1:bsz], tf.tile(ranged, [1, num_negatives * tsz]) * tsz)\n",
    "    \n",
    "    neg_idxs = tf.concat([neg_idxs[:1], a, neg_idxs[bsz:]], axis = 0)\n",
    "    neg_idxs = tf.reshape(neg_idxs, [-1])\n",
    "    negs = tf.gather(y, neg_idxs, axis=1)\n",
    "    negs = tf.reshape(negs, (fsz, bsz, num_negatives, tsz))\n",
    "    negs = tf.transpose(negs, [2, 1, 0, 3])\n",
    "    return negs\n",
    "\n",
    "# b, t, d -> b, d, t\n",
    "y = tf.transpose(x.copy(), (0, 2, 1))\n",
    "neg = negative_sample(y)\n",
    "neg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(11), Dimension(2), Dimension(10), Dimension(7)])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets = tf.concat([tf.expand_dims(y, axis = 0), neg], axis = 0)\n",
    "targets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 7, 10)\n",
      "WARNING:tensorflow:From <ipython-input-8-1612d6acfa02>:5: conv2d_transpose (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv2DTranspose` instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(11), Dimension(2), Dimension(7), Dimension(10), Dimension(12)])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = tf.shape(targets)[0]\n",
    "print(x.shape)\n",
    "x_ = tf.expand_dims(x, axis = -1)\n",
    "\n",
    "x_ = tf.layers.conv2d_transpose(x_, prediction_steps, (1, 1))\n",
    "x_ = tf.expand_dims(x_, axis = 0) \n",
    "x_ = tf.tile(x_, [b, 1, 1, 1, 1])\n",
    "x_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11, 2, 10, 7, 12)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "with open('convtranspose.pkl', 'rb') as fopen:\n",
    "    x_ = np.array(pickle.load(fopen))\n",
    "    \n",
    "x_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "\n",
    "jin = 0\n",
    "rin = 0\n",
    "for _, k, stride in features:\n",
    "    if rin == 0:\n",
    "        rin = k\n",
    "    rin = rin + (k - 1) * jin\n",
    "    if jin == 0:\n",
    "        jin = stride\n",
    "    else:\n",
    "        jin *= stride\n",
    "offset = math.ceil(rin / jin)\n",
    "\n",
    "offset = int(offset)\n",
    "offset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([Dimension(220)]), TensorShape([Dimension(220)]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copies = tf.shape(x_)[0]\n",
    "bsz = tf.shape(x_)[1]\n",
    "dim = tf.shape(x_)[2]\n",
    "tsz = tf.shape(x_)[3]\n",
    "steps = tf.shape(x_)[4]\n",
    "\n",
    "steps = tf.math.minimum(steps, tsz - offset)\n",
    "predictions = tf.zeros(bsz * copies * (tsz - offset + 1) * \\\n",
    "                       steps - ((steps + 1) * steps // 2) * copies * bsz)\n",
    "labels = tf.zeros_like(predictions)\n",
    "predictions.shape, labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_ = tf.cast(x_, tf.float32)\n",
    "targets = tf.cast(targets, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(11), Dimension(2), Dimension(10), Dimension(7), Dimension(12)])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 tf.Tensor(8, shape=(), dtype=int32) tf.Tensor(88, shape=(), dtype=int32) 3 (11, 2, 10, 4) (11, 2, 10, 4)\n",
      "tf.Tensor(88, shape=(), dtype=int32) tf.Tensor(6, shape=(), dtype=int32) tf.Tensor(154, shape=(), dtype=int32) 4 (11, 2, 10, 3) (11, 2, 10, 3)\n",
      "tf.Tensor(154, shape=(), dtype=int32) tf.Tensor(2, shape=(), dtype=int32) tf.Tensor(176, shape=(), dtype=int32) 6 (11, 2, 10, 1) (11, 2, 10, 1)\n",
      "tf.Tensor(176, shape=(), dtype=int32) tf.Tensor(-4, shape=(), dtype=int32) tf.Tensor(132, shape=(), dtype=int32) 9 (11, 2, 10, 0) (11, 2, 10, 0)\n"
     ]
    }
   ],
   "source": [
    "def body(i, start, end, predictions, labels, offset):\n",
    "    offset = i + offset\n",
    "    end = start + (tsz - offset) * bsz * copies\n",
    "    pos_num = (end - start) // copies\n",
    "    print(start, pos_num, end, offset, x_[:, :, :, :-offset, i].shape, targets[:, :, :, offset:].shape)\n",
    "    s = tf.reduce_sum((x_[:, :, :, :-offset, i] * targets[:, :, :, offset:]), axis = 2)\n",
    "    s = tf.reshape(s, [-1])\n",
    "    s = tf.pad(s, [[start, tf.shape(predictions)[0] - (start + tf.shape(s)[0])]])\n",
    "    predictions = tf.add(predictions, s)\n",
    "    pos_num = pos_num if pos_num > 0 else 0\n",
    "    l = tf.ones((pos_num))\n",
    "    l = tf.pad(l, [[start, tf.shape(labels)[0] - (start + pos_num)]])\n",
    "    labels = tf.add(labels, l)\n",
    "    return i + 1, end, end, predictions, labels, offset\n",
    "\n",
    "def condition(i, start, end, predictions, labels, offset):\n",
    "    return i < steps\n",
    "\n",
    "ranged = tf.Variable(tf.constant(0))\n",
    "_, _, _, predictions, labels, _ = tf.while_loop(condition, body, [0, 0, 0, predictions, labels, offset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_weights = tf.abs(tf.sign(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=486, shape=(), dtype=float32, numpy=0.06309498>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_probs = tf.math.log_sigmoid(predictions)\n",
    "per_example_loss = -1 * (log_probs * labels)\n",
    "numerator = tf.reduce_sum(label_weights * per_example_loss)\n",
    "denominator = tf.reduce_sum(label_weights) + 1e-5\n",
    "numerator / denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:180: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=505, shape=(), dtype=float32, numpy=0.7334087>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numerator = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,\n",
    "            logits=predictions) * label_weights\n",
    "numerator = tf.reduce_sum(numerator)\n",
    "denominator = tf.reduce_sum(label_weights) + 1e-5\n",
    "numerator / denominator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}