synth_dctts.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of synth_dctts.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "0lPC8qTVNmwv",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Synthesize speech from text\n",
        "- from trained dctts model\n",
        "- code adpated from: https://github.com/Kyubyong/dc_tts"
      ]
    },
    {
      "metadata": {
        "id": "e-cR95Zuoi0I",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Load dependencies"
      ]
    },
    {
      "metadata": {
        "id": "oL_mIqoGLDcK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from train_transfer import Graph\n",
        "from utils import *\n",
        "from data_load import load_data\n",
        "from scipy.io.wavfile import write, read\n",
        "from tqdm import tqdm\n",
        "import codecs\n",
        "import re\n",
        "import os\n",
        "import unicodedata\n",
        "from IPython.display import Audio\n",
        "import scipy.signal as sg"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8D5n_YHVtDSh",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Parameters"
      ]
    },
    {
      "metadata": {
        "id": "p9sgsJNftLwX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "hp_vocab = \"PE abcdefghijklmnopqrstuvwxyz'.?\" # P: Padding, E: EOS. \n",
        "hp_max_N = 180 # Maximum number of characters.\n",
        "hp_max_T = 210 # Maximum number of mel frames.\n",
        "hp_n_mels = 80  # Number of Mel banks to generate\n",
        "hp_restoredir = \"/content/gdrive/My Drive/dctts_colab/logdir/scarjo\"\n",
        "hp_sr = 22050\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7m7Vcl43N0hl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Load models"
      ]
    },
    {
      "metadata": {
        "id": "OXHDMJvWj1ZK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        " %%capture\n",
        " # Load graph\n",
        " g = Graph(mode=\"synthesize\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ySiENpbBMmd_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Helper functions"
      ]
    },
    {
      "metadata": {
        "id": "-aAZYciBpHxW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def load_vocab():\n",
        "    char2idx = {char: idx for idx, char in enumerate(hp_vocab)}\n",
        "    idx2char = {idx: char for idx, char in enumerate(hp_vocab)}\n",
        "    return char2idx, idx2char\n",
        "  \n",
        "def text_normalize(text):\n",
        "    text = ''.join(char for char in unicodedata.normalize('NFD', text)\n",
        "                           if unicodedata.category(char) != 'Mn') # Strip accents\n",
        "\n",
        "    text = text.lower()\n",
        "    text = re.sub(\"[^{}]\".format(hp_vocab), \" \", text)\n",
        "    text = re.sub(\"[ ]+\", \" \", text)\n",
        "    return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7ip2NRq_NCQA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "sents  = [' a strange  game. the only winning move is not to play.E']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2wcIZUbmM7OK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## may have to add spaces to improve pronunciation (skip the normalization)"
      ]
    },
    {
      "metadata": {
        "id": "1qQC3o6xrMFS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "char2idx, idx2char = load_vocab()\n",
        "\n",
        "texts = np.zeros((len(sents), hp_max_N), np.int32)\n",
        "for i, sent in enumerate(sents):\n",
        "    texts[i, :len(sent)] = [char2idx[char] for char in sent]\n",
        "    \n",
        "L = texts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZjPncq6_sR8Y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "with tf.Session() as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "\n",
        "    # Restore parameters\n",
        "    var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'Text2Mel')\n",
        "    saver1 = tf.train.Saver(var_list=var_list)\n",
        "    saver1.restore(sess, tf.train.latest_checkpoint(hp_restoredir + \"-1\"))\n",
        "    print(\"Text2Mel Restored!\")\n",
        "\n",
        "    var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'SSRN') + \\\n",
        "               tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'gs')\n",
        "    saver2 = tf.train.Saver(var_list=var_list)\n",
        "    saver2.restore(sess, tf.train.latest_checkpoint(hp_restoredir + \"-2\"))\n",
        "    print(\"SSRN Restored!\")\n",
        "\n",
        "    # Feed Forward\n",
        "    ## mel\n",
        "    Y = np.zeros((len(L), hp_max_T, hp_n_mels), np.float32)\n",
        "    prev_max_attentions = np.zeros((len(L),), np.int32)\n",
        "    for j in tqdm(range(hp_max_T)):\n",
        "        _gs, _Y, _max_attentions, _alignments = \\\n",
        "            sess.run([g.global_step, g.Y, g.max_attentions, g.alignments],\n",
        "                     {g.L: L,\n",
        "                      g.mels: Y,\n",
        "                      g.prev_max_attentions: prev_max_attentions})\n",
        "        Y[:, j, :] = _Y[:, j, :]\n",
        "        prev_max_attentions = _max_attentions[:, j]\n",
        "\n",
        "    # Get magnitude\n",
        "    Z = sess.run(g.Z, {g.Y: Y})\n",
        "    mag = Z[0]\n",
        "    wav = spectrogram2wav(mag)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HGZDoC9Z8_u7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Audio(wav,rate=hp.sr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fLgN3JKYFPW1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "b, a = sg.butter(4, 7300. / (hp_sr / 2.), 'low')\n",
        "wav_fil = sg.filtfilt(b, a, wav)\n",
        "Audio(wav_fil,rate=hp.sr)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}