Chap_4_NLP_tokenization_pad_sequences.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_tokenization_pad_sequences.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPdPpC1usNA/PIY4wk3zqVK",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/raqueeb/nlp_bangla/blob/master/NLP_tokenization_pad_sequences.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ldIWtofK4zF1",
        "colab_type": "text"
      },
      "source": [
        "## সহজ বাংলায় 'বাংলা' ন্যাচারাল ল্যাঙ্গুয়েজ প্রসেসিং (এনএলপি) - ২"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZHl0zGUi2rd1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "train_data = [\n",
        "  \"আচ্ছা, ডেটা কিভাবে কথা বলে?\",\n",
        "  \"পড়ছিলাম হান্স রোসলিং এর একটা বই, ফ্যাক্টফুলনেস।\",\n",
        "  \"ধারণা থেকে নয়, বরং ডেটাকে কথা বলতে দিলে আমাদের সব বিপদ কাটবে।\",\n",
        "  \"এই লোক পৃথিবীকে দেখিয়েছিলেন কিভাবে ২০০ বছরের ডেটা আমাদের বাঁচার সময় বাড়িয়েছে!\"\n",
        "]\n",
        "\n",
        "test_data = [\n",
        "  \"এই অ্যানিমেশন আমরা করবো আমাদের পিসিতে।\",\n",
        "  \"সরাসরি চালান নিচের লিংক থেকে, হচ্ছে তো?\",\n",
        "  \"পাল্টান প্যারামিটার, চালান নিজের মতো করে।\"\n",
        "]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s-tgVqK23sVt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_words = 1000\n",
        "oov_token = '<UNK>'\n",
        "pad_type = 'post'\n",
        "trunc_type = 'post'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTGi2Xmg30l9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# আমাদের ট্রেনিং ডেটা টোকেনাইজ করি, বাংলায় স্টপওয়ার্ড হিসেবে দাড়ি '।'কে ফেলে দিতে হবে।\n",
        "tokenizer = Tokenizer(num_words=num_words, filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n।', oov_token=oov_token)\n",
        "tokenizer.fit_on_texts(train_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rJgeoGTq32ob",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ট্রেনিং ডেটার ওয়ার্ড ইনডেক্স বের করি\n",
        "word_index = tokenizer.word_index"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5eKrnNOe8rIx",
        "colab_type": "code",
        "outputId": "8d7b2d56-f830-415b-b5d7-f6fc60a9fda4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "tokenizer.get_config()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'char_level': False,\n",
              " 'document_count': 4,\n",
              " 'filters': '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n।',\n",
              " 'index_docs': '{\"6\": 1, \"2\": 2, \"4\": 2, \"3\": 2, \"7\": 1, \"8\": 1, \"10\": 1, \"14\": 1, \"11\": 1, \"12\": 1, \"13\": 1, \"9\": 1, \"18\": 1, \"19\": 1, \"17\": 1, \"23\": 1, \"16\": 1, \"5\": 2, \"22\": 1, \"24\": 1, \"20\": 1, \"21\": 1, \"15\": 1, \"25\": 1, \"29\": 1, \"30\": 1, \"32\": 1, \"28\": 1, \"33\": 1, \"26\": 1, \"27\": 1, \"31\": 1}',\n",
              " 'index_word': '{\"1\": \"<UNK>\", \"2\": \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\", \"3\": \"\\\\u0995\\\\u09bf\\\\u09ad\\\\u09be\\\\u09ac\\\\u09c7\", \"4\": \"\\\\u0995\\\\u09a5\\\\u09be\", \"5\": \"\\\\u0986\\\\u09ae\\\\u09be\\\\u09a6\\\\u09c7\\\\u09b0\", \"6\": \"\\\\u0986\\\\u099a\\\\u09cd\\\\u099b\\\\u09be\", \"7\": \"\\\\u09ac\\\\u09b2\\\\u09c7\", \"8\": \"\\\\u09aa\\\\u09dc\\\\u099b\\\\u09bf\\\\u09b2\\\\u09be\\\\u09ae\", \"9\": \"\\\\u09b9\\\\u09be\\\\u09a8\\\\u09cd\\\\u09b8\", \"10\": \"\\\\u09b0\\\\u09cb\\\\u09b8\\\\u09b2\\\\u09bf\\\\u0982\", \"11\": \"\\\\u098f\\\\u09b0\", \"12\": \"\\\\u098f\\\\u0995\\\\u099f\\\\u09be\", \"13\": \"\\\\u09ac\\\\u0987\", \"14\": \"\\\\u09ab\\\\u09cd\\\\u09af\\\\u09be\\\\u0995\\\\u09cd\\\\u099f\\\\u09ab\\\\u09c1\\\\u09b2\\\\u09a8\\\\u09c7\\\\u09b8\", \"15\": \"\\\\u09a7\\\\u09be\\\\u09b0\\\\u09a3\\\\u09be\", \"16\": \"\\\\u09a5\\\\u09c7\\\\u0995\\\\u09c7\", \"17\": \"\\\\u09a8\\\\u09df\", \"18\": \"\\\\u09ac\\\\u09b0\\\\u0982\", \"19\": \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\\\\u0995\\\\u09c7\", \"20\": \"\\\\u09ac\\\\u09b2\\\\u09a4\\\\u09c7\", \"21\": \"\\\\u09a6\\\\u09bf\\\\u09b2\\\\u09c7\", \"22\": \"\\\\u09b8\\\\u09ac\", \"23\": \"\\\\u09ac\\\\u09bf\\\\u09aa\\\\u09a6\", \"24\": \"\\\\u0995\\\\u09be\\\\u099f\\\\u09ac\\\\u09c7\", \"25\": \"\\\\u098f\\\\u0987\", \"26\": \"\\\\u09b2\\\\u09cb\\\\u0995\", \"27\": \"\\\\u09aa\\\\u09c3\\\\u09a5\\\\u09bf\\\\u09ac\\\\u09c0\\\\u0995\\\\u09c7\", \"28\": \"\\\\u09a6\\\\u09c7\\\\u0996\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09bf\\\\u09b2\\\\u09c7\\\\u09a8\", \"29\": \"\\\\u09e8\\\\u09e6\\\\u09e6\", \"30\": \"\\\\u09ac\\\\u099b\\\\u09b0\\\\u09c7\\\\u09b0\", \"31\": \"\\\\u09ac\\\\u09be\\\\u0981\\\\u099a\\\\u09be\\\\u09b0\", \"32\": \"\\\\u09b8\\\\u09ae\\\\u09df\", \"33\": \"\\\\u09ac\\\\u09be\\\\u09dc\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09c7\"}',\n",
              " 'lower': True,\n",
              " 'num_words': 1000,\n",
              " 'oov_token': '<UNK>',\n",
              " 'split': ' ',\n",
              " 'word_counts': '{\"\\\\u0986\\\\u099a\\\\u09cd\\\\u099b\\\\u09be\": 1, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\": 2, \"\\\\u0995\\\\u09bf\\\\u09ad\\\\u09be\\\\u09ac\\\\u09c7\": 2, \"\\\\u0995\\\\u09a5\\\\u09be\": 2, \"\\\\u09ac\\\\u09b2\\\\u09c7\": 1, \"\\\\u09aa\\\\u09dc\\\\u099b\\\\u09bf\\\\u09b2\\\\u09be\\\\u09ae\": 1, \"\\\\u09b9\\\\u09be\\\\u09a8\\\\u09cd\\\\u09b8\": 1, \"\\\\u09b0\\\\u09cb\\\\u09b8\\\\u09b2\\\\u09bf\\\\u0982\": 1, \"\\\\u098f\\\\u09b0\": 1, \"\\\\u098f\\\\u0995\\\\u099f\\\\u09be\": 1, \"\\\\u09ac\\\\u0987\": 1, \"\\\\u09ab\\\\u09cd\\\\u09af\\\\u09be\\\\u0995\\\\u09cd\\\\u099f\\\\u09ab\\\\u09c1\\\\u09b2\\\\u09a8\\\\u09c7\\\\u09b8\": 1, \"\\\\u09a7\\\\u09be\\\\u09b0\\\\u09a3\\\\u09be\": 1, \"\\\\u09a5\\\\u09c7\\\\u0995\\\\u09c7\": 1, \"\\\\u09a8\\\\u09df\": 1, \"\\\\u09ac\\\\u09b0\\\\u0982\": 1, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\\\\u0995\\\\u09c7\": 1, \"\\\\u09ac\\\\u09b2\\\\u09a4\\\\u09c7\": 1, \"\\\\u09a6\\\\u09bf\\\\u09b2\\\\u09c7\": 1, \"\\\\u0986\\\\u09ae\\\\u09be\\\\u09a6\\\\u09c7\\\\u09b0\": 2, \"\\\\u09b8\\\\u09ac\": 1, \"\\\\u09ac\\\\u09bf\\\\u09aa\\\\u09a6\": 1, \"\\\\u0995\\\\u09be\\\\u099f\\\\u09ac\\\\u09c7\": 1, \"\\\\u098f\\\\u0987\": 1, \"\\\\u09b2\\\\u09cb\\\\u0995\": 1, \"\\\\u09aa\\\\u09c3\\\\u09a5\\\\u09bf\\\\u09ac\\\\u09c0\\\\u0995\\\\u09c7\": 1, \"\\\\u09a6\\\\u09c7\\\\u0996\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09bf\\\\u09b2\\\\u09c7\\\\u09a8\": 1, \"\\\\u09e8\\\\u09e6\\\\u09e6\": 1, \"\\\\u09ac\\\\u099b\\\\u09b0\\\\u09c7\\\\u09b0\": 1, \"\\\\u09ac\\\\u09be\\\\u0981\\\\u099a\\\\u09be\\\\u09b0\": 1, \"\\\\u09b8\\\\u09ae\\\\u09df\": 1, \"\\\\u09ac\\\\u09be\\\\u09dc\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09c7\": 1}',\n",
              " 'word_docs': '{\"\\\\u0986\\\\u099a\\\\u09cd\\\\u099b\\\\u09be\": 1, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\": 2, \"\\\\u0995\\\\u09a5\\\\u09be\": 2, \"\\\\u0995\\\\u09bf\\\\u09ad\\\\u09be\\\\u09ac\\\\u09c7\": 2, \"\\\\u09ac\\\\u09b2\\\\u09c7\": 1, \"\\\\u09aa\\\\u09dc\\\\u099b\\\\u09bf\\\\u09b2\\\\u09be\\\\u09ae\": 1, \"\\\\u09b0\\\\u09cb\\\\u09b8\\\\u09b2\\\\u09bf\\\\u0982\": 1, \"\\\\u09ab\\\\u09cd\\\\u09af\\\\u09be\\\\u0995\\\\u09cd\\\\u099f\\\\u09ab\\\\u09c1\\\\u09b2\\\\u09a8\\\\u09c7\\\\u09b8\": 1, \"\\\\u098f\\\\u09b0\": 1, \"\\\\u098f\\\\u0995\\\\u099f\\\\u09be\": 1, \"\\\\u09ac\\\\u0987\": 1, \"\\\\u09b9\\\\u09be\\\\u09a8\\\\u09cd\\\\u09b8\": 1, \"\\\\u09ac\\\\u09b0\\\\u0982\": 1, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\\\\u0995\\\\u09c7\": 1, \"\\\\u09a8\\\\u09df\": 1, \"\\\\u09ac\\\\u09bf\\\\u09aa\\\\u09a6\": 1, \"\\\\u09a5\\\\u09c7\\\\u0995\\\\u09c7\": 1, \"\\\\u0986\\\\u09ae\\\\u09be\\\\u09a6\\\\u09c7\\\\u09b0\": 2, \"\\\\u09b8\\\\u09ac\": 1, \"\\\\u0995\\\\u09be\\\\u099f\\\\u09ac\\\\u09c7\": 1, \"\\\\u09ac\\\\u09b2\\\\u09a4\\\\u09c7\": 1, \"\\\\u09a6\\\\u09bf\\\\u09b2\\\\u09c7\": 1, \"\\\\u09a7\\\\u09be\\\\u09b0\\\\u09a3\\\\u09be\": 1, \"\\\\u098f\\\\u0987\": 1, \"\\\\u09e8\\\\u09e6\\\\u09e6\": 1, \"\\\\u09ac\\\\u099b\\\\u09b0\\\\u09c7\\\\u09b0\": 1, \"\\\\u09b8\\\\u09ae\\\\u09df\": 1, \"\\\\u09a6\\\\u09c7\\\\u0996\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09bf\\\\u09b2\\\\u09c7\\\\u09a8\": 1, \"\\\\u09ac\\\\u09be\\\\u09dc\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09c7\": 1, \"\\\\u09b2\\\\u09cb\\\\u0995\": 1, \"\\\\u09aa\\\\u09c3\\\\u09a5\\\\u09bf\\\\u09ac\\\\u09c0\\\\u0995\\\\u09c7\": 1, \"\\\\u09ac\\\\u09be\\\\u0981\\\\u099a\\\\u09be\\\\u09b0\": 1}',\n",
              " 'word_index': '{\"<UNK>\": 1, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\": 2, \"\\\\u0995\\\\u09bf\\\\u09ad\\\\u09be\\\\u09ac\\\\u09c7\": 3, \"\\\\u0995\\\\u09a5\\\\u09be\": 4, \"\\\\u0986\\\\u09ae\\\\u09be\\\\u09a6\\\\u09c7\\\\u09b0\": 5, \"\\\\u0986\\\\u099a\\\\u09cd\\\\u099b\\\\u09be\": 6, \"\\\\u09ac\\\\u09b2\\\\u09c7\": 7, \"\\\\u09aa\\\\u09dc\\\\u099b\\\\u09bf\\\\u09b2\\\\u09be\\\\u09ae\": 8, \"\\\\u09b9\\\\u09be\\\\u09a8\\\\u09cd\\\\u09b8\": 9, \"\\\\u09b0\\\\u09cb\\\\u09b8\\\\u09b2\\\\u09bf\\\\u0982\": 10, \"\\\\u098f\\\\u09b0\": 11, \"\\\\u098f\\\\u0995\\\\u099f\\\\u09be\": 12, \"\\\\u09ac\\\\u0987\": 13, \"\\\\u09ab\\\\u09cd\\\\u09af\\\\u09be\\\\u0995\\\\u09cd\\\\u099f\\\\u09ab\\\\u09c1\\\\u09b2\\\\u09a8\\\\u09c7\\\\u09b8\": 14, \"\\\\u09a7\\\\u09be\\\\u09b0\\\\u09a3\\\\u09be\": 15, \"\\\\u09a5\\\\u09c7\\\\u0995\\\\u09c7\": 16, \"\\\\u09a8\\\\u09df\": 17, \"\\\\u09ac\\\\u09b0\\\\u0982\": 18, \"\\\\u09a1\\\\u09c7\\\\u099f\\\\u09be\\\\u0995\\\\u09c7\": 19, \"\\\\u09ac\\\\u09b2\\\\u09a4\\\\u09c7\": 20, \"\\\\u09a6\\\\u09bf\\\\u09b2\\\\u09c7\": 21, \"\\\\u09b8\\\\u09ac\": 22, \"\\\\u09ac\\\\u09bf\\\\u09aa\\\\u09a6\": 23, \"\\\\u0995\\\\u09be\\\\u099f\\\\u09ac\\\\u09c7\": 24, \"\\\\u098f\\\\u0987\": 25, \"\\\\u09b2\\\\u09cb\\\\u0995\": 26, \"\\\\u09aa\\\\u09c3\\\\u09a5\\\\u09bf\\\\u09ac\\\\u09c0\\\\u0995\\\\u09c7\": 27, \"\\\\u09a6\\\\u09c7\\\\u0996\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09bf\\\\u09b2\\\\u09c7\\\\u09a8\": 28, \"\\\\u09e8\\\\u09e6\\\\u09e6\": 29, \"\\\\u09ac\\\\u099b\\\\u09b0\\\\u09c7\\\\u09b0\": 30, \"\\\\u09ac\\\\u09be\\\\u0981\\\\u099a\\\\u09be\\\\u09b0\": 31, \"\\\\u09b8\\\\u09ae\\\\u09df\": 32, \"\\\\u09ac\\\\u09be\\\\u09dc\\\\u09bf\\\\u09df\\\\u09c7\\\\u099b\\\\u09c7\": 33}'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3jCRsIQA35md",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# টেনিং ডেটার বাক্যগুলোকে সিকোয়েন্স এ ফেলি\n",
        "train_sequences = tokenizer.texts_to_sequences(train_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fAAR2P339QF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ট্রেনিং এর সর্বোচ্চ লেনথ বের করি\n",
        "maxlen = max([len(x) for x in train_sequences])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C67PWt7R4JxY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ট্রেনিং এর সিকোয়েন্স এর প্যাডিং যোগ করি\n",
        "train_padded = pad_sequences(train_sequences, padding=pad_type, truncating=trunc_type, maxlen=maxlen)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tanuN9ag4KF5",
        "colab_type": "code",
        "outputId": "f3c04ce4-f806-4ddb-d592-bad7a4b8b878",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        }
      },
      "source": [
        "# আমাদের কাজের আউটপুটগুলো দেখি ভিন্ন ভাবে, তবে বাংলায় স্টপওয়ার্ড হিসেবে '।'কে ফেলে দিয়েছি।\n",
        "print(\"Word index:\\n\", word_index)\n",
        "print(\"\\nTraining sequences:\\n\", train_sequences)\n",
        "print(\"\\nPadded training sequences:\\n\", train_padded)\n",
        "print(\"\\nPadded training shape:\", train_padded.shape)\n",
        "print(\"Training sequences data type:\", type(train_sequences))\n",
        "print(\"Padded Training sequences data type:\", type(train_padded))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Word index:\n",
            " {'<UNK>': 1, 'ডেটা': 2, 'কিভাবে': 3, 'কথা': 4, 'আমাদের': 5, 'আচ্ছা': 6, 'বলে': 7, 'পড়ছিলাম': 8, 'হান্স': 9, 'রোসলিং': 10, 'এর': 11, 'একটা': 12, 'বই': 13, 'ফ্যাক্টফুলনেস': 14, 'ধারণা': 15, 'থেকে': 16, 'নয়': 17, 'বরং': 18, 'ডেটাকে': 19, 'বলতে': 20, 'দিলে': 21, 'সব': 22, 'বিপদ': 23, 'কাটবে': 24, 'এই': 25, 'লোক': 26, 'পৃথিবীকে': 27, 'দেখিয়েছিলেন': 28, '২০০': 29, 'বছরের': 30, 'বাঁচার': 31, 'সময়': 32, 'বাড়িয়েছে': 33}\n",
            "\n",
            "Training sequences:\n",
            " [[6, 2, 3, 4, 7], [8, 9, 10, 11, 12, 13, 14], [15, 16, 17, 18, 19, 4, 20, 21, 5, 22, 23, 24], [25, 26, 27, 28, 3, 29, 30, 2, 5, 31, 32, 33]]\n",
            "\n",
            "Padded training sequences:\n",
            " [[ 6  2  3  4  7  0  0  0  0  0  0  0]\n",
            " [ 8  9 10 11 12 13 14  0  0  0  0  0]\n",
            " [15 16 17 18 19  4 20 21  5 22 23 24]\n",
            " [25 26 27 28  3 29 30  2  5 31 32 33]]\n",
            "\n",
            "Padded training shape: (4, 12)\n",
            "Training sequences data type: <class 'list'>\n",
            "Padded Training sequences data type: <class 'numpy.ndarray'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_BM2AVZ44KJT",
        "colab_type": "code",
        "outputId": "b7469de0-7d8f-4ff3-ba39-828af62d1627",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        }
      },
      "source": [
        "test_sequences = tokenizer.texts_to_sequences(test_data)\n",
        "test_padded = pad_sequences(test_sequences, padding=pad_type, truncating=trunc_type, maxlen=maxlen)\n",
        "\n",
        "print(\"Testing sequences:\\n\", test_sequences)\n",
        "print(\"\\nPadded testing sequences:\\n\", test_padded)\n",
        "print(\"\\nPadded testing shape:\",test_padded.shape)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Testing sequences:\n",
            " [[25, 1, 1, 1, 5, 1], [1, 1, 1, 1, 16, 1, 1], [1, 1, 1, 1, 1, 1]]\n",
            "\n",
            "Padded testing sequences:\n",
            " [[25  1  1  1  5  1  0  0  0  0  0  0]\n",
            " [ 1  1  1  1 16  1  1  0  0  0  0  0]\n",
            " [ 1  1  1  1  1  1  0  0  0  0  0  0]]\n",
            "\n",
            "Padded testing shape: (3, 12)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0PwuWgt4ZvQ",
        "colab_type": "code",
        "outputId": "9ad3a61d-05f3-4c53-d942-e952305c5d6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "for x, y in zip(test_data, test_padded):\n",
        "  print('{} -> {}'.format(x, y))\n",
        "\n",
        "print(\"\\nWord index (for reference):\", word_index)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "এই অ্যানিমেশন আমরা করবো আমাদের পিসিতে। -> [25  1  1  1  5  1  0  0  0  0  0  0]\n",
            "সরাসরি চালান নিচের লিংক থেকে, হচ্ছে তো? -> [ 1  1  1  1 16  1  1  0  0  0  0  0]\n",
            "পাল্টান প্যারামিটার, চালান নিজের মতো করে। -> [1 1 1 1 1 1 0 0 0 0 0 0]\n",
            "\n",
            "Word index (for reference): {'<UNK>': 1, 'ডেটা': 2, 'কিভাবে': 3, 'কথা': 4, 'আমাদের': 5, 'আচ্ছা': 6, 'বলে': 7, 'পড়ছিলাম': 8, 'হান্স': 9, 'রোসলিং': 10, 'এর': 11, 'একটা': 12, 'বই': 13, 'ফ্যাক্টফুলনেস': 14, 'ধারণা': 15, 'থেকে': 16, 'নয়': 17, 'বরং': 18, 'ডেটাকে': 19, 'বলতে': 20, 'দিলে': 21, 'সব': 22, 'বিপদ': 23, 'কাটবে': 24, 'এই': 25, 'লোক': 26, 'পৃথিবীকে': 27, 'দেখিয়েছিলেন': 28, '২০০': 29, 'বছরের': 30, 'বাঁচার': 31, 'সময়': 32, 'বাড়িয়েছে': 33}\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}