notebook/24_keras_text_generation_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# small LSTM Network to Generate Text for Alice in Wonderland\n",
    "import sys\n",
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Total Characters:  147773\n",
      "Total Vocab:  49\n"
     ]
    }
   ],
   "source": [
    "# load ascii text and covert to lowercase\n",
    "filename = \"wonderland.txt\"\n",
    "raw_text = open(filename).read()\n",
    "raw_text = raw_text.lower()\n",
    "\n",
    "# create mapping of unique chars to integers\n",
    "chars = sorted(list(set(raw_text)))\n",
    "char_to_int = dict((c, i) for i, c in enumerate(chars))\n",
    "int_to_char = dict((i, c) for i, c in enumerate(chars))\n",
    "\n",
    "# summarize the loaded data\n",
    "n_chars = len(raw_text)\n",
    "n_vocab = len(chars)\n",
    "print \"Total Characters: \", n_chars\n",
    "print \"Total Vocab: \", n_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Patterns:  147673\n"
     ]
    }
   ],
   "source": [
    "# prepare the dataset of input to output pairs encoded as integers\n",
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in range(0, n_chars - seq_length, 1):\n",
    "    seq_in = raw_text[i:i + seq_length]\n",
    "    seq_out = raw_text[i + seq_length]\n",
    "    dataX.append([char_to_int[char] for char in seq_in])\n",
    "    dataY.append(char_to_int[seq_out])\n",
    "\n",
    "n_patterns = len(dataX)\n",
    "print \"Total Patterns: \", n_patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(147673, 100, 1)\n"
     ]
    }
   ],
   "source": [
    "# reshape X to be [samples, time steps, features]\n",
    "X = numpy.reshape(dataX, (n_patterns, seq_length, 1))\n",
    "\n",
    "print X.shape\n",
    "\n",
    "# normalize\n",
    "X = X / float(n_vocab)\n",
    "\n",
    "# one hot encode the output variable\n",
    "y = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.9560Epoch 00000: loss improved from inf to 2.95598, saving model to weights-improvement-00-2.9560.hdf5\n",
      "147673/147673 [==============================] - 977s - loss: 2.9560   \n",
      "Epoch 2/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.7466Epoch 00001: loss improved from 2.95598 to 2.74656, saving model to weights-improvement-01-2.7466.hdf5\n",
      "147673/147673 [==============================] - 974s - loss: 2.7466   \n",
      "Epoch 3/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.6333Epoch 00002: loss improved from 2.74656 to 2.63322, saving model to weights-improvement-02-2.6332.hdf5\n",
      "147673/147673 [==============================] - 975s - loss: 2.6332   \n",
      "Epoch 4/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.5487Epoch 00003: loss improved from 2.63322 to 2.54852, saving model to weights-improvement-03-2.5485.hdf5\n",
      "147673/147673 [==============================] - 973s - loss: 2.5485   \n",
      "Epoch 5/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.4743Epoch 00004: loss improved from 2.54852 to 2.47418, saving model to weights-improvement-04-2.4742.hdf5\n",
      "147673/147673 [==============================] - 970s - loss: 2.4742   \n",
      "Epoch 6/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.4169Epoch 00005: loss improved from 2.47418 to 2.41682, saving model to weights-improvement-05-2.4168.hdf5\n",
      "147673/147673 [==============================] - 977s - loss: 2.4168   \n",
      "Epoch 7/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.3678Epoch 00006: loss improved from 2.41682 to 2.36773, saving model to weights-improvement-06-2.3677.hdf5\n",
      "147673/147673 [==============================] - 984s - loss: 2.3677   \n",
      "Epoch 8/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.3263Epoch 00007: loss improved from 2.36773 to 2.32622, saving model to weights-improvement-07-2.3262.hdf5\n",
      "147673/147673 [==============================] - 980s - loss: 2.3262   \n",
      "Epoch 9/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.2808Epoch 00008: loss improved from 2.32622 to 2.28075, saving model to weights-improvement-08-2.2808.hdf5\n",
      "147673/147673 [==============================] - 972s - loss: 2.2808   \n",
      "Epoch 10/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.2447Epoch 00009: loss improved from 2.28075 to 2.24465, saving model to weights-improvement-09-2.2447.hdf5\n",
      "147673/147673 [==============================] - 971s - loss: 2.2447   \n",
      "Epoch 11/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.2044Epoch 00010: loss improved from 2.24465 to 2.20465, saving model to weights-improvement-10-2.2046.hdf5\n",
      "147673/147673 [==============================] - 971s - loss: 2.2046   \n",
      "Epoch 12/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.1748Epoch 00011: loss improved from 2.20465 to 2.17463, saving model to weights-improvement-11-2.1746.hdf5\n",
      "147673/147673 [==============================] - 975s - loss: 2.1746   \n",
      "Epoch 13/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.1395Epoch 00012: loss improved from 2.17463 to 2.13941, saving model to weights-improvement-12-2.1394.hdf5\n",
      "147673/147673 [==============================] - 971s - loss: 2.1394   \n",
      "Epoch 14/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.1068Epoch 00013: loss improved from 2.13941 to 2.10678, saving model to weights-improvement-13-2.1068.hdf5\n",
      "147673/147673 [==============================] - 971s - loss: 2.1068   \n",
      "Epoch 15/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.0734Epoch 00014: loss improved from 2.10678 to 2.07346, saving model to weights-improvement-14-2.0735.hdf5\n",
      "147673/147673 [==============================] - 971s - loss: 2.0735   \n",
      "Epoch 16/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.0450Epoch 00015: loss improved from 2.07346 to 2.04500, saving model to weights-improvement-15-2.0450.hdf5\n",
      "147673/147673 [==============================] - 978s - loss: 2.0450   \n",
      "Epoch 17/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 2.0157Epoch 00016: loss improved from 2.04500 to 2.01562, saving model to weights-improvement-16-2.0156.hdf5\n",
      "147673/147673 [==============================] - 980s - loss: 2.0156   \n",
      "Epoch 18/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 1.9886Epoch 00017: loss improved from 2.01562 to 1.98852, saving model to weights-improvement-17-1.9885.hdf5\n",
      "147673/147673 [==============================] - 972s - loss: 1.9885   \n",
      "Epoch 19/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 1.9637Epoch 00018: loss improved from 1.98852 to 1.96371, saving model to weights-improvement-18-1.9637.hdf5\n",
      "147673/147673 [==============================] - 974s - loss: 1.9637   \n",
      "Epoch 20/20\n",
      "147584/147673 [============================>.] - ETA: 0s - loss: 1.9375Epoch 00019: loss improved from 1.96371 to 1.93767, saving model to weights-improvement-19-1.9377.hdf5\n",
      "147673/147673 [==============================] - 984s - loss: 1.9377   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x124470910>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "# define the checkpoint\n",
    "filepath=\"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "# fit the model\n",
    "model.fit(X, y, nb_epoch=20, batch_size=128, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" at?' thought alice to herself.\n",
      "\n",
      "'of the mushroom,' said the caterpillar, just as if she had asked  \"\n",
      "and\n",
      "berute an in sas oo toe thneg harten ano ohe to the tabbit saabit  soode and the tai oh the tabbit saseere the was toe tiieg her head the\n",
      "hert an the could. \n",
      "\n",
      "'i whanl sas a little 'laseer,' said the kante 'int io a long tare to\n",
      "an in the tai shan i \n",
      "and yhu  toene to the thet '\n",
      "\n",
      "the dormouse sea whit so ho a lintle oo the sooee aeain, and thete\n",
      "tas nor inre the was a little sooee an the cad bedn the wanted the \n",
      "aeren th the tas a little sooe  a tirtll of the tuoen, and then she whs\n",
      "low lene the madt of the \n",
      "aoucdrse to tey at the cad teten hnr her  a finde of thing sare tiiee an\n",
      "the rabte was oo the tabbit  aadute in \n",
      "the rast was i van g lent of \n",
      "thete shen io the same to tee\n",
      "a aota                                                                                                                                                                                                                                                                                           \n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "\n",
    "# load the network weights\n",
    "filename = \"weights-improvement-19-1.9377.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print \"Seed:\"\n",
    "print \"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\"\n",
    "\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "    x = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "    x = x / float(n_vocab)\n",
    "    prediction = model.predict(x, verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print \"\\nDone.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 2.8185Epoch 00000: loss improved from inf to 2.81843, saving model to weights-improvement-00-2.8184-bigger.hdf5\n",
      "147673/147673 [==============================] - 2609s - loss: 2.8184  \n",
      "Epoch 2/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 2.4504Epoch 00001: loss improved from 2.81843 to 2.45029, saving model to weights-improvement-01-2.4503-bigger.hdf5\n",
      "147673/147673 [==============================] - 3224s - loss: 2.4503  \n",
      "Epoch 3/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 2.2569Epoch 00002: loss improved from 2.45029 to 2.25687, saving model to weights-improvement-02-2.2569-bigger.hdf5\n",
      "147673/147673 [==============================] - 2497s - loss: 2.2569  \n",
      "Epoch 4/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 2.1200Epoch 00003: loss improved from 2.25687 to 2.12005, saving model to weights-improvement-03-2.1200-bigger.hdf5\n",
      "147673/147673 [==============================] - 2500s - loss: 2.1200  \n",
      "Epoch 5/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 2.0132Epoch 00004: loss improved from 2.12005 to 2.01317, saving model to weights-improvement-04-2.0132-bigger.hdf5\n",
      "147673/147673 [==============================] - 2500s - loss: 2.0132  \n",
      "Epoch 6/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.9329Epoch 00005: loss improved from 2.01317 to 1.93286, saving model to weights-improvement-05-1.9329-bigger.hdf5\n",
      "147673/147673 [==============================] - 2503s - loss: 1.9329  \n",
      "Epoch 7/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.8668Epoch 00006: loss improved from 1.93286 to 1.86676, saving model to weights-improvement-06-1.8668-bigger.hdf5\n",
      "147673/147673 [==============================] - 2509s - loss: 1.8668  \n",
      "Epoch 8/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.8131Epoch 00007: loss improved from 1.86676 to 1.81328, saving model to weights-improvement-07-1.8133-bigger.hdf5\n",
      "147673/147673 [==============================] - 2504s - loss: 1.8133  \n",
      "Epoch 9/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.7632Epoch 00008: loss improved from 1.81328 to 1.76312, saving model to weights-improvement-08-1.7631-bigger.hdf5\n",
      "147673/147673 [==============================] - 2505s - loss: 1.7631  \n",
      "Epoch 10/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.7193Epoch 00009: loss improved from 1.76312 to 1.71942, saving model to weights-improvement-09-1.7194-bigger.hdf5\n",
      "147673/147673 [==============================] - 2520s - loss: 1.7194  \n",
      "Epoch 11/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.6797Epoch 00010: loss improved from 1.71942 to 1.67978, saving model to weights-improvement-10-1.6798-bigger.hdf5\n",
      "147673/147673 [==============================] - 2507s - loss: 1.6798  \n",
      "Epoch 12/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.6500Epoch 00011: loss improved from 1.67978 to 1.65006, saving model to weights-improvement-11-1.6501-bigger.hdf5\n",
      "147673/147673 [==============================] - 2510s - loss: 1.6501  \n",
      "Epoch 13/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.6135Epoch 00012: loss improved from 1.65006 to 1.61350, saving model to weights-improvement-12-1.6135-bigger.hdf5\n",
      "147673/147673 [==============================] - 2504s - loss: 1.6135  \n",
      "Epoch 14/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.5831Epoch 00013: loss improved from 1.61350 to 1.58312, saving model to weights-improvement-13-1.5831-bigger.hdf5\n",
      "147673/147673 [==============================] - 2507s - loss: 1.5831  \n",
      "Epoch 15/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.5560Epoch 00014: loss improved from 1.58312 to 1.55596, saving model to weights-improvement-14-1.5560-bigger.hdf5\n",
      "147673/147673 [==============================] - 2506s - loss: 1.5560  \n",
      "Epoch 16/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.5329Epoch 00015: loss improved from 1.55596 to 1.53298, saving model to weights-improvement-15-1.5330-bigger.hdf5\n",
      "147673/147673 [==============================] - 2507s - loss: 1.5330  \n",
      "Epoch 17/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.5111Epoch 00016: loss improved from 1.53298 to 1.51107, saving model to weights-improvement-16-1.5111-bigger.hdf5\n",
      "147673/147673 [==============================] - 2503s - loss: 1.5111  \n",
      "Epoch 18/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4875Epoch 00017: loss improved from 1.51107 to 1.48756, saving model to weights-improvement-17-1.4876-bigger.hdf5\n",
      "147673/147673 [==============================] - 2509s - loss: 1.4876  \n",
      "Epoch 19/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4671Epoch 00018: loss improved from 1.48756 to 1.46710, saving model to weights-improvement-18-1.4671-bigger.hdf5\n",
      "147673/147673 [==============================] - 2507s - loss: 1.4671  \n",
      "Epoch 20/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4489Epoch 00019: loss improved from 1.46710 to 1.44890, saving model to weights-improvement-19-1.4489-bigger.hdf5\n",
      "147673/147673 [==============================] - 2507s - loss: 1.4489  \n",
      "Epoch 21/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4329Epoch 00020: loss improved from 1.44890 to 1.43291, saving model to weights-improvement-20-1.4329-bigger.hdf5\n",
      "147673/147673 [==============================] - 2513s - loss: 1.4329  \n",
      "Epoch 22/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4184Epoch 00021: loss improved from 1.43291 to 1.41841, saving model to weights-improvement-21-1.4184-bigger.hdf5\n",
      "147673/147673 [==============================] - 2512s - loss: 1.4184  \n",
      "Epoch 23/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.4028Epoch 00022: loss improved from 1.41841 to 1.40275, saving model to weights-improvement-22-1.4027-bigger.hdf5\n",
      "147673/147673 [==============================] - 2512s - loss: 1.4027  \n",
      "Epoch 24/50\n",
      "147648/147673 [============================>.] - ETA: 4s - loss: 1.3876 Epoch 00023: loss improved from 1.40275 to 1.38760, saving model to weights-improvement-23-1.3876-bigger.hdf5\n",
      "147673/147673 [==============================] - 25989s - loss: 1.3876 \n",
      "Epoch 25/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3769Epoch 00024: loss improved from 1.38760 to 1.37692, saving model to weights-improvement-24-1.3769-bigger.hdf5\n",
      "147673/147673 [==============================] - 2465s - loss: 1.3769  \n",
      "Epoch 26/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3682Epoch 00025: loss improved from 1.37692 to 1.36812, saving model to weights-improvement-25-1.3681-bigger.hdf5\n",
      "147673/147673 [==============================] - 2520s - loss: 1.3681  \n",
      "Epoch 27/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3537Epoch 00026: loss improved from 1.36812 to 1.35363, saving model to weights-improvement-26-1.3536-bigger.hdf5\n",
      "147673/147673 [==============================] - 2468s - loss: 1.3536  \n",
      "Epoch 28/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3452Epoch 00027: loss improved from 1.35363 to 1.34518, saving model to weights-improvement-27-1.3452-bigger.hdf5\n",
      "147673/147673 [==============================] - 2472s - loss: 1.3452  \n",
      "Epoch 29/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3343Epoch 00028: loss improved from 1.34518 to 1.33437, saving model to weights-improvement-28-1.3344-bigger.hdf5\n",
      "147673/147673 [==============================] - 2469s - loss: 1.3344  \n",
      "Epoch 30/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3264Epoch 00029: loss improved from 1.33437 to 1.32641, saving model to weights-improvement-29-1.3264-bigger.hdf5\n",
      "147673/147673 [==============================] - 2471s - loss: 1.3264  \n",
      "Epoch 31/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3180Epoch 00030: loss improved from 1.32641 to 1.31808, saving model to weights-improvement-30-1.3181-bigger.hdf5\n",
      "147673/147673 [==============================] - 2476s - loss: 1.3181  \n",
      "Epoch 32/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3080Epoch 00031: loss improved from 1.31808 to 1.30802, saving model to weights-improvement-31-1.3080-bigger.hdf5\n",
      "147673/147673 [==============================] - 2475s - loss: 1.3080  \n",
      "Epoch 33/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.3006Epoch 00032: loss improved from 1.30802 to 1.30061, saving model to weights-improvement-32-1.3006-bigger.hdf5\n",
      "147673/147673 [==============================] - 2475s - loss: 1.3006  \n",
      "Epoch 34/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2960Epoch 00033: loss improved from 1.30061 to 1.29597, saving model to weights-improvement-33-1.2960-bigger.hdf5\n",
      "147673/147673 [==============================] - 2482s - loss: 1.2960  \n",
      "Epoch 35/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2890Epoch 00034: loss improved from 1.29597 to 1.28904, saving model to weights-improvement-34-1.2890-bigger.hdf5\n",
      "147673/147673 [==============================] - 2482s - loss: 1.2890  \n",
      "Epoch 36/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2836Epoch 00035: loss improved from 1.28904 to 1.28356, saving model to weights-improvement-35-1.2836-bigger.hdf5\n",
      "147673/147673 [==============================] - 2475s - loss: 1.2836  \n",
      "Epoch 37/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2762Epoch 00036: loss improved from 1.28356 to 1.27629, saving model to weights-improvement-36-1.2763-bigger.hdf5\n",
      "147673/147673 [==============================] - 2476s - loss: 1.2763  \n",
      "Epoch 38/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2741Epoch 00037: loss improved from 1.27629 to 1.27417, saving model to weights-improvement-37-1.2742-bigger.hdf5\n",
      "147673/147673 [==============================] - 2474s - loss: 1.2742  \n",
      "Epoch 39/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2696Epoch 00038: loss improved from 1.27417 to 1.26959, saving model to weights-improvement-38-1.2696-bigger.hdf5\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2696  \n",
      "Epoch 40/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2647Epoch 00039: loss improved from 1.26959 to 1.26466, saving model to weights-improvement-39-1.2647-bigger.hdf5\n",
      "147673/147673 [==============================] - 2476s - loss: 1.2647  \n",
      "Epoch 41/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2565Epoch 00040: loss improved from 1.26466 to 1.25653, saving model to weights-improvement-40-1.2565-bigger.hdf5\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2565  \n",
      "Epoch 42/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2588Epoch 00041: loss did not improve\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2588  \n",
      "Epoch 43/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2531Epoch 00042: loss improved from 1.25653 to 1.25303, saving model to weights-improvement-42-1.2530-bigger.hdf5\n",
      "147673/147673 [==============================] - 2476s - loss: 1.2530  \n",
      "Epoch 44/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2488Epoch 00043: loss improved from 1.25303 to 1.24881, saving model to weights-improvement-43-1.2488-bigger.hdf5\n",
      "147673/147673 [==============================] - 2475s - loss: 1.2488  \n",
      "Epoch 45/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2425Epoch 00044: loss improved from 1.24881 to 1.24247, saving model to weights-improvement-44-1.2425-bigger.hdf5\n",
      "147673/147673 [==============================] - 2475s - loss: 1.2425  \n",
      "Epoch 46/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2399Epoch 00045: loss improved from 1.24247 to 1.23989, saving model to weights-improvement-45-1.2399-bigger.hdf5\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2399  \n",
      "Epoch 47/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2383Epoch 00046: loss improved from 1.23989 to 1.23831, saving model to weights-improvement-46-1.2383-bigger.hdf5\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2383  \n",
      "Epoch 48/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2336Epoch 00047: loss improved from 1.23831 to 1.23354, saving model to weights-improvement-47-1.2335-bigger.hdf5\n",
      "147673/147673 [==============================] - 2477s - loss: 1.2335  \n",
      "Epoch 49/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2319Epoch 00048: loss improved from 1.23354 to 1.23191, saving model to weights-improvement-48-1.2319-bigger.hdf5\n",
      "147673/147673 [==============================] - 2476s - loss: 1.2319  \n",
      "Epoch 50/50\n",
      "147648/147673 [============================>.] - ETA: 0s - loss: 1.2319Epoch 00049: loss did not improve\n",
      "147673/147673 [==============================] - 2480s - loss: 1.2319  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x137595bd0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(LSTM(256))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "# define the checkpoint\n",
    "filepath=\"weights-improvement-{epoch:02d}-{loss:.4f}-bigger.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]\n",
    "\n",
    "# fit the model\n",
    "model.fit(X, y, nb_epoch=50, batch_size=64, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" goose, with the bones and the beak--\n",
      "    pray how did you manage to do it?'\n",
      "\n",
      "   'in my youth,' sa \"\n",
      "id the king, 'the marter was a little\n",
      "\n",
      "now and alice fou so the table. but the was not a lomg tur of the had\n",
      "lent the tank as the could gold in the court, and she thought the was\n",
      "\n",
      "of the thoee of the thoee of the table. 'i don't know the door tay of\n",
      "the sable about the sight wild the tiings bll the tenser on the temtence\n",
      "on the tay the other uile the was all the temtenced the tame as the\n",
      "cook of the court, and she thought the whole party she was sert hear\n",
      "\n",
      "the was teriectly as she could got some time with the tempence of the\n",
      "court, and she thought the whole party she was solething to see it out\n",
      "the was to say the thme, and the whole parcerand she was to say the\n",
      "\n",
      "it!s garden, and teem the would lane any rere\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# define the LSTM model\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2]), return_sequences=True))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(LSTM(256))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "\n",
    "# load the network weights\n",
    "filename = \"weights-improvement-48-1.2319-bigger.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "\n",
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print \"Seed:\"\n",
    "print \"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\"\n",
    "\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "    x = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "    x = x / float(n_vocab)\n",
    "    prediction = model.predict(x, verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print \"\\nDone.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}