doc2vec.py

# Doc2Vec Model
# ---------------------------------------
#
# In this example, we will download and preprocess the movie
#  review data.
#
# From this data set we will compute/fit a Doc2Vec model to get
# Document vectors.  From these document vectors, we will split the
# documents into train/test and use these doc vectors to do sentiment
# analysis on the movie review dataset.
#
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import random
import os
import pickle
import string
import requests
import collections
import io
import tarfile
import urllib.request
import text_helpers
from nltk.corpus import stopwords
from tensorflow.python.framework import ops

ops.reset_default_graph()
os.chdir(os.path.dirname(os.path.realpath(__file__)))

# Make a saving directory if it doesn't exist
data_folder_name = 'temp'
if not os.path.exists(data_folder_name):
    os.makedirs(data_folder_name)

# Start a graph session
sess = tf.Session()

# Declare model parameters
batch_size = 500
vocabulary_size = 7500
generations = 100000
# generations = 100
model_learning_rate = 0.001

embedding_size = 200  # Word embedding size
doc_embedding_size = 100  # Document embedding size
concatenated_size = embedding_size + doc_embedding_size

num_sampled = int(batch_size / 2)  # Number of negative examples to sample.
window_size = 3  # How many words to consider to the left.

# Add checkpoints to training
save_embeddings_every = 5000
print_valid_every = 5000
print_loss_every = 100

# Declare stop words
# stops = stopwords.words('english')
stops = []

# We pick a few test words for validation.
# valid_words = ['love', 'hate', 'happy', 'sad', 'man', 'woman']
# Later we will have to transform these into indices
valid_words = ['love', 'man']

# Load the movie review data
print('Loading Data')
# texts, target = text_helpers.load_movie_data(data_folder_name)
texts, target = text_helpers.load_product_data()

# Normalize text
print('Normalizing Text Data')
texts = text_helpers.normalize_text(texts, stops)

# Texts must contain at least 3 words
target = [target[ix] for ix, x in enumerate(texts) if len(x.split()) > window_size]
texts = [x for x in texts if len(x.split()) > window_size]
assert (len(target) == len(texts))

# Build our data set and dictionaries
print('Creating Dictionary')
word_dictionary = text_helpers.build_dictionary(texts, vocabulary_size)
word_dictionary_rev = dict(zip(word_dictionary.values(), word_dictionary.keys()))
text_data = text_helpers.text_to_numbers(texts, word_dictionary)

# Get validation word keys
valid_examples = [word_dictionary[x] for x in valid_words]

print('Creating Model')
# Define Embeddings:
embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
doc_embeddings = tf.Variable(tf.random_uniform([len(texts), doc_embedding_size], -1.0, 1.0))

# NCE loss parameters
nce_weights = tf.Variable(tf.truncated_normal([vocabulary_size, concatenated_size],
                                              stddev=1.0 / np.sqrt(concatenated_size)))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

# Create data/target placeholders
x_inputs = tf.placeholder(tf.int32, shape=[None, window_size + 1])  # plus 1 for doc index
y_target = tf.placeholder(tf.int32, shape=[None, 1])
valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

# Lookup the word embedding
# Add together element embeddings in window:
embed = tf.zeros([batch_size, embedding_size])
for element in range(window_size):
    embed += tf.nn.embedding_lookup(embeddings, x_inputs[:, element])

doc_indices = tf.slice(x_inputs, [0, window_size], [batch_size, 1])
doc_embed = tf.nn.embedding_lookup(doc_embeddings, doc_indices)


print("done", embed, doc_embed, tf.squeeze(doc_embed))
# concatenate embeddings
final_embed = tf.concat([embed, tf.squeeze(doc_embed)], 1)

# Get loss from prediction
loss = tf.reduce_mean(tf.nn.nce_loss(nce_weights, nce_biases, y_target, final_embed,
                                     num_sampled, vocabulary_size))

# Create optimizer
optimizer = tf.train.GradientDescentOptimizer(learning_rate=model_learning_rate)
train_step = optimizer.minimize(loss)

# Cosine similarity between words
norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
normalized_embeddings = embeddings / norm
valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
similarity = tf.matmul(valid_embeddings, normalized_embeddings, transpose_b=True)

# Create model saving operation
saver = tf.train.Saver({"embeddings": embeddings, "doc_embeddings": doc_embeddings})

# Add variable initializer.
init = tf.initialize_all_variables()
sess.run(init)
# model_checkpoint_path = os.path.join(os.getcwd(), data_folder_name, 'doc2vec_movie_embeddings.ckpt')
# saver.restore(sess, model_checkpoint_path)

# Run the skip gram model.
print('Starting Training')
loss_vec = []
loss_x_vec = []

model_init = True
if model_init:
    for i in range(generations):
        batch_inputs, batch_labels = text_helpers.generate_batch_data(text_data, batch_size,
                                                                      window_size, method='doc2vec')
        feed_dict = {x_inputs: batch_inputs, y_target: batch_labels}

        # Run the train step
        sess.run(train_step, feed_dict=feed_dict)

        # Return the loss
        if (i + 1) % print_loss_every == 0:
            loss_val = sess.run(loss, feed_dict=feed_dict)
            loss_vec.append(loss_val)
            loss_x_vec.append(i + 1)
            print('Loss at step {} : {}'.format(i + 1, loss_val))

        # Validation: Print some random words and top 5 related words
        if (i + 1) % print_valid_every == 0:
            sim = sess.run(similarity, feed_dict=feed_dict)
            for j in range(len(valid_words)):
                valid_word = word_dictionary_rev[valid_examples[j]]
                top_k = 5  # number of nearest neighbors
                nearest = (-sim[j, :]).argsort()[1:top_k + 1]
                log_str = "Nearest to {}:".format(valid_word)
                for k in range(top_k):
                    close_word = word_dictionary_rev[nearest[k]]
                    log_str = '{} {},'.format(log_str, close_word)
                print(log_str)

        # Save dictionary + embeddings
        if (i + 1) % save_embeddings_every == 0:
            # Save vocabulary dictionary
            with open(os.path.join(data_folder_name, 'movie_vocab.pkl'), 'wb') as f:
                pickle.dump(word_dictionary, f)

            # Save embeddings
            model_checkpoint_path = os.path.join(os.getcwd(), data_folder_name, 'doc2vec_movie_embeddings.ckpt')
            save_path = saver.save(sess, model_checkpoint_path)
            print('Model saved in file: {}'.format(save_path))
        # print(i)


docs = sess.run(doc_embeddings)
# print(len(texts), len(docs), len(docs[0]), len(texts), len(texts[0]))
# saver.restore(sess, model_checkpoint_path)

# Start logistic model-------------------------
max_words = 20
logistic_batch_size = 500

# Split dataset into train and test sets
# Need to keep the indices sorted to keep track of document index
# train_indices = np.sort(np.random.choice(len(target), round(0.9 * len(target)), replace=False))
# test_indices = np.sort(np.array(list(set(range(len(target))) - set(train_indices))))
# texts_train = [x for ix, x in enumerate(texts) if ix in train_indices]
# texts_test = [x for ix, x in enumerate(texts) if ix in test_indices]
# target_train = np.array([x for ix, x in enumerate(target) if ix in train_indices])
# target_test = np.array([x for ix, x in enumerate(target) if ix in test_indices])
#
# # Convert texts to lists of indices
# text_data_train = np.array(text_helpers.text_to_numbers(texts_train, word_dictionary))
# text_data_test = np.array(text_helpers.text_to_numbers(texts_test, word_dictionary))
#
# # Pad/crop movie reviews to specific length
# text_data_train = np.array([x[0:max_words] for x in [y + [0] * max_words for y in text_data_train]])
# text_data_test = np.array([x[0:max_words] for x in [y + [0] * max_words for y in text_data_test]])

# print(target)
print("starting")
# from sklearn.model_selection import train_test_split
# X_train, X_test, y_train, y_test = train_test_split(docs, target, test_size=0.2, random_state=42, shuffle=False)
# X_train = np.array(X_train)
# y_train = np.array(y_train)
# X_test = np.array(X_test)
# y_test = np.array(y_test)

docs = np.array(docs)
target = np.array(target)

ratio = int( docs.shape[0]/10 )
X_train = docs[ratio:,:]
X_test = docs[:ratio,:]
y_train = target[ratio:,:]
y_test = target[:ratio,:]


print(len(X_train), len(X_train[0]), len(y_train), len(y_train[0]))

from tensorflow import keras

model = keras.Sequential()
activation_function = keras.layers.PReLU()

# Input Layer
model.add(keras.layers.Dense(405, kernel_initializer=keras.initializers.random_uniform, input_dim=doc_embedding_size,
                             kernel_regularizer=keras.regularizers.l2(0.01),
                             bias_regularizer=keras.regularizers.l2(0.01), name="input"))
model.add(activation_function)

# Hidden Layer
model.add(keras.layers.Dense(405, kernel_initializer='normal', name="hidden1"))
model.add(activation_function)
model.add(keras.layers.Dropout(0.2))

model.add(keras.layers.Dense(405, kernel_initializer='normal', name="hidden2"))
model.add(activation_function)
model.add(keras.layers.Dropout(0.2))

# Output Layer
model.add(keras.layers.Dense(4, kernel_initializer='normal', name="output", activation="relu"))

model.compile(loss='mean_absolute_error', optimizer="Adam", metrics=['mean_absolute_error'])

checkpoint = keras.callbacks.ModelCheckpoint("./model/best.h5", monitor='val_loss', verbose=1, save_best_only=True,
                                                 mode='auto')
callbacks_list = []

# model = create_neural_model()
model.fit(X_train, y_train, epochs=generations, batch_size=256, validation_split=0.1, callbacks=callbacks_list)
print("Current one: ", model.evaluate(X_test, y_test, batch_size=256))

# # Define Logistic placeholders
# log_x_inputs = tf.placeholder(tf.int32, shape=[None, max_words + 1])  # plus 1 for doc index
# log_y_target = tf.placeholder(tf.int32, shape=[None, 1])
#
# # Define logistic embedding lookup (needed if we have two different batch sizes)
# # Add together element embeddings in window:
# log_embed = tf.zeros([logistic_batch_size, embedding_size])
# for element in range(max_words):
#     log_embed += tf.nn.embedding_lookup(embeddings, log_x_inputs[:, element])
#
# log_doc_indices = tf.slice(log_x_inputs, [0, max_words], [logistic_batch_size, 1])
# log_doc_embed = tf.nn.embedding_lookup(doc_embeddings, log_doc_indices)
#
# # concatenate embeddings
# log_final_embed = tf.concat([log_embed, tf.squeeze(log_doc_embed)], 1)
#
# # Define model:
# # Create variables for logistic regression
# A = tf.Variable(tf.random_normal(shape=[concatenated_size, 1]))
# b = tf.Variable(tf.random_normal(shape=[1, 1]))
#
# # Declare logistic model (sigmoid in loss function)
# model_output = tf.add(tf.matmul(log_final_embed, A), b)
#
# # Declare loss function (Cross Entropy loss)
# logistic_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=model_output, labels=tf.cast(log_y_target,
#                                                                                               tf.float32)))
#
# # Actual Prediction
# prediction = tf.round(tf.sigmoid(model_output))
# predictions_correct = tf.cast(tf.equal(prediction, tf.cast(log_y_target, tf.float32)), tf.float32)
# accuracy = tf.reduce_mean(predictions_correct)
#
# # Declare optimizer
# logistic_opt = tf.train.GradientDescentOptimizer(learning_rate=0.01)
# logistic_train_step = logistic_opt.minimize(logistic_loss, var_list=[A, b])
#
# # Intitialize Variables
# init = tf.initialize_all_variables()
# sess.run(init)
#
# # Start Logistic Regression
# print('Starting Logistic Doc2Vec Model Training')
# train_loss = []
# test_loss = []
# train_acc = []
# test_acc = []
# i_data = []
# for i in range(10000):
#     rand_index = np.random.choice(text_data_train.shape[0], size=logistic_batch_size)
#     rand_x = text_data_train[rand_index]
#     # Append review index at the end of text data
#     rand_x_doc_indices = train_indices[rand_index]
#     rand_x = np.hstack((rand_x, np.transpose([rand_x_doc_indices])))
#     rand_y = np.transpose([target_train[rand_index]])
#
#     feed_dict = {log_x_inputs: rand_x, log_y_target: rand_y}
#     sess.run(logistic_train_step, feed_dict=feed_dict)
#
#     # Only record loss and accuracy every 100 generations
#     if (i + 1) % 100 == 0:
#         rand_index_test = np.random.choice(text_data_test.shape[0], size=logistic_batch_size)
#         rand_x_test = text_data_test[rand_index_test]
#         # Append review index at the end of text data
#         rand_x_doc_indices_test = test_indices[rand_index_test]
#         rand_x_test = np.hstack((rand_x_test, np.transpose([rand_x_doc_indices_test])))
#         rand_y_test = np.transpose([target_test[rand_index_test]])
#
#         test_feed_dict = {log_x_inputs: rand_x_test, log_y_target: rand_y_test}
#
#         i_data.append(i + 1)
#
#         train_loss_temp = sess.run(logistic_loss, feed_dict=feed_dict)
#         train_loss.append(train_loss_temp)
#
#         test_loss_temp = sess.run(logistic_loss, feed_dict=test_feed_dict)
#         test_loss.append(test_loss_temp)
#
#         train_acc_temp = sess.run(accuracy, feed_dict=feed_dict)
#         train_acc.append(train_acc_temp)
#
#         test_acc_temp = sess.run(accuracy, feed_dict=test_feed_dict)
#         test_acc.append(test_acc_temp)
#     if (i + 1) % 500 == 0:
#         acc_and_loss = [i + 1, train_loss_temp, test_loss_temp, train_acc_temp, test_acc_temp]
#         acc_and_loss = [np.round(x, 2) for x in acc_and_loss]
#         print('Generation # {}. Train Loss (Test Loss): {:.2f} ({:.2f}). Train Acc (Test Acc): {:.2f} ({:.2f})'.format(
#             *acc_and_loss))
#
# # Plot loss over time
# plt.plot(i_data, train_loss, 'k-', label='Train Loss')
# plt.plot(i_data, test_loss, 'r--', label='Test Loss', linewidth=4)
# plt.title('Cross Entropy Loss per Generation')
# plt.xlabel('Generation')
# plt.ylabel('Cross Entropy Loss')
# plt.legend(loc='upper right')
# plt.show()
#
# # Plot train and test accuracy
# plt.plot(i_data, train_acc, 'k-', label='Train Set Accuracy')
# plt.plot(i_data, test_acc, 'r--', label='Test Set Accuracy', linewidth=4)
# plt.title('Train and Test Accuracy')
# plt.xlabel('Generation')
# plt.ylabel('Accuracy')
# plt.legend(loc='lower right')
# plt.show()