semeval.py

# -*- coding: utf-8 -*-
"""SemEval2020-Task5-Subtask-1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/lenyabloko/SemEval2020/blob/master/SemEval2020_Task5_Subtask_1.ipynb

UPLOAD FILES - Place [train.csv](https://github.com/arielsho/Subtask-1/archive/master.zip) and [test.csv](https://github.com/arielsho/Subtask-1-test/archive/master.zip) files directly under your `gdrive/My Drive/Subtask-1/`, before starting (follow the prompt URL and get authentication token)
"""

from __future__ import absolute_import, division, print_function

from utils import *

args = {
    'data_dir': '',
    'model_type': 'roberta',
    'model_name': 'roberta-base',
    'task_name': 'binary',
    'output_dir': 'outputs/',
    'cache_dir': 'cache/',
    'do_train': True,
    'do_eval': True,
    'fp16': False,
    'fp16_opt_level': 'O1',
    'max_seq_length': 256,
    'output_mode': 'classification',
    'train_batch_size': 32,
    'eval_batch_size': 32,

    'gradient_accumulation_steps': 1,
    'num_train_epochs': 4,
    'weight_decay': 0,
    'learning_rate': 4e-5,
    'adam_epsilon': 1e-8,
    'warmup_steps': 0,
    'max_grad_norm': 1.0,

    'logging_steps': 50,
    'evaluate_during_training': False,
    'save_steps': 2000,
    'eval_all_checkpoints': True,

    'overwrite_output_dir': True,
    'reprocess_input_data': True,
    'notes': 'Using train.csv'
}

import json
import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)
import random
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm_notebook, trange

from pytorch_transformers import (WEIGHTS_NAME, BertConfig, BertForSequenceClassification, BertTokenizer,
                                  XLMConfig, XLMForSequenceClassification, XLMTokenizer,
                                  XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer,
                                  RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)

from pytorch_transformers import AdamW, WarmupLinearSchedule
from tensorboardX import SummaryWriter
from utils import (convert_examples_to_features,output_modes, processors)
from sklearn.metrics import mean_squared_error, matthews_corrcoef, confusion_matrix
from scipy.stats import pearsonr

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

with open('args.json', 'w') as f:
    json.dump(args, f)
if os.path.exists(args['output_dir']) and os.listdir(args['output_dir']) and args['do_train'] and not args['overwrite_output_dir']:
    raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args['output_dir']))


print(device)
print(os.cpu_count()-2)


def load_and_cache_examples(task, tokenizer, dataset=True, evaluate=True):
    processor = processors[task]()
    output_mode = args['output_mode']

    mode = 'train' if train else 'test'
    cached_features_file = os.path.join('./content/', "features_{mode}_{args['model_name']}_{args['max_seq_length']}_{task}")

    if os.path.exists(cached_features_file) and not args['reprocess_input_data']:
        logger.info("Loading features from cached file %s", cached_features_file)
        features = torch.load(cached_features_file)
    else:
        logger.info("Creating features from dataset file at: / %s", './content/')
        label_list = processor.get_labels()

        if dataset:
          if not evaluate:
             logger.info("Loading train.tsv file from: / %s", './content/')
             examples = processor.get_train_examples('./content/')
          else:
             logger.info("Loading dev.tsv file from: / %s", './content/')
             examples = processor.get_dev_examples('./content/')
        else:
            logger.info("Loading test.tsv file from: / %s", './content/')
            examples = processor.get_test_examples('./content/')

        features = convert_examples_to_features(examples, label_list, args['max_seq_length'], tokenizer, output_mode,
            cls_token_at_end=bool(args['model_type'] in ['xlnet']),            # xlnet has a cls token at the end
            cls_token=tokenizer.cls_token,
            sep_token=tokenizer.sep_token,
            cls_token_segment_id=2 if args['model_type'] in ['xlnet'] else 0,
            pad_on_left=bool(args['model_type'] in ['xlnet']),                 # pad on the left for xlnet
            pad_token_segment_id=4 if args['model_type'] in ['xlnet'] else 0,
            process_count=1)

        logger.info("Saving features into cached file %s", cached_features_file)
        torch.save(features, cached_features_file)

    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
    if output_mode == "classification":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
    elif output_mode == "regression":
        all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.float)

    dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
    return dataset


def train(train_dataset, model, tokenizer):
    tb_writer = SummaryWriter()

    train_sampler = RandomSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args['train_batch_size'])

    t_total = len(train_dataloader) // args['gradient_accumulation_steps'] * args['num_train_epochs']

    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args['weight_decay']},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args['learning_rate'], eps=args['adam_epsilon'])
    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args['warmup_steps'], t_total=t_total)

    if args['fp16']:
        try:
            from apex import amp
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
        model, optimizer = amp.initialize(model, optimizer, opt_level=args['fp16_opt_level'])

    logger.info("***** Running training *****")
    logger.info("  Num examples = %d", len(train_dataset))
    logger.info("  Num Epochs = %d", args['num_train_epochs'])
    logger.info("  Total train batch size  = %d", args['train_batch_size'])
    logger.info("  Gradient Accumulation steps = %d", args['gradient_accumulation_steps'])
    logger.info("  Total optimization steps = %d", t_total)

    global_step = 0
    tr_loss, logging_loss = 0.0, 0.0
    model.zero_grad()
    train_iterator = trange(int(args['num_train_epochs']), desc="Epoch")

    for _ in train_iterator:
        epoch_iterator = tqdm_notebook(train_dataloader, desc="Iteration")
        for step, batch in enumerate(epoch_iterator):
            model.train()
            batch = tuple(t.to(device) for t in batch)
            inputs = {'input_ids':      batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,  # XLM don't use segment_ids
                      'labels':         batch[3]}
            outputs = model(**inputs)
            loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)
            print(loss)

            if args['gradient_accumulation_steps'] > 1:
                loss = loss / args['gradient_accumulation_steps']

            if args['fp16']:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args['max_grad_norm'])

            else:
                loss.backward()
                torch.nn.utils.clip_grad_norm_(model.parameters(), args['max_grad_norm'])

            tr_loss += loss.item()
            if (step + 1) % args['gradient_accumulation_steps'] == 0:
                scheduler.step()  # Update learning rate schedule
                optimizer.step()
                model.zero_grad()
                global_step += 1

                if args['logging_steps'] > 0 and global_step % args['logging_steps'] == 0:
                    # Log metrics
                    if args['evaluate_during_training']:  # Only evaluate when single GPU otherwise metrics may not average well
                        results, wrong = evaluate(model, tokenizer)
                        for key, value in results.items():
                            tb_writer.add_scalar('eval_{}'.format(key), value, global_step)
                    tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step)
                    tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args['logging_steps'], global_step)
                    logging_loss = tr_loss

                if args['save_steps'] > 0 and global_step % args['save_steps'] == 0:
                    # Save model checkpoint
                    output_dir = os.path.join(args['output_dir'], 'checkpoint-{}'.format(global_step))
                    if not os.path.exists(output_dir):
                        os.makedirs(output_dir)
                    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
                    model_to_save.save_pretrained(output_dir)
                    logger.info("Saving model checkpoint to %s", output_dir)


    return global_step, tr_loss / global_step



def get_eval_report(labels, preds, dataset):
    mismatched = labels != preds
    if dataset:
      examples = processor.get_dev_examples('./content/')
    else:
      examples = processor.get_test_examples('./content/')

    lineup = list(zip(examples, mismatched))
    mismatches = [i for (i, v) in lineup if v]

    report = []
    eval_output_dir = args['output_dir']
    output_submit_file = os.path.join(eval_output_dir, "submit_results.csv")
    with open(output_submit_file, "w") as writer:
      for example in examples:
        prediction = example.label
        if any(mismatch.sentenceId == example.sentenceId for mismatch in mismatches):
          prediction = str(int(not bool(int(prediction))))
          print("Prediction changed for sentence: "+example.sentenceId + " was "+example.label+ " now "+prediction)
        writer.write("%s,%s\n" % (example.sentenceId, prediction))

    output_report_file = os.path.join(eval_output_dir, "report_results.csv")
    with open(output_report_file, "w") as reporter:
       for mismatch in mismatches:
        reporter.write("%s,%s,%s,%s\n" % (mismatch.sentenceId, mismatch.label, mismatch.text_a, mismatch.text_b))

    submit_df = pd.read_csv(eval_output_dir + 'submit_results.csv', header=None)
    submit_df.to_csv(eval_output_dir+'subtask1.csv', index=False, header=['sentenceID', 'pred_label'])

    mcc = matthews_corrcoef(labels, preds)
    tn, fp, fn, tp = confusion_matrix(labels, preds).ravel()
    return {
        "mcc": mcc,
        "tp": tp,
        "tn": tn,
        "fp": fp,
        "fn": fn
    }, mismatches

def compute_metrics(task_name, preds, labels, dataset):
    assert len(preds) == len(labels)
    print("******* Computing metrics for "+task_name+" ********")
    return get_eval_report(labels, preds, dataset)

def evaluate(model, tokenizer, prefix="", dataset=True):
    # Loop to handle MNLI double evaluation (matched, mis-matched)
    eval_output_dir = args['output_dir']

    results = {}
    EVAL_TASK = args['task_name']

    eval_dataset = load_and_cache_examples(EVAL_TASK, tokenizer, dataset, evaluate=True)
    if not os.path.exists(eval_output_dir):
        os.makedirs(eval_output_dir)

    eval_sampler = SequentialSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args['eval_batch_size'])

    # Eval!
    logger.info("***** Running evaluation {} *****".format(prefix))
    logger.info("  Num examples = %d", len(eval_dataset))
    logger.info("  Batch size = %d", args['eval_batch_size'])
    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None

    for batch in tqdm_notebook(eval_dataloader, desc="Evaluating"):
        model.eval() # set model in evaluation mode
        batch = tuple(t.to(device) for t in batch)

        with torch.no_grad():
            inputs = {'input_ids':      batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2] if args['model_type'] in ['bert', 'xlnet'] else None,  # XLM don't use segment_ids
                      'labels':         batch[3]}

            outputs = model(**inputs) # evaluate batch using fine-tuned model loaded from checkpoint

            tmp_eval_loss, logits = outputs[:2]
            eval_loss += tmp_eval_loss.mean().item()

        nb_eval_steps += 1
        if preds is None:
            preds = logits.detach().cpu().numpy()
            out_label_ids = inputs['labels'].detach().cpu().numpy()
        else:
            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
            out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)

    #END of loop
    eval_loss = eval_loss / nb_eval_steps

    # Predictions
    if args['output_mode'] == "classification":
        preds = np.argmax(preds, axis=1)
    elif args['output_mode'] == "regression":
        preds = np.squeeze(preds)

    result, wrong = compute_metrics(EVAL_TASK, preds, out_label_ids, dataset)
    results.update(result)

    output_eval_file = os.path.join(eval_output_dir, "eval_results.txt")
    with open(output_eval_file, "w") as writer:
        logger.info("***** Eval results {} *****".format(prefix))
        for key in sorted(result.keys()):
            logger.info("  %s = %s", key, str(result[key]))
            writer.write("%s = %s\n" % (key, str(result[key])))

    return results, wrong




def main():
    import utils

    import glob
    import logging
    import os
    import random
    import json

    import pandas as pd
    import numpy as np

    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)


    PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
    TEMPLATE_DIRS = (
        os.path.normpath(os.path.join(PROJECT_ROOT, '/content')),
    )

    """FORMAT DATA  - skip, if supplied modified .tsv"""


    prefix = './content/'
    train_df = pd.read_csv(prefix + 'train.csv', header=None)
    train_df=train_df.drop(index=0)

    dev_df = pd.DataFrame({
        'id':train_df[0],
        'labels':train_df[1],
        'alpha':['a']*train_df.shape[0],
        'text': train_df[2].replace(r'\n', ' ', regex=True)
    })


    train_df = pd.DataFrame({
        'id':train_df[0],
        'labels':train_df[1],
        'alpha':['a']*train_df.shape[0],
        'text': train_df[2].replace(r'\n', ' ', regex=True)
    })


    test_df = pd.read_csv(prefix + 'test.csv', header=None)
    test_df = test_df.drop(index=0)

    test_df = pd.DataFrame({
        'id':test_df[0],
        'labels':[0]*test_df.shape[0],
        'alpha':['a']*test_df.shape[0],
        'text': test_df[1].replace(r'\n', ' ', regex=True)
    })


    train_df.to_csv(prefix+'train.tsv', sep='\t', index=False, header=False)
    test_df.to_csv(prefix+'test.tsv', sep='\t', index=False, header=False)
    dev_df.to_csv(prefix + 'dev.tsv', sep='\t', index=False, header=False)


    """LOAD PRE-TRAINED MODEL"""

    MODEL_CLASSES = {
        'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),
        'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
        'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
        'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer)
    }

    config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']]

    config = config_class.from_pretrained(args['model_name'], num_labels=2, finetuning_task=args['task_name'])
    tokenizer = tokenizer_class.from_pretrained(args['model_name'])

    model = model_class.from_pretrained(args['model_name'])
    model.to(device);

    task = args['task_name']

    processor = processors[task]()
    label_list = processor.get_labels()
    num_labels = len(label_list)

    """RUN FINE-TUNING - (set `num_train_epochs` in `args` above)"""

    if args['do_train']:
        train_dataset = load_and_cache_examples(task, tokenizer)
        global_step, tr_loss = train(train_dataset, model, tokenizer)
        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)

    if args['do_train']:
        if not os.path.exists(args['output_dir']):
                os.makedirs(args['output_dir'])
        logger.info("Saving model checkpoint to %s", args['output_dir'])

        model_to_save = model.module if hasattr(model, 'module') else model
        model_to_save.save_pretrained(args['output_dir'])
        tokenizer.save_pretrained(args['output_dir'])
        torch.save(args, os.path.join(args['output_dir'], 'training_args.bin'))

    """EVALUATE"""

    results = {}
    if args['do_eval']:
        checkpoints = [args['output_dir']]
        if args['eval_all_checkpoints']:
            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args['output_dir'] + '/**/' + WEIGHTS_NAME, recursive=True)))
            logging.getLogger("pytorch_transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
            logger.info("Evaluate all checkpoints saved in: %s", checkpoints)
            for checkpoint in checkpoints:
                directory = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""

                model = model_class.from_pretrained(checkpoint)
                model.to(device) # reload model from checkpoint
                result, wrong = evaluate(model, tokenizer, prefix=directory, dataset=True) # change to False for test

                result = dict((k + '_{}'.format(checkpoint), v) for k, v in result.items())
                results.update(result)


if __name__ == '__main__':
    main()