t5/t5_tf_german_text_summary_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# German Textsummary CNN Daily Mail Results\n",
    "We will try out the trained t5 network from the tpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "from transformers import T5Tokenizer, TFT5ForConditionalGeneration\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 10\n",
    "\n",
    "SHUFFEL_SIZE = 1024\n",
    "\n",
    "learning_rate = 3e-5\n",
    "\n",
    "model_size = \"t5-base\"\n",
    "\n",
    "MAX_ARTICLE_LEN = 512\n",
    "\n",
    "MAX_HIGHLIGHT_LEN = 150"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained(model_size)\n",
    "model = TFT5ForConditionalGeneration.from_pretrained(model_size)\n",
    "\n",
    "task_specific_params = model.config.task_specific_params\n",
    "if task_specific_params is not None:\n",
    "    model.config.update(task_specific_params.get(\"summarization\", {}))\n",
    "    \n",
    "pad_token_id = tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_de_prefix = tf.reshape(tokenizer.encode(\"summarize: en_to_ger \", return_tensors=\"tf\"), (-1,))\n",
    "de_en_prefix = tf.reshape(tokenizer.encode(\"summarize: ger_to_en \", return_tensors=\"tf\"), (-1,))\n",
    "en_en_prefix = tf.reshape(tokenizer.encode(\"summarize: en_to_en \", return_tensors=\"tf\"), (-1,))\n",
    "de_de_prefix = tf.reshape(tokenizer.encode(\"summarize: ger_to_ger \", return_tensors=\"tf\"), (-1,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"tf_t5for_conditional_generation\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "shared (TFSharedEmbeddings)  multiple                  24674304  \n",
      "_________________________________________________________________\n",
      "encoder (TFT5MainLayer)      multiple                  84954240  \n",
      "_________________________________________________________________\n",
      "decoder (TFT5MainLayer)      multiple                  113275392 \n",
      "=================================================================\n",
      "Total params: 222,903,936\n",
      "Trainable params: 222,903,936\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "val_loss = tf.keras.metrics.Mean(name='val_loss')\n",
    "val_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='val_accuracy')\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7fc5900a7fa0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_file = \"../models/ckpt/checkpoint_en.ckpt\"\n",
    "model.load_weights(ckpt_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset\n",
    "We will load the translated CNN Daily Mail dataset from the tfrecords files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "MAX_ARTICLE_LEN = 512\n",
    "MAX_HIGHLIGHT_LEN = 150\n",
    "GLOBAL_BATCH_SIZE = 8\n",
    "\n",
    "def get_tfrecord_dataset(file_name):\n",
    "    features = {\n",
    "        'ger_x': tf.io.FixedLenFeature([MAX_ARTICLE_LEN-8], tf.int64),\n",
    "        'ger_x_mask': tf.io.FixedLenFeature([MAX_ARTICLE_LEN-8], tf.int64),\n",
    "        'ger_y': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN], tf.int64),\n",
    "        'ger_y_ids': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN - 1], tf.int64),\n",
    "        'ger_y_labels': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN - 1], tf.int64),\n",
    "\n",
    "        'en_x': tf.io.FixedLenFeature([MAX_ARTICLE_LEN-8], tf.int64),\n",
    "        'en_x_mask': tf.io.FixedLenFeature([MAX_ARTICLE_LEN-8], tf.int64),\n",
    "        'en_y': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN], tf.int64),\n",
    "        'en_y_ids': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN - 1], tf.int64),\n",
    "        'en_y_labels': tf.io.FixedLenFeature([MAX_HIGHLIGHT_LEN - 1], tf.int64),\n",
    "    }\n",
    "\n",
    "    dataset = tf.data.TFRecordDataset(f\"../data/{file_name}.tfrecord\")\n",
    "\n",
    "    # Taken from the TensorFlow models repository: https://github.com/tensorflow/models/blob/befbe0f9fe02d6bc1efb1c462689d069dae23af1/official/nlp/bert/input_pipeline.py#L24\n",
    "    def decode_record(record, features):\n",
    "        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n",
    "        example = tf.io.parse_single_example(record, features)\n",
    "\n",
    "        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n",
    "        # So cast all int64 to int32.\n",
    "        for name in list(example.keys()):\n",
    "            t = example[name]\n",
    "            if t.dtype == tf.int64:\n",
    "                t = tf.cast(t, tf.int32)\n",
    "            example[name] = t\n",
    "        return example\n",
    "\n",
    "\n",
    "    def select_data_from_record(record):\n",
    "        i  = tf.random.uniform((1,),0,4,dtype=tf.int32)[0]\n",
    "        return tf.concat([en_en_prefix, record['en_x']], axis=0), tf.concat([tf.ones(8, dtype=tf.int32), record['en_x_mask']], axis=0), record['en_y'], record['en_y_ids'], record['en_y_labels']\n",
    " \n",
    "    dataset = dataset.map(lambda record: decode_record(record, features))\n",
    "    dataset = dataset.map(select_data_from_record)\n",
    "    dataset = dataset.shuffle(100)\n",
    "    return dataset.batch(GLOBAL_BATCH_SIZE)\n",
    "\n",
    "test_ds = get_tfrecord_dataset(\"corss_lingual_test_cnn_daily_mail\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------\n",
      "summarize: en_to_en A robot has been arrested' by Swiss police after it bought a supply of illegal drugs on a hidden region of the internet known as the 'dark net'. The automated computer program was designed as an online shopping system that would spend up to $100 each week by randomly purchasing an item offered for sale on the deep web. The robot would then have its purchase mailed to a group of artists who then put the items in an exhibition in the town of St Gallen, in north east Switzerland. The Random Darknet Shopper purchased a number of items for sale on the deep web (pictured) including trainers, jeans, cigarettes, a set of masterkeys and a DVD case (far left) containing 10 ecstasy tablets . The items it bought included a pair of Nike trainers, counterfeit designer jeans, the scan of a Hungarian passport and a set of UK Fire Brigade masterkeys. However, it seems the Random Darknet Shopper crossed the line when it purchased 10 Ecstasy tablets, which arrived in a vacuum packed bag hidden in a DVD case. The dark net is a subsection of the deep web - the part of the internet that does not show up in searches or on social media. Most of the information on the web is far down on dynamically generated sites, unable to be found or seen by traditional search engines, which are rather like dragging a net across the surface of the sea, missing much of the information in the depths. The dark net is used as a way of sharing information and trading goods, but the anonymous and encrypted nature of it has attracted large amounts of illegal activity. The Silk Road website and its successor that were recently shut down was used to sell drugs in exchange for bitcoins, the electronic currency. Other dark net sites allow users to share pornographic photographs, hacked information, credit card numbers and other illegal goods. The Silk Road used an underground computer network known as the The Onion Router (TOR), which is a matrix of encrypted websites and servers that disguise the identity of users. It uses numerous layers of security and encryption, hiding the IP address and the activity of the user. Just 0.26 per cent of the daily internet traffic from the UK accesses this hidden part of the internet. Police seized the drugs, the computer and the rest of the items it had purchased\n",
      "----------\n",
      "summarize: en_to_en Nicola Sturgeon will today launch an extraordinary bid to ‘lead the UK’ as she unveils a string of policies apparently tailor-made for a power-sharing deal with Labour. The SNP leader will unveil a manifesto including proposals for British foreign policy, welfare payments, energy bills and English university tuition fees. Several of her election pledges overlap with Labour policy, including slashing tuition fees in England from a maximum of £9,000 a year to £6,000, and rolling back competition in the English NHS. Others are calculated to drag a minority Labour government to the left, including halting the roll-out of Iain Duncan Smith’s popular changes to benefits and cancelling the Trident nuclear weapons system. Scroll down for video . Control: The Tories warn SNP leader Nicola Sturgeon will be Ed Milibands puppet master if he becomes Prime Minister after the election . Several election pledges in Miss Sturgeons manifesto overlap with Labour policy, including slashing tuition fees in England from a maximum of £9,000 a year to £6,000, and rolling back competition in the English NHS . The SNP also wants the UK to formally recognise Palestine as a state, give energy regulator Ofgem the power to force suppliers to cut prices and to increase public spending on infrastructure in northern England and Wales. At the heart of the SNP’s manifesto are plans for what Miss Sturgeon has called ‘modest’ spending increases funded by yet more borrowing – but which in fact total £180billion over five years. Scotland’s First Minister, who is not herself running for a Westminster seat, said yesterday the SNP had much in common with Left-wing Labour MPs and aimed to build a ‘progressive alliance’ for people ‘right across the UK’. ‘There is an opportunity to build progressive alliances for progressive policies. It is a real rather than a pretend alternative to austerity,’ she told the BBC’s Andrew Marr Show. Today’s SNP election manifesto will be the first to be the focus of intense national scrutiny. Polls suggest the party is on course for a landslide victory north of the border, winning as many as 50 seats and holding the balance of power at Westminster. That could trigger constitutional turmoil of the sort that followed the\n",
      "----------\n",
      "summarize: en_to_en A paramedic who pretended he was gay to get close to women before sexually assaulting them has been struck off the medical register. Christopher Bridger, 25, from Stevenage, Hertfordshire, attacked three women after separate drinking sessions and was jailed for 12 years after being convicted of rape and four other abuse charges last year. The HCPC Conduct and Competence Committee today removed him from the register after hearing his crimes and describing them as a serious breach of trust'. Christopher Bridger, 25, who was jailed for 12 years after he sexually assaulted three women, has been struck off the medical register . A jury at Guildford Crown Court, Surrey, found him guilty of raping a fellow student while he was studying to be a paramedic at St George's University Hospital in London in 2008. He had accompanied her back to her halls following a Freshers' Week fancy dress party and began kissing and cuddling her, despite being told to stop. He then raped her but astonishingly broke down in tears afterwards and said: 'I just want to like girls.' The woman told the jury she ended up comforting Bridger, despite knowing he was in the wrong. His other victims were co-workers at South East Coast Ambulance Service NHS Trust, where he started working in 2010. A lesbian colleague told the court she was molested by Bridger after a staff Christmas party while her girlfriend was in the same hotel bed. The HCPC Conduct and Competence Committee found his crimes were a 'serious breach of trust' The women, aged in their 20s - who cannot be named for legal reasons - were forced to relive their ordeals after the ambulance worker accused them of lying during a trial in July last year. His colleague explained how Bridger came up to her hotel room after she got extremely intoxicated at the party in December 2011. He climbed into bed between his victim and her partner and the woman awoke to find him sexually assaulting her and pleasuring himself as her girlfriend lay asleep next to them. She kept quiet, fearing her partner wouldn't understand what had happened, but the day after on his birthday, he sheepishly sent the woman a number of text messages apologising\n",
      "----------\n",
      "summarize: en_to_en A teacher has been banned from the classroom after he let pupils change their exam answers invalidating his whole schools SATs results. Ian Guffick asked pupils to make changes to the the national curriculum tests for 11-year-olds, which assess English, maths and science at Mitton Manor Primary School in Tewskesbury, Gloucestershire. This was despite the exam session finishing, a conduct panel heard. Ian Guffick asked pupils at Mitton Manor Primary School in Tewkesbury, pictured, to make the changes to their SATs exam answers . The 31-year-old also altered a number of the pupil's exam answers himself before the papers were sent off. After the suspected breach of exam rules, school officials were tipped off, resulting in an investigation being launched in June. The probe by the local education authority found that some pupils taking a test had changed answers outside exam conditions and that Guffick had done the same. It then led to the Department for Education annulling all SATs exam results for the entire school last year, the hearing was told. Guffick was later forced to give up his job at the school and had since admitted the allegations and accepted it amounted to unacceptable professional conduct. The Department for Education annulled the whole school's SAT's results after a probe by the local education authority (file picture) However, he insisted in a statement that he did not ask the pupils to change their answers, but to make them more legible. But the panel, run by the National College for Teaching and Leadership, said that no changes of any kind should be made to exam scripts outside test conditions. Panel chair Martin Pilkington said, although the teacher’s conduct had been dishonest, it fell short of fraud or serious dishonesty. But he added: 'Whilst the panel accepts that Mr Guffick may have felt under pressure to achieve good results for himself and the school, that can in no way excuse his behaviour. 'It is clear to the panel that he deeply regrets his conduct and the panel also considered that he was forthcoming in explaining what he had done when it was investigated by the school and local authority.' Paul Heathcote, acting on behalf of Education Secretary Nicky Morgan, banned Guffick from the classroom\n",
      "----------\n",
      "summarize: en_to_en Tom Lineham scored two interception tries in an eye-catching hat-trick as Hull FC beat Widnes 22-8 to make it back-to-back wins for the first time since last June. Lineham scored a treble in a losing cause at Castleford last month but his efforts on Friday night settled a drab contest at the KC Stadium. The 22-year-old winger showed good opportunism and no little speed on two occasions to pluck passes out of the air and race over 90 metres to add to his early opener. The hat-trick puts him out in front as Super Leagues most prolific finisher in 2015. Tom Lineham scored two interception tries in an eye-catching hat-trick as Hull FC beat Widnes 22-8 . Jamie Shaul went over late on to make sure of the two competition points for Hull, with tries from Kevin Brown and Patrick Ah Van unable to prevent Widnes from falling to a fifth away defeat out of five. It was a third match in nine days for both sides, although Widnes had reason to feel more hard done by after their Easter Monday trip to Catalans. Ah Van had an early opportunity to give the Vikings an extra spring in their step but a foot in touch saw his effort ruled out, although Rhys Hanbury's pass was forward in any case. There was no doubt when Lineham slid over in the corner at the other end after Shaul had spun out of a tackle and found his winger with a nice cut-out pass. The Vikings were defending doggedly near their own line to prevent a second Hull try when Stefan Marsh and Chris Dean were both forced off following a sickening clash of heads. Lineham had a good chance to lay the platform for another Hull raid following a break down the right but his pass was intercepted by former Black and Whites hooker Aaron Heremaia. Widnes worked their way downfield and were back level when Brown, whose high kick in the build-up appeared to have gone forward off Ah Van, forced the ball down from dummy half. The Vikings lost a third player as Lloyd White hobbled off, but they thought they had edged in front when\n",
      "----------\n",
      "summarize: en_to_en Danny Willett gave a rules official, who had been in his line-of-sight, a verbal blast which was clearly audible to spectators surrounding the 17th green at the Masters on Thursday. Englishman Willett vented his anger after his second shot from beyond the green trickled all the way across the putting surface and left the preachers son facing a bogey or worse. 'Of anyone you should know the rules,' Willett shouted loudly at the official, who was sitting about 60 yards away in a golf cart, outside the gallery ropes. Danny Willett waits to play a shot on the fifth during the first round at 2015 Masters on Thursday . The 27-year-old Englishman carded a one-under 71 during his first ever round at Augusta . Willett was still visibly angry as he left the green after salvaging a bogey, though he had calmed down by time he spoke to reporters about 30 minutes later. He said his ire had been raised because the official had been in his line-of-sight as he was preparing to play his shot. 'We were being timed (for slow play), which I can appreciate,' Willett said after carding a one-under 71 at Augusta National. 'It's a little bit tricky out here, so it takes a little bit of time, but you'd like to think the referee that's timing you knows exactly where to put his buggy and where not to put his buggy. Willett, 27, a two-time European Tour winner, is playing in his first Masters. Willett was left frustrated with a rules official for getting in his line-of-sight during his round .\n",
      "----------\n",
      "summarize: en_to_en Obese workers are lazy' and 'unable to fulfil their roles' and as a result are less likely to be hired, a new survey has revealed. Almost half of 1,000 British companies questioned said they are less inclined to recruit an applicant after interview if they are obese or overweight. Among the reasons given were that overweight workers 'are unable to play a full role in the business', 'they're lazy', and 'they wouldn't be able to do the job required', the research shows. Beverley Sunderland, managing director, at Crosslands Employment Solicitors, who commissioned the research, said: 'Our research shows that prejudiced attitudes towards hiring obese workers are rife among British employers.' A survey of 1,000 British companies revealed almost half are less inclined to recruit an obese applicant, fearing 'they're lazy', and 'unable to fulfil their roles as required', because of their weight . The findings come in the wake of a European court case brought by unions and lawyers on behalf of 25st childminder Karsten Kaltoft, 50, who claimed he was sacked by his local authority in Denmark in 2010 for being too fat. In December European judges ruled that obesity qualified as a disability if it 'hinders the full and effective participation of the person... on an equal basis with other workers', when doing their job. Body Mass Index, calculated from a person's weight and height, is accepted as a reliable indicator of obesity. According to the World Health Organisation, the ideal BMI for most adults is between 18.5 and 24.9. If a person's BMI is 25 to 29.9, they are classed as overweight. At between 30 and 39.9 they are obese and 40 or over is very – or morbidly – obese. Doctors say being overweight or obese can increase the risk of health problems, including heart disease, strokes, type 2 diabetes and some cancers. Earlier this year the European Court of Justice's advocate general, Niilo Jaaskinen, who advises the court, suggested that very severe obesity – classified as a BMI of more than 40 – could be considered a disability. But yesterday's ruling does not state what\n",
      "----------\n",
      "summarize: en_to_en Notorious mass murderer Peter Sutcliffe, who has spent three decades in jail, is to be moved to a cushy low security unit, it has been claimed . The families of victims of Yorkshire Ripper Peter Sutcliffe are furious at plans to move him from Broadmoor to a low-security Priory unit. Mass murderer Sutcliffe has spent more than three decades in Broadmoor for the savage killings of 13 women. But now, despite being one of the country’s most notorious serial killers, Sutcliffe may be moved to a cushy Priory unit set in Berkshire woodland. Sutcliffe is being lined up for the move after he expressed remorse and said he no longer hears the voices from God' which he had blamed for his killing spree at his 1981 trial. A report is being compiled which will be heard next month at a prison tribunal which will rule if the move goes ahead. But families of his victims are furious about the move. Neil Jackson, whose mother Emily, 46, became the Ripper's second victim in 1976, told the Sun: 'He should not be moved at all. 'As he's in Broadmoor, he should stay there and they should throw away the key. This just rubs salt into the wounds of families and I believe any remorse he has shown is fake.' A source at the prison told the newspaper Sutcliffe was optimistic about the move - despite being previously turned down for a transfer to the unit which boasts facilities such as a fully-fitted gym and music room. Plans of his referral follow allegations earlier this year that Sutcliffe, who used weapons including a hammer, screwdriver and knives to mutilate women across the north of England, was enjoying an easy lifestyle in jail. Scroll down for video . Potential plans to move Britain's most notorious mass murderer from the high security Broadmoor prison hospital to the Priory . Hospital: Broadmoor's corridors look ordinary, but it houses some of Britain's most disturbed individuals . He was said to be eating chocolate, watching television and listening to music, as well as being privileged to receive visitors four days a\n"
     ]
    }
   ],
   "source": [
    "for i in test_ds.take(1):\n",
    "    summaries = i[0]\n",
    "    pred = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summaries]\n",
    "    for j in pred:\n",
    "        print(\"----------\")\n",
    "        print(j)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "### Define Rouge Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rouge_score import rouge_scorer\n",
    "from rouge_score import scoring\n",
    "\n",
    "class RougeScore:\n",
    "    '''\n",
    "    mostly from https://github.com/google-research/text-to-text-transfer-transformer/blob/master/t5/evaluation/metrics.py \n",
    "    '''\n",
    "    \n",
    "    def __init__(self, score_keys=None)-> None:\n",
    "        super().__init__()\n",
    "        if score_keys is None:  \n",
    "            self.score_keys = [\"rouge1\", \"rouge2\", \"rougeLsum\"]\n",
    "        \n",
    "        self.scorer = rouge_scorer.RougeScorer(self.score_keys)\n",
    "        self.aggregator = scoring.BootstrapAggregator()\n",
    "        \n",
    "        \n",
    "    @staticmethod\n",
    "    def prepare_summary(summary):\n",
    "            # Make sure the summary is not bytes-type\n",
    "            # Add newlines between sentences so that rougeLsum is computed correctly.\n",
    "            summary = summary.replace(\" . \", \" .\\n\")\n",
    "            return summary\n",
    "    \n",
    "    def __call__(self, target, prediction):\n",
    "        \"\"\"Computes rouge score.''\n",
    "        Args:\n",
    "        targets: string\n",
    "        predictions: string\n",
    "        \"\"\"\n",
    "\n",
    "        target = self.prepare_summary(target)\n",
    "        prediction = self.prepare_summary(prediction)\n",
    "        \n",
    "        self.aggregator.add_scores(self.scorer.score(target=target, prediction=prediction))\n",
    "\n",
    "        return \n",
    "    \n",
    "    def reset_states(self):\n",
    "        self.rouge_list = []\n",
    "\n",
    "    def result(self):\n",
    "        result = self.aggregator.aggregate()\n",
    "        \n",
    "        for key in self.score_keys:\n",
    "            score_text = \"%s = %.2f, 95%% confidence [%.2f, %.2f]\"%(\n",
    "                key,\n",
    "                result[key].mid.fmeasure*100,\n",
    "                result[key].low.fmeasure*100,\n",
    "                result[key].high.fmeasure*100\n",
    "            )\n",
    "            print(score_text)\n",
    "        \n",
    "        return {key: result[key].mid.fmeasure*100 for key in self.score_keys}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Rouge Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 : time genreate batch: 2.575255823135376\n",
      "10 : time genreate batch: 32.49284262657166\n",
      "20 : time genreate batch: 29.764947986602785\n",
      "30 : time genreate batch: 32.90486211776734\n",
      "40 : time genreate batch: 34.29788541793823\n",
      "50 : time genreate batch: 35.51083147525787\n",
      "60 : time genreate batch: 34.58894753456116\n",
      "70 : time genreate batch: 37.21555645465851\n",
      "80 : time genreate batch: 30.486803269386293\n",
      "90 : time genreate batch: 33.2992760181427\n",
      "100 : time genreate batch: 30.249897408485413\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "rouge_score = RougeScore()\n",
    "start_time = time.time()\n",
    "\n",
    "for i, (input_ids, input_mask, y, y_ids, y_labels) in enumerate(test_ds):   \n",
    "    summaries = model.generate(\n",
    "        input_ids=input_ids, \n",
    "        attention_mask=input_mask, \n",
    "        early_stopping=True, \n",
    "        max_length=150, \n",
    "        length_penalty=2.0,\n",
    "#         min_length=40, \n",
    "#         do_sample= True, \n",
    "        repetition_penalty=2.0,\n",
    "        top_k=50, \n",
    "        top_p=0.95, \n",
    "        num_return_sequences=1\n",
    "    )\n",
    "\n",
    "    articles = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in input_ids]\n",
    "    \n",
    "    pred = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summaries]\n",
    "    real = [tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in y]\n",
    "    \n",
    "    for pred_sent, real_sent, article_sent in zip(pred, real, articles):\n",
    "        rouge_score(pred_sent, real_sent)\n",
    "        predictions.append(str(\"article: \" + article_sent + \"\\n\\npred sentence: \" + pred_sent + \"\\n\\nreal sentence: \" + real_sent))\n",
    "    \n",
    "    if (i % 10) == 0:\n",
    "        elapsed = (time.time() - start_time) / 10\n",
    "        print(i,\": time genreate batch:\", elapsed)\n",
    "        start_time = time.time()\n",
    "    if i > 100:\n",
    "        # otherwise it will take ages\n",
    "        break\n",
    "\n",
    "\n",
    "# rouge_score.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rouge1 = 39.18, 95% confidence [38.46, 39.92]\n",
      "rouge2 = 16.91, 95% confidence [16.14, 17.69]\n",
      "rougeLsum = 35.65, 95% confidence [34.87, 36.40]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'rouge1': 39.17698947813463,\n",
       " 'rouge2': 16.913290479393883,\n",
       " 'rougeLsum': 35.651022094356236}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rouge_score.result()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets have a look at some of these predicted summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "len_predictions = len(predictions)\n",
    "\n",
    "def get_random_prediction():\n",
    "    return predictions[np.random.randint(len_predictions)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en The brave man who stood up against a neo-Nazi at an anti-Islam rally has spoken out about the poignant moment that was captured on camera. During the Reclaim Australia rallies on the Easter weekend, anti-racism protester Jacob King was snapped staring into the eyes of a bald man with a swastika tattooed behind his ear. Mr King, from Melbourne, told Daily Mail Australia he had his arms spread out because he thought the man was going to attack him and his fellow protesters at Federation Square. Anti-racism protester Jacob King, from Melbourne, had a face off with a neo-Nazi at the Easter weekends Reclaim Australia rallies . Mr King was raised in Melbourne and says he lived in a multicultural part of the Victorian capital where he was exposed to racial tensions . The anti-racism protester said he learned Islamophobic views from a young age but thankfully managed to 'unlearn' these views' 'A few scuffles broke out as a result of those trying to push and fight their way through the picket lines,' he said. 'I remember looking at the bald man from the photo and it seemed to me that he and his group were there solely to be violent and express incoherent hatred, which took away from the already limited opportunities to engage in civilised discussion. 'I have however felt this kind of anger before, and this understanding produced in me a moment of unexpected compassion for him. 'I placed myself in between him and the anti-racism protesters. My arms were outstretched to keep people back and when the man got in my face I was expecting him to attack me. But he didn't. 'As he began to relax I told him there would be no more fighting today. I wasn't expecting him to comply so easily, but I respect him for doing so.' Mr King said he was born and had grown up in a multicultural part of Melbourne, where he had been on the giving and receiving ends of 'racial prejudice and bigoted views'. 'I learned Islamophobic views from a young\n",
      "\n",
      "pred sentence: <extra_id_0>ARTICLE: Anti-racism protester Jacob King, from Melbourne, stood up against a neo-Nazi at an anti-Islam rally . He was snapped staring into the eyes of bald man with swastikan tattooed behind his ear During the Reclaim Australia rallies on Easter weekend, Mr King said he learned Islamophobic views from 'a young age but thankfully managed to unlearn these views\n",
      "\n",
      "real sentence: Jacob King has been identified as man who stood up to anti-Islam protester with swastika tattooed behind his ear . In a photo of Mr King holding his arms out and going toe-to-toe with aggressor was snapped by Kenji Wardenclyffe . The anti-racism protester told Daily Mail Australia he had asked the neo-Nazi to back down and not fight his group .\n"
     ]
    }
   ],
   "source": [
    "print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en These two Australian dogs have got talent! A pair of border collies were caught on camera performing a balancing act. Using a human as their base, pooches Ace and Holly stood up high on their hind legs, with their front paws up. Scroll down for video . Here we go! Border collie Ace is seen on top of his owner and trainer Dai Aokis lap . Don't forget about me! Soon, Holly the border collie hops onto Aoki's legs . The brief clip, set to 'Les Toreadors' from 'Carmen,' first shows a man, identified as their owner and trainer Dai Aoki, lying on the ground with 22-month-old Ace on his lap. Soon, Aoki puts his legs in the air - and 28-month-old Holly hops on. Once Holly is up, Aoki stretches his hands up toward the sky. Soon enough, Holly and Ace stand up solely on their hind legs - and hold the pose for several seconds. This isn't the first time the dogs have shown off their skills. Aoki's Instagram page and YouTube pages feature multiple videos of Ace and Holly doing tricks. Aoki also works as a dog trainer, and runs the Waterloo, Australia-based company Positive Dog Care. Neat trick: The trio are seen performing their stunt together . Ta-da! Holly and Ace finish the trick by standing on their hind legs . Family: Ace (far left) and Holly (far right) are seen with their owner and trainer, Dai Aoki. Aoki runs the Waterloo, Australia-based dog training company Positive Dog Care . Puppy Tales pointed out that the TV show 'The Living Room' deemed Ace Australia's Smartest Dog in 2014. Aoki told the website in January 2015: 'I only use positive reinforcement in the training that I do. 'And, equally importantly, I always have fun with my dogs. 'I actually don’t really use the word \"training\" – we are just having a good time together and learning new things.' Aoki also told Puppy Tales: 'I have had a few dogs in my life and I have never learned\n",
      "\n",
      "pred sentence: : Ace Australia's Smartest Dog in 2014 . Owner Dai Aokis dog training company Positive Dog Care is based in Waterloof, Australia - and runs the company he runs a successful business with .\n",
      "\n",
      "real sentence: Border collies Ace and Holly were caught on camera performing a gravity-defying feat together . The two pooches stood up on their hind legs while balancing on their owner and trainer Dai Aoki . They have appeared in a number of videos showing off their tricks .\n"
     ]
    }
   ],
   "source": [
    "print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en Steven Finn believes he has rediscovered the form that made him one of the most exciting fast-bowling prospects in England. Finn was overlooked for the West Indies tour, but has spent time since the World Cup working on his run-up – and watching videos of his best spells as a reminder of why he became the youngest English bowler to take 50 Test wickets. ‘I’ve had my trials and tribulations over the last 12 months,’ he said. ‘But I feel everything at the crease is as I want it to be. Steven Finn believes hes regained his previous best form and is ready to push for an England place . Finn admits he's 'had my trials and tribulations over the last 12 months' but he's got his 'head straight' Finn smiles as he helps launch ECB's Club Open Days on Tuesday at Brondesbury Cricket Club . ‘My running style is something I’ve had my issues with – kneeing the stumps and shortening my run-up, which is what c**ked me up, really. It’s about trying to get it back to being natural. ‘We’ve got footage that I use when I’ve been bowling at my best. When I was running up as a carefree 21-year-old, I just legged it into the crease. 'It was natural, and I bowled quickly, consistently. My bowling’s looking pretty similar to that at the moment.’ Finn’s claims will be music to the selectors’ ears as they begin to fret about England’s fast-bowling stocks. Finn says he's been watching footage of when he was on the top of his form and is returning to that style . Finn’s form return comes at a good time as Englands fast bowling stocks are starting to worry selectors . The 26-year-old Finn has played only one Championship game for Middlesex this season, but insisted he felt ready to add to his 23 Test caps, the last of which he won back in 2013. ‘I don’t feel I’d have done myself a disservice had I been on the West Indies trip,’ he said. ‘In the World\n",
      "\n",
      "pred sentence: <extra_id_0>NET . Steven Finn says he's regained his previous best form and is ready to push for an England place - despite being overlooked for the West Indies tour . Finn admitting he has had trials and tribulation over the last 12 months, but he feels everything at the wicket is as I want it to be\n",
      "\n",
      "real sentence: Steven Finn was left out of the England squad for the West Indies tour . The Middlesex quick bowler has regained form after a tough 12 months . Finn said hes back to bowling like he was as 'a carefree 21-year-old His last of 23 Test caps came for England back in 2013 .\n"
     ]
    }
   ],
   "source": [
    "print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en Transgender model Andreja Pejic has revealed that she was told by people within the fashion industry that she would no longer seem ‘special’ or ‘interesting’ once she completed her transition into a woman. The 23-year-old Serbian-born fashion star, who appears in a solo editorial in the May issue of Vogue, told the publication that she was advised not to undergo her gender reassignment surgery because it would transform her from an interesting enigma into just another pretty girl. Andreja noted that there were people who warned her not to transition, explaining: ‘There was definitely a lot of, \"Oh, you’re going to lose what’s special about you. You’re not going to be interesting anymore. There are loads of pretty girls out there.\" Scroll down for video . Fashion pioneer: Transgender model Andreja Pejic posed for photographer Patrick Demarchelier for a spread in Vogue. The 23-year-old revealed to the magazine that she is the new face of Make Up For Ever . Major debut: The model, who underwent gender-confirmation surgery last year, took to the runway for the first time as a woman during designer Giles Deacon's Autumn/Winter 2015 show in February (pictured) One agent even told her that 'it's better to be androgynous than a tranny', but Andreja refused to fall prey to those opinions. 'It is about showing that this is not just a gimmick,' she said of being transgender. Andreja also revealed during the interview that she has been signed as the newest face for beauty brand Make Up For Ever, and is thought to be the first ever transgender model to land a major cosmetics campaign. And her success is, she hopes, a sign that there is a cultural shift occurring within the industry when it comes to gender identity. 'There are just more categories now,' she told Vogue. 'It’s good. We’re finally figuring out that gender and sexuality are more complicated.' Andreja was born in what is now known as Bosnia-Herzegovina and later to emigrated to Melbourne, Australia. She grew up as a boy named Andre\n",
      "\n",
      "pred sentence: <extra_id_0>Indiferenta Pejic, 23, was born in Bosnia-Herzegovinia and emigrated to Melbourne, Australia . The Serbian-born fashion star underwent gender-confirmation surgery last year . Andreja has been signed as the new face of Make Up For Ever .\n",
      "\n",
      "real sentence: The 23-year-old also revealed that she has been signed as the new face of beauty brand Make Up For Ever . Andrea is thought to be the first ever transgender model to land a major cosmetics campaign . The Serbian-born fashion star underwent gender-confirmation surgery last year . Before her surgery, Andreja made a name for herself within the fashion industry as both a male and a female model, with her androgynous look .\n"
     ]
    }
   ],
   "source": [
    "print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en Caller ID as we know it could be getting a revamp. Facebook has launched a new app called Hello designed to give you more control over who can call you and how their details appear - even if their number isnt stored in your contacts book. It pulls in any publicly-shared data from a user's Facebook profile and lets you easily block unwanted callers. Scroll down for video . Facebook has launched a new app called Hello (pictured) designed to give you more control over who can call you and how their details appear - even if their number isn't stored in your contacts book. It pulls in any publicly-shared data from a user's Facebook profile and lets you easily block unwanted callers . The free app is currently in testing and is only available on Android devices in the US. It is expected to roll out to other regions and devices if testing is successful. 'Billions of calls are made everyday on mobile phones and people often have very little information about who’s calling them,' explained Facebook's product manager Andrea Vaccari in a blog post. Facebook has tweaked its News Feed setting to make posts from friends more prominent. The rules about seeing multiple posts from the same person in a row have been relaxed to show more from friends. Content posted directly by friends who people interact with more or have a direct relationship with will now appear higher up in News Feed so they are less likely to miss it. The latest update will also make posts and Pages that friends have liked and commented on appear lower down. Facebook said: 'The goal of News Feed is to show you the content that matters to you. 'This means we need to give you the right mix of updates from friends and businesses. 'This balance is different for everyone [and] as more people and pages are sharing more content, we need to keep improving News Feed to get this balance right.' 'Today we are starting to test Hello, a new app built by the Messenger team. 'Hello connects with Facebook so you can see who’s calling, block unwanted calls and search for people and places.' When a user receives a call, Hello shows them info about who’s calling, even if they don’t have that number saved. Users will only see information that people have already shared\n",
      "\n",
      "pred sentence: <extra_id_0>Caller ID as we know it could be getting a revamp, says Facebooks Andrea Vaccari . The free app is currently in testing and is only available on Android devices in the US - it is expected to roll out to other regions and devices if testing is successful 'Hello connects with Facebook so you can see who’s calling, block unwanted calls and search for people and places\n",
      "\n",
      "real sentence: When a user receives a call, Hello will show them info about who’s calling . This includes any public information collated from their Facebook profile . Hello also shows how many people have blocked an unknown number . Free app is currently in beta and only available on Android devices .\n"
     ]
    }
   ],
   "source": [
    "print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "article: summarize: en_to_en Four tries and a total of 30 points from goalkicking winger Josh Mantellato broke brave Bradfords hearts as Hull KR avoided a potential Ladbrokes Challenge Cup banana skin with a 50-30 victory at the Odsal Stadium. Rovers twice came came from behind, trailing 12-0 in the early stages to the Kingstone Press Championship heavyweights, before easing clear with five tries in the final 18 minutes. Amid a pulsating and sometimes controversial cup tie, the hosts ultimately had no answer to Italian international Mantellato, whose last three tries came within the space of seven decisive minutes. It was only the East Yorkshire side's second cup tie success since reaching the quarter-finals in 2011 and their first Challenge Cup victory at Bradford since 1926, albeit only the third time the clubs had met in the competition since then. Winger Josh Mantellato scored four tries and kicked 14 points for Hull KR against Bradford Bulls . Having forced several errors from the Super League side in the opening exchanges, it took a fired-up Bulls less than six minutes to go ahead as Adam O'Brien barged his way through and under the posts. Ryan Shaw converted on his way to a tally of five goals. The Bulls struck again when O'Brien and Etu Uaisele put Dale Ferguson over, although Shaw got a fortuitous bounce off the upright in adding the extras this time. Rovers bounced back, capitalising on an unforced error from Shaw to halve the early deficit when winger Mantellato dived over in the corner then converted his own try. Rovers lost forward Greg Burke to a clash of heads midway through the half but they levelled when an Albert Kelly cross-field kick was tapped back by Mantellato for Kris Welham to touchdown with Mantellato again stroking over the extras. A scintillating weaving solo run by Kelly gave the visitors the led for the first time and Mantellato's conversion sent the Robins in at half time with an 18-12 advantage. A minute before the interval the temperature was cranked up a notch when, out of the blue, Rovers Adam Walker launched a blatant punch at\n",
      "\n",
      "pred sentence: <extra_id_0>Zone kicker Josh Mantellato scored four tries and kicked 14 points for Hull KR . Hull beat Bradford Bulls 50-30 to avoid Ladbrokes Challenge Cup banana skin vs Rover'ers - the first Challenge Cup win at Bradford since 1926 . Rovers trailed 12-0 in the early stages before easing clear with five tries in the final 18 minutes despite a pulsating and sometimes controversial cup tie .\n",
      "\n",
      "real sentence: Goalkicking winger Josh Mantellato dominated the scoring in Bradford . Rovers twice came came from behind against the Championship side . Hull KR blew out the scoreline with five tries in the final 18 minutes .\n",
      "article: summarize: en_to_en Louis van Gaal has revealed that he has spoken to Marouane Fellaini about keeping his cool when the Manchester United midfielder returns to Goodison Park on Sunday for the first time since his £27million move from Everton. Fellaini, who followed manager David Moyes to Old Trafford in the summer of 2013, is set for a more hostile reception than Juan Mata received at his old club Chelsea last weekend. The Belgium international became a symbol for the failure of the Moyes era but has turned his career around under Van Gaal and goes back to Merseyside as an established first-team player at United. Marouane Fellaini (second right) shares a joke with goalkeeper David de Gea during training on Friday . United are gearing up for a trip to Everton, and the game will be Fellainis first return to his former club . Louis van Gaal has spoken to Fellaini about keeping his cool on his first return to Goodison Park . Asked about suggestions at Everton that Fellaini has a short fuse, Van Gaal said: ‘I have spoken with Fellaini because I know this. Fellaini is always dry in his answers. He said \"it’s a game\". Yes, but it’s a special game for you. I hope the fans shall receive him like the Chelsea fans received Juan Mata because I like that very much. I hope it’s like that. ‘I am always communicating with my players about philosophy, performance, attitude, behaviour, and maybe one or two aspects more. 'That is what I’m doing, not more than that. When players believe in you they want to convince me because every player playing for Manchester United has qualities. 'Also Marouane Fellaini has qualities and we want to let his qualities dominate in the team and that is what we are trying to do. I believe that we are doing that. That’s why his performances are better just now.’ United have confirmed that Rafael has been ruled out of the remaining five games of the season with a rib injury, and Van Gaal expects to be without Michael Carrick, Phil Jones, Marcos Rojo and Daley Blind for the trip to Goodison after all\n",
      "\n",
      "pred sentence: toffees host Everton at Goodison Park on Sunday in their Premier League clash . Louis van Gaal has spoken with Marouan Fellainis about keeping his cool - his first return since his £27million move from Evertoon . Van Gaals side have confirmed that Rafael has been ruled out of the remaining five games of the season with a rib injury if he returns .\n",
      "\n",
      "real sentence: Manchester United travel to Everton in the Premier League this weekend . It is Marouane Fellaini's first return to Goodison Park since joining United . Louis van Gaal admits he has told Fellaini to keep his cool . United are likely to be without Michael Carrick, Phil Jones, Marcos Rojo and Daley Blind for the trip to Merseyside . Everton vs Man United: Team news, kick-off time and probable line-ups .\n",
      "article: summarize: en_to_en The stylist behind singer Rita Oras unique, over-the-top hairstyles has revealed how much work the chart-topper has to put into ensuring her blonde locks remain totally healthy. Chris Appleton, who has been working with natural brunette Rita, 24, for the past two years, confessed to Daily Mail Online that her demanding haircare routine does require a hefty amount of work and 'maintenance'. 'We have to color the hair every three weeks,' he explained. 'We've managed to get the timing perfect so that she's not coloring it all the time. If you keep bleaching it, you push it too far and it can look awful, which has been done in the past by other celebrities! Scroll down for video . Style icon: Chris Appleton (R) has been working as singer Rita Ora's hairstylist for two years . Haircare regimen: According to Chris, Rita's blonde locks need to be colored every three weeks, a process which can dry out the hair if done too frequently . 'I also make my own hair masks, which I make her sleep in, to ensure that her hair stays in the best condition possible. 'I'm so strict with Rita, I want to make sure that her hair always looks great, and that means a lot of work. When she has any spare time, I make her condition.' But despite the seemingly endless hours of maintenance required in order to keep Rita's barnet looking its best, Chris insists that the singer isn't planning to follow in Kim Kardashian's footsteps and make a change to brunette any time soon. 'Blonde is such a statement, despite the fact that it's a lot of maintenance behind the scenes,' he said. 'We've never done anything deep or dark and we change the actual style so much that I'm eager to keep the color consistent.' The singer did have a chance to experiment with a darker hairstyle when she appeared in the recently-released Fifty Shades of Grey movie however; Rita's role as Mia, the younger sister of charismatic businessman Christian Grey, saw her don an extremely short, dark wig, which Chris believes helped Rita to\n",
      "\n",
      "pred sentence: <extra_id_0> 'I'm so strict with Rita, I want to make sure that her hair always looks great, and that means a lot of work. When she has any spare time, I make her condition,' said Chris Appleton - who has been working with the singer for two years . 'Blonde is such an statement, despite the fact that its ... i'm eager to keep the color consistent,' he said referring back to Rite's blonde locks .\n",
      "\n",
      "real sentence: Hairstylist Chris Appleton explained the inspiration for 24-year-old Ritas unique and over-the-top hairstyles . He confessed that her blonde mane requires a lot of 'maintenance' and has to be bleached every three weeks . Unlike Kim Kardashian however, Chris doesnt believe Rita will be going back to brunette any time soon .\n",
      "article: summarize: en_to_en The baby boom generation is set to leave a wave of funeral debt to its children and grandchildren, a report warns. The number of deaths every year in Britain – which have been falling for the past 40 years – will rise by 20 per cent over the next two decades, as the lives of unprecedented numbers born in the post-baby boom come to an end. At the same time the price of a funeral is rising fast, thanks to higher costs for cremation, rising undertakers’ bills as funeral firms are faced with bad debts, and the increasing fees demanded by churches. Loss of baby boomers: The number of UK deaths per will rise by 20 per cent over the next two decades, as funeral costs have already risen by 80 per cent over the past ten years . The report, prepared by an old age think tank, the International Longevity Centre-UK, and backed by insurance firm OneFamily, suggests that younger people who already feel financially hard pressed in comparison to their parents will still be under money pressure even after their deaths. Younger people are finding it increasingly difficult to afford homes or to start families, and in the long-term their pensions are much less secure than those of their parents, many of whom enjoyed the generous and fast-disappearing defined benefit pension system. The report from ILC-UK to be published next week said that death rates are set to go up following a long period in which medical advances, better diet, healthier lifestyles and the growing unpopularity of smoking, and the decline of dangerous heavy industrial jobs, have meant fewer people have been dying despite a rising population. ‘After decades of the number of deaths in the UK falling, 2015 will mark a tipping point where the historic trend starts to reverse,’ it said. ‘As the oldest of the baby boomer generation move through their later years, deaths are predicted to rise by 20 per cent.’ Expensive end: A ‘simple’ funeral, with a cremation, a minister and an undertaker, now costs an average of £3,590, an old age think tank has found . It added: ‘This will have significant implications for current forecasts of funeral costs and highlights a growing need for families to broach the delicate subject of end of life expenses.’ Baroness Sally Greengros\n",
      "\n",
      "pred sentence: to rise by 20 per cent over the next two decades, says old age think tank . Costs for cremation, undertakers and funeral firms are facing bad debts - and church fees . Younger people will still be under money pressure even after their deaths, report says in future .\n",
      "\n",
      "real sentence: Number of UK deaths per will rise by 20% over the next two decades . Funeral costs have already risen by 80 per cent over the past ten years . 'Simple’ funeral, with cremation, minister and undertaker, now costs £3,590 .\n",
      "article: summarize: en_to_en An English photographer has traveled all over the world - and along the way created stunning panoramic images using a film camera. John Chapple has captured all types of landscapes, including grassy fields in England, major US cities, and sandy beaches in Australia. Chapple, originally from North Devon, first worked as a news and show business photographer before he got into landscape photography, his website says. Scroll down for video . Quite the view! Photographer John Chapple captured this shot of the Santa Monica Pier in Santa Monica, California . Escape: Water pours down a grassy California hillside in this image . West Coast adventure: Chapple captured downtown city skyscrapers - as well as miles of the Greater Los Angeles area and mountains in the distance . Dried up: Desert shrubbery is seen as sun touches down on mountains in Mojave, California . Pathway: Chapple spied this structure in Queensland, Australia . Getaway: Birds swim through the waters of Western Australia in this shot . Stormy weather: Clouds perch over grassy areas in Northam Burrows, North Devon . For his panoramic images he uses a Linhof Technorama 617s III,' his website says. 'This camera is unique in that the negatives are a huge 6 x 17 centimeters, with only four exposures per film. 'He also used a 50 megapixel digital Hasselblad and a top of the range 1DS Canon digital camera.' When asked for advice during a 2012 interview with Landscape Photography Magazine, Chapple said to 'Invest in a good alarm clock, a comfortable pair of boots and a sturdy tripod and you can't go wrong.' Chapple told the magazine at the time 'My dream is to one day be able to make a living with my landscape photography. Right now it's something that I love doing, and squeeze into my hectic life when I can. 'I can't imagine a greater life, being able to shoot landscapes AND make my monthly mortgage payment. There's nothing greater professionally than being paid to do what you love.' Blur: Chapple took this image in Queensland, Australia. He has worked as both a news and showbusiness photographer .\n",
      "\n",
      "pred sentence: <extra_id_0>EN-English photographer John Chapple has captured all types of landscapes, including grassy fields in England, major US cities, and sandy beaches in Australia . Chapples panoramic images use a Linhof Technorama 617s III film camera with only four exposure per film .\n",
      "\n",
      "real sentence: English photographer John Chapple has traveled all over the world - and along the way created stunning panoramic images using a film camera . Chapple, originally from North Devon, first worked as a news and show business photographer before he got into landscape photography . He has captured grassy fields in England, major US cities, and sandy beaches in Australia . Chapple often takes landscapes with a Linhof Technorama 617s III camera .\n",
      "article: summarize: en_to_en (CNN)A photo of a baby boy being pulled from the rubble of the Nepal earthquake has become the defining image of a disaster that has devastated the country. His tiny face is coated in dust from the debris that crashed around him as the earth shook on Saturday, April 25, killing more than 5,000 people and injuring at least twice as many more. His name is Sonit Awal, and hes just four months old. On Saturday, Sonit was at home in Muldhoka, Bhaktapur, east of the capital Kathmandu, when the 7.8-magnitude quake sent tremors through Nepal and neighboring countries, according to Kathmandu Today. The house collapsed, burying the child, leaving his father Shyam Awal frantically searching for him amid the rubble. A desperate Awal called the Nepalese Army, and they too scrambled through the dirt and debris until midnight, before giving up hope and leaving. Sonit's father had lost all hope of finding his son alive, the paper reported, until he heard faint cries from below. On Sunday morning, the soldiers returned and at 10 a.m they lifted Sonit clear of the fallen rocks, beams, bricks and dust that had trapped him for 22 hours. His rescuers held him high in their hands to the sun, though his eyes were still wedged tightly shut. Sonits face was exposed during the whole ordeal, though a hooded top and shawl protected his head and body during the cold night spent under rubble. The baby was taken to Bhaktapur Hospital and found to be uninjured; . His unlikely rescue provides some hope to a country that has experienced so much loss.\n",
      "\n",
      "pred sentence: <extra_id_0>ANA: Sonit Awal was at home when the 7.8-magnitude quake sent tremors through Nepal and neighboring countries . His father Shyam Awal had lost all hope of finding his son alive until he heard faint cries from below . On Sunday, soldiers returned and lifted Sonits face clear of fallen rocks, beams, bricked and dust a day later .\n",
      "\n",
      "real sentence: Baby Sonit Awal found in rubble of Nepal earthquake, Sunday morning . Spent 22 hours buried under his home after 7.8-magnitude quake .\n",
      "article: summarize: en_to_en (CNN)Ciudad Juarez, Mexico, was once known as the murder capital of the world. Back in 2010, at the height of cartel violence, the city averaged 8.5 killings per day. But five years later, local officials say the city is much safer, and plans are underway to lure foreign tourists and investors back to Juarez. This month the city launched the tourism campaign \"Juarez is Waiting for You.\" The rebranding effort started quietly a year ago, and on April 10, it was on full display. Mayor Enrique Serrano officially kicked off the campaign, giving what he called an \"unprecedented\" high-profile tour to regional leaders from the United States and Mexico. U.S. Rep. Robert \"Beto\" ORourke of Texas was one of those on a leg of the Juarez tour. His congressional district includes El Paso, Texas, which sits directly across the Rio Grande. O'Rourke says there's good reason for locals to be hopeful. \"As a region, El Paso and Juarez represent 20% of all U.S-Mexico trade. The binational ties are strong and have remained strong,\" O'Rourke says. \"Yes, we had a really difficult time for a ... period. Juarez was at one time the deadliest city in the world.\" O'Rourke speaks of a time between 2009 and 2012 when men, women and children were killed indiscriminately. Many were helplessly caught in the cartel violence. Others were victims of the drug turf war. It wasn't that long ago, O'Rourke says, that he thought twice about crossing the bridge into Juarez. \"(Now) I travel to Juarez regularly to have lunch or meet people or just to go. I always feel safe and secure.\" A spokesman for the Chihuahua state attorney general's office told CNN that at one point, there were days when Juarez had more than 20 killings. \"That was normal,\" spokesman Julio Castaneda told CNN. \"It's safer now.\" The numbers from the attorney general's office seem to bear that out. More than 3,000 people were killed in the city\n",
      "\n",
      "pred sentence: <extra_id_0>Ciudad Juarez, Mexico, was once known as the murder capital of the world . In 2010, the city averaged 8.5 killings per day . This month the city launched the tourism campaign \"Juarez is Waiting for You\" U.S Rep. Robert \"Beto\" ORourke of Texas was one of those on the tour .\n",
      "\n",
      "real sentence: Cartel violence helped make Juarez the murder capital of the world five years ago . But the murder rate in the city has declined rapidly since 2010 . Now city leaders are working to bring visitors and foreign investment back to Juarez .\n",
      "article: summarize: en_to_en Between the bad behaviour and the bad advice, Raheem Sterling is not exactly endearing himself to potential suitors right now. Hippy crack and shisha pipes, mixed in with accusations of greed after being encouraged by his agent to reject Liverpool’s offer of close to £100,000-a-week, might not be quite enough to send rival clubs running. After all, as Liverpool know only too well, you can bite and racially abuse opponents and still secure a move to Barcelona. But the evidence is sure to be a concern when the world’s richest clubs assess the character as much as the class of an individual before parting with the £40million upwards Liverpool will demand to secure the services of a player like Sterling. Liverpool players Raheem Sterling and Jordon Ibe (left) have been pictured smoking a shisha pipe . This is the second time Sterling has been pictured smoking from a shisha pipe in the past few days . The recent images of Sterling will be a concern to clubs considering parting with £40million to sign the star . Luis Suarez secured a move to Barcelona despite being involved in a number of incidents on the pitch . English football would appear to have yet another flawed talent, another player seemingly lacking the wit to make best use of the ability he has been blessed with. There is a reason why we rarely see the finest players on the planet caught in as compromising a position as Sterling found himself on Monday night. Becoming as good as Lionel Messi or Cristiano Ronaldo — Gareth Bale even — requires more than ability alone. It requires the kind of professionalism and dedication that seems to be absent in English players too often photographed smoking or indulging in legal highs. As the latest photographs show, this is not an isolated incident and he will be lectured on the dangers of nitrous oxide when he meets with Brendan Rodgers tomorrow. It is more of a health issue than an image issue for Liverpool’s manager, as Sterling did at least opt for the privacy of his own home to suck on a balloon filled with nitrous oxide — unlike Tottenham’s Kyle Walker, for instance. But Rodgers will also suggest to the unsettled 20-year-old that\n",
      "\n",
      "pred sentence: toffees boss Brendan Rodger will lecture Sterling on the danger of nitrous oxide . Rahem Sterling has been pictured smoking from a shishi pipe in the past few days - this is not an isolated incident . The 20-year old has been linked with a move to Barcelona from Liverpool despite being involved in a number incidents on the pitch involving Luis Suarez & Jordon Ibe .\n",
      "\n",
      "real sentence: New pictures show Raheem Sterling and Jordon Ibe with shisha pipes . The Liverpool pair are dressed in casual clothing and have a pipe each . Pictures emerged last week of Liverpool star Sterling smoking shisha . Footage also emerged of him inhaling nitrous oxide from a balloon . The pictures create a fresh problem for Liverpool boss Brendan Rodgers . The images will be a concern for any potential suitors of the Liverpool star .\n",
      "article: summarize: en_to_en Muhammad Naviede, 60, (pictured outside the Old Bailey in 1995) was killed instantly in a plane crash . A banking fraudster with links to Cherie Blair was killed instantly in a plane crash minutes after he sent a text saying his aircraft was going down, an investigation has revealed. Muhammad Naviede, who was jailed in 1995 for nine years for a £45million fraud, died after his Piper Tomahawk aircraft plummeted into a field near Padbury in Buckinghamshire. The 60-year-old, whose daughter is a former X-Factor contestant, sent a text to a relative shortly before the crash on August 20 last year saying: I'm in a plane out of control and it's going down.' An investigation by the Air Accident Investigation Branch (AAIB) said the message was 'unusual' as the leased plane did not enter a spin until after it was sent. The experienced pilot had also tried to call the same relative before writing 148 characters within 25 seconds in a text - something the AAIB said would require 'considerable dexterity'. The report said: 'The pilot had accumulated considerable experience in a variety of aircraft in the years that he had been flying. He was familiar with the Piper Tomahawk aircraft and was in recent flying practice. 'It was suggested in his final text message that he had lost control of the aircraft, to such an extent that he did not expect to survive. 'However, after this message was sent, the aircraft continued in flight for more than two minutes, before entering a spin from which it did not recover.' Investigators found no fault with the aircraft and said weather conditions were unlikely to have caused the crash. Mr Naviede, of Regent's Park, London, had previously undertaken aerobatic training, meaning he 'would have recognised, and been able to recover from, a spin', according to the AAIB. The investigation also revealed the father-of-two had not booked a final destination for the fatal flight. Mr Naviede, whose personal wealth was once estimated to be £117million, was the former head of trade finance company Arrows, which collapsed owing more than £100 million to\n",
      "\n",
      "pred sentence: <extra_id_0>EN-Z of Cherie Blair was jailed in 1995 for nine years for £45million fraud . Muhammad Navide, 60, sent text to relative saying I'm in a plane out of control and it's going down' Investigation found no fault with aircraft and weather conditions unlikely to have caused crash . Father-of-two's Piper Tomhawk aircraft plummeted into field near Padbury, Buckinghamshire .\n",
      "\n",
      "real sentence: Muhammad Naviede died after his Piper Tomahawk aircraft crashed in field . The 60-year-old sent text to relative shortly before plane span out of control . Air Accident Investigation Branch investigation says message is unusual His daughter was on X Factor and brother entertained Cherie Blair at home . The father-of-two was jailed for nine years in 1995 for a £45million fraud .\n",
      "article: summarize: en_to_en Matthew Kenney smoked Flakka and then ran naked . A Florida man who was high on a designer drug called Flakka stripped and ran naked through traffic in Fort Lauderdale to escape from imaginary killers who he believed stole his clothes and wanted to murder him. Matthew Kenney, 34, told police he smoked Flakka before he streaked though traffic early on Saturday evening while only wearing a pair of sneakers. Flakka, which can be injected, snorted, smoked, swallowed or taken with other substances, has been nicknamed $5 insanity' for its mind-bending effects and cheap cost. After he was arrested, Kenney told police he would 'rather die than be caught by these unknown people', the Sun Sentinel reported. He added that 'if I got hit by a car they would stop chasing me' according to a Fort Lauderdale Police reported. Kenney has previous arrests for disorderly conduct, making a riot and possession of a controlled substance. He was hospitalized for a psychiatric evaluation. Flakka is usually made from the chemical alpha-PVP, a synthetic version of the stimulant cathinone. That is the same type of chemical that is used to make bath salts. Scroll down for video . Kenney, 34, ran though traffic early on Saturday evening while only wearing sneakers in Fort Lauderdale, Florida . The suspect said he was escaping imaginary killers who he believed stole his clothes and wanted to murder him . The use of flakka a designer drug that can be even stronger than crystal meth or bath salts, is up in Florida . Flakka resembles a mix of crack cocaine and meth and it has a a strong odor 'like a sweaty sock, WPBF 25 News reported. Once ingested, the drug causes a feeling of euphoria, hallucinations and sometimes psychosis or even superhuman strength. The high lasts for a couple hours and users are left with a strong desire to use again. There were more than 670 Flakka occurrences in Florida in 2014, up from 85 in 2012 and zero in 2010, according to the Drug Enforcement Administration.\n",
      "\n",
      "pred sentence: : Matthew Kenney smoked Flakka before he streaked through traffic early on Saturday evening while only wearing sneakers . He told police he was escaping imaginary killers who he believed stole his clothes and wanted to murder him . Flakk, which can be injected, snorted, smoked, swallow or taken with other substances, has been nicknamed $5 in-insanity for its mind-bending effects and cheap cost\n",
      "\n",
      "real sentence: Matthew Kenney, 34, said he smoked Flakka before he went streaking . Was arrested on Saturday after run through Fort Lauderdale, Florida . Drug is made from same version of stimulant used to produce bath salts . It causes euphoria, hallucinations, psychosis and superhuman strength . Kenney has prior arrests and was hospitalized for a psychiatric evaluation .\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(get_random_prediction())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downsides\n",
    "The start token makes usually no sense."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_path = \"../data/t5base_result_english.txt\"\n",
    "open(result_path, \"w\")\n",
    "for pred in predictions:\n",
    "    with open(result_path, \"a\") as file:\n",
    "        file.write(pred + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "textsummary",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}