Text_Summariser(F).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import os\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import urllib.request\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from string import punctuation\n",
    "stop_words = set(stopwords.words('english') + list(punctuation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get file from text doc\n",
    "f=open('england_vs_colombia.txt','r')\n",
    "text=f.readlines()\n",
    "text=''.join(text) #converting the list to type str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = sent_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_string_special_characters(s):\n",
    "    \"\"\"\n",
    "    This function removes special characters from within a string\n",
    "    \n",
    "    param: \n",
    "        s(str): single input string.\n",
    "      \n",
    "    return: \n",
    "        stripped(str): A string with special characters removed\n",
    "    \"\"\"\n",
    "\n",
    "    # Replace special character with ' '\n",
    "    stripped = re.sub('[^\\w\\s]', '', s)\n",
    "    stripped = re.sub('_', '', stripped)\n",
    "\n",
    "    # Change any whitespace to one space\n",
    "    stripped = re.sub('\\s+', ' ', stripped)\n",
    "\n",
    "    # Remove start and end white spaces\n",
    "    stripped = stripped.strip()\n",
    "    \n",
    "    return stripped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sents = [remove_string_special_characters(s) for s in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sentences_combi = \" \".join(sents)\n",
    "words = word_tokenize(sentences_combi)\n",
    "\n",
    "#removing stopwords\n",
    "words = [word.lower() for word in words if word not in stop_words]\n",
    "\n",
    "#creating a frequency table listing each word's frequency \n",
    "freqTable = dict()\n",
    "for word in words:\n",
    "    if word in freqTable:\n",
    "        freqTable[word] += 1\n",
    "    else:\n",
    "        freqTable[word] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#assigning a score to every sentence\n",
    "\n",
    "sentence_info = []\n",
    "sum_values=0\n",
    "\n",
    "#adding the frequency of every non-stop word to give the value for\n",
    "#that sentence\n",
    "for i in range(1,len(sents)):\n",
    "    \n",
    "    words = word_tokenize(sents[i])\n",
    "    words = [word.lower() for word in words]\n",
    "    count = 0\n",
    "    for word in words:\n",
    "        if word in freqTable:\n",
    "            count += freqTable[word]\n",
    "    sum_values += count\n",
    "            \n",
    "    temp = {\"sentence_id\": i, \"sentence_text\":sentences[i], \"sentence_count\": count}\n",
    "    sentence_info.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg=sum_values/(len(sentence_info))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_summary(sentence_info):\n",
    "    for sent in sentence_info:\n",
    "        if sent['sentence_count'] >= avg:\n",
    "            print(sent['sentence_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summary:\n",
      "\n",
      "We don't have to be bowed by the pressure of the past.\n",
      "Before the game, England had won only two World Cup knockout matches since 1990 and looked to be making it three when captain Harry Kane gave them the lead from the penalty spot.\n",
      "But in the third minute of stoppage time at the Spartak Stadium, Colombia defender Yerry Mina headed home to take the game into extra time and then to penalties.\n",
      "Jordan Henderson's effort was saved by David Ospina, but Colombia's Mateus Uribe hit the crossbar before Pickford dived to his right to deny Bacca with his left hand.\n",
      "We played so well in the 90 minutes.\n",
      "\"We looked at technique, how we needed to be as a team, the goalkeeper's role.\n",
      "In a tempestuous game, six Colombia players were shown yellow cards, including Wilmar Barrios for what looked like a headbutt in the Colombia penalty area on midfielder Henderson.\n",
      "Colombia players also surrounded referee Mark Geiger for about three minutes after he awarded a penalty for Carlos Sanchez's foul on Kane.\n",
      "Colombia manager Jose Pekerman said: \"People in England or others should not think of Colombia players like this.\n",
      "We have to find the right balance so both teams can play with the same intentions.\n",
      "\"We should not only look to Colombia but also England's players.\n",
      "Southgate said: \"We didn't rise to it in general, which was brilliant.\n",
      "\"There were many, many fouls in the game and I do not think we committed anywhere near the number they did.\n",
      "Kane leads the race for the Golden Boot as the topscorer in the tournament, having netted six goals in three games - two more than Belgium striker Romelu Lukaku.\n",
      "\"We played well, we controlled the game and it hits you hard when something like that happens,\" Kane said of Mina's equaliser.\n",
      "We had to go again in extra time and then the penalty shootout.\n",
      "\"It's been terrible for us over the years - of course we know that - and to step up when it mattered and do what we did, I'm so proud of everyone involved.\"\n",
      "Kane and Marcus Rashford scored from the spot for England before Henderson missed, but Kieran Trippier levelled and Dier sealed victory by beating Ospina low to his right.\n",
      "\"We're a young squad but we learnt a lot out there today and we would have grown up a lot too,\" said Kane.\n",
      "\"We're in a great position and that could be a big moment going forward in the tournament.\n",
      "Pickford made a stunning save - diving high to his left - to deny Uribe in stoppage time, but the resulting corner led to Colombia's goal.\n",
      "The Everton keeper was questioned for conceding Adnan Januzaj's strike in the defeat by Belgium in the final group game.\n",
      "Pickford, 24, said: \"I was in a massive whirlwind but I saved it with my left hand.\n",
      "\"I might be young but I've got good mental strength and experience and I used that today.\n",
      "Ideally we don't want to be going to a penalty shootout but we are delighted for the fans and the whole country.\"\n",
      "England defender Kieran Trippier: \"We've practised [penalties] after loads of training sessions when we've all been fatiguing, but I had full faith in all the boys.\n",
      "\"I believe in my own ability and I believed that I could score.\n",
      "\"We had to dig deep in this game.\n",
      "Colombia did well - they put us on the back foot for 20 minutes - and we handled the game perfectly.\n",
      "\"We didn't change the way we played.\n",
      "We carried on playing out from the back like we do in training and like the gaffer wants us to do.\n",
      "\"They were trying to wind us up but we kept our cool heads.\n",
      "We knew they were going to taunt us before the game.\"\n",
      "The gaffer has been reiterating it's not all about luck; it's a skill under pressure and the boys did it well.\n",
      "\"They (Colombia) did everything they could to try and frustrate us - falling on the floor, rolling about and continuous fouling.\n",
      "\"We controlled the game for the 90 minutes then the big sucker punch was at the end in the corner.\n",
      "We could have quite easily gone under at that point but the boys showed great courage and character to come back.\n",
      "We've got another big game coming up and one we're going to look forward to.\"\n"
     ]
    }
   ],
   "source": [
    "print('Summary:\\n')\n",
    "get_summary(sentence_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get text doc from web\n",
    "def get_text_from_webpage(url):\n",
    "    \n",
    "    \"\"\"\n",
    "    return the text of the article at the \n",
    "    specified url\n",
    "    \"\"\"\n",
    "    \n",
    "    page=urlopen(url)\n",
    "    soup=BeautifulSoup(page,\"html.parser\")\n",
    "    text=' '.join(map(lambda p: p.text, soup.find_all('p')))\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get file from text doc from net(.txt)\n",
    "def get_text_from_net(url):\n",
    "    text=requests.get(url).text\n",
    "    return text"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}