notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv(\"dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.text.isna()==False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.rename(columns={\"sub-title\": \"subtitle\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['text'] = df['text'].apply(lambda x: x.replace(\"— GIB © RIPRODUZIONE RISERVATA\", \"\"))\n",
    "df['text'] = df['text'].apply(lambda x: x.replace(\"© riproduzione riservata\", \"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[(df['target']=='maltrattamento') | (df['target']=='aggressione') | (df['target']=='violenza sessuale')].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install spacy_dbpedia_spotlight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy_dbpedia_spotlight\n",
    "\n",
    "nlp = spacy_dbpedia_spotlight.create('it')\n",
    "print(nlp.pipe_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install SPARQLWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from SPARQLWrapper import SPARQLWrapper, JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_types = {\"http://dbpedia.org/ontology/Person\", \"http://dbpedia.org/ontology/Organisation\", \n",
    "            \"http://dbpedia.org/ontology/Building\", \"http://dbpedia.org/ontology/Place\", \n",
    "            \"http://dbpedia.org/ontology/TimePeriod\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_entities={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_entities(text):\n",
    "    \n",
    "    types=set()\n",
    "    entities={}\n",
    "    \n",
    "    doc=nlp(text)\n",
    "    # print(\"---------------------------------------------------------------\")\n",
    "    # print([(ent.text, ent.kb_id_, ent._.dbpedia_raw_result['@similarityScore']) for ent in doc.ents])\n",
    "    for ent in doc.ents:\n",
    "        # print(\"---------------------\")\n",
    "        # print(ent.text)\n",
    "        if(float(ent._.dbpedia_raw_result['@similarityScore'])>=0.7):\n",
    "            # print(ent.kb_id_)\n",
    "            # print(ent._.dbpedia_raw_result['@support'])\n",
    "            # print(ent._.dbpedia_raw_result['@similarityScore'])\n",
    "            key = ent.kb_id_.replace(\"http://it.dbpedia\", \"http://dbpedia\")\n",
    "            if(key in entities):\n",
    "                count=entities[key]\n",
    "                entities[key]=count+1\n",
    "            else:\n",
    "                entities[key]=1\n",
    "                \n",
    "            if(key in all_entities):\n",
    "                count=all_entities[key]\n",
    "                all_entities[key]=count+1\n",
    "            else:\n",
    "                all_entities[key]=1\n",
    "            \n",
    "            sparql = SPARQLWrapper(\"http://dbpedia.org/sparql/\")\n",
    "            sparql.setReturnFormat(JSON)\n",
    "            \n",
    "            query = \"\"\"select distinct ?type where {\n",
    "                       <\"\"\" + key + \"\"\">  a/rdfs:subClassOf{0,5} ?type . }\n",
    "                    \"\"\"\n",
    "            sparql.setQuery(query)\n",
    "            try:\n",
    "                ret = sparql.queryAndConvert()\n",
    "                for r in ret[\"results\"][\"bindings\"]:\n",
    "                    if(\"http://dbpedia.org/ontology\" in r[\"type\"][\"value\"]):\n",
    "                        #print(r[\"type\"][\"value\"])\n",
    "                        types.add(r[\"type\"][\"value\"])\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "            \n",
    "            if len(types)==0:\n",
    "                query = \"\"\"select distinct ?type where {\n",
    "                           <\"\"\" + key + \"\"\"> owl:equivalentClass{0,1}/a ?type . }\n",
    "                        \"\"\"\n",
    "                sparql.setQuery(query)\n",
    "                try:\n",
    "                    ret = sparql.queryAndConvert()\n",
    "                    for r in ret[\"results\"][\"bindings\"]:\n",
    "                        if(\"http://dbpedia.org/ontology\" in r[\"type\"][\"value\"]):\n",
    "                            #print(r[\"type\"][\"value\"])\n",
    "                            types.add(r[\"type\"][\"value\"])\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "            \n",
    "            if len(types)==0:\n",
    "                query = \"\"\"select distinct ?type where {\n",
    "                           <\"\"\" + key + \"\"\"> dbo:wikiPageRedirects/a ?type . }\n",
    "                        \"\"\"\n",
    "                sparql.setQuery(query)\n",
    "                try:\n",
    "                    ret = sparql.queryAndConvert()\n",
    "                    for r in ret[\"results\"][\"bindings\"]:\n",
    "                        if(\"http://dbpedia.org/ontology\" in r[\"type\"][\"value\"]):\n",
    "                            #print(r[\"type\"][\"value\"])\n",
    "                            types.add(r[\"type\"][\"value\"])\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "            \n",
    "            if len(types)==0:\n",
    "                query = \"\"\"select distinct ?type where {\n",
    "                           <\"\"\" + key + \"\"\"> owl:sameAs{0,1}/a ?type . }\n",
    "                        \"\"\"\n",
    "                sparql.setQuery(query)        \n",
    "                try:\n",
    "                    ret = sparql.queryAndConvert()\n",
    "                    for r in ret[\"results\"][\"bindings\"]:\n",
    "                        if(\"http://dbpedia.org/ontology\" in r[\"type\"][\"value\"]):\n",
    "                            #print(r[\"type\"][\"value\"])\n",
    "                            types.add(r[\"type\"][\"value\"])\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "              \n",
    "            flag=False\n",
    "            for t in my_types:\n",
    "                if t in types:\n",
    "                    flag=True\n",
    "            if flag==True:\n",
    "                if(key in entities):\n",
    "                    count=entities[key]\n",
    "                    entities[key]=count+1\n",
    "                else:\n",
    "                    entities[key]=1\n",
    "\n",
    "                if(key in all_entities):\n",
    "                    count=all_entities[key]\n",
    "                    all_entities[key]=count+1\n",
    "                else:\n",
    "                    all_entities[key]=1\n",
    "    # print(entities)\n",
    "    return entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entities']=df['text'].apply(lambda x: extract_entities(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entities']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_score(dict_):\n",
    "    new_dict={}\n",
    "    for k in dict_:\n",
    "        new_dict[k]=(dict_[k]/all_entities[k])\n",
    "    return new_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entities_with_score']=df['entities'].apply(lambda x: update_score(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['entities_with_score']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install git+https://github.com/LIAAD/yake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "language = \"it\"\n",
    "max_ngram_size = 1\n",
    "deduplication_thresold = 0.7\n",
    "deduplication_algo = 'jaro'\n",
    "windowSize = 4\n",
    "numOfKeywords = 30\n",
    "\n",
    "file = open(\"italian\")    # INSERT PATH TO THE FILE WITH STOPWORDS\n",
    "stopword_set = set(file.read().lower().split(\"\\n\"))\n",
    "        \n",
    "custom_kw_extractor = yake.KeywordExtractor(lan=language, stopwords=stopword_set, n=max_ngram_size, dedupLim=deduplication_thresold, dedupFunc=deduplication_algo, windowsSize=windowSize, top=numOfKeywords, features=None)\n",
    "\n",
    "def extract_keyphrases(x):\n",
    "    keyphrases={}\n",
    "    keys=[]\n",
    "    scores=[]\n",
    "    keywords = custom_kw_extractor.extract_keywords(x)\n",
    "    print(keywords)\n",
    "    for kw in keywords:\n",
    "        keys.append(kw[0].lower())\n",
    "        scores.append(kw[1])\n",
    "    print(scores)\n",
    "    min_ = min(scores)\n",
    "    max_ = max(scores)\n",
    "    if(min_!=max_):\n",
    "        scores = [1-((num-min_)/(max_-min_)) for num in scores]\n",
    "        for i in range(5):\n",
    "            keyphrases[keys[i]]=scores[i]\n",
    "    else:\n",
    "        scores = [1]\n",
    "        keyphrases[keys[0]]=scores[0]\n",
    "    \n",
    "    print(scores)\n",
    "    print(keyphrases)\n",
    "    return keyphrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['keyphrases'] = df['text'].apply(lambda x: extract_keyphrases(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['keyphrases']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys=set()\n",
    "\n",
    "def read_keyphrases(x):\n",
    "    k = list(x.keys())\n",
    "    for el in k:\n",
    "        keys.add(el)\n",
    "    # keys.add(x.keys())\n",
    "    return keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_keyphrases = df['keyphrases'].apply(lambda x: read_keyphrases(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['my_entities_with_score'] = df['entities_with_score'].apply(lambda x: str(x)[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['my_keyphrases'] = df['keyphrases'].apply(lambda x: str(x)[1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"enhanced_dataset.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from neo4j import GraphDatabase\n",
    "\n",
    "class NewDb:\n",
    "    def __init__(self, uri, user, password):\n",
    "        self.driver = GraphDatabase.driver(uri, auth=(user, password))\n",
    "        \n",
    "    def close(self):\n",
    "        self.driver.close()\n",
    "        \n",
    "    def _prova_creazione(self):\n",
    "        with self.driver.session() as session:\n",
    "            session.run(\"CALL n10s.graphconfig.init({handleVocabUris: 'MAP', handleMultival: 'ARRAY', keepLangTag: false, keepCustomDataTypes: true, applyNeo4jNaming: true })\")\n",
    "            session.write_transaction(self._creazione)\n",
    "            \n",
    "    @staticmethod\n",
    "    def _creazione(tx):\n",
    "        tx.run(\"\"\"LOAD CSV WITH HEADERS FROM \\\"file:///enhanced_dataset.csv\\\" AS line FIELDTERMINATOR \\\",\\\" \n",
    "                MERGE(c:Document {id: line.id})\n",
    "                SET c += {url: line.url, title: line.title, text: line.text, publication_date: line.publication_date}\n",
    "                WITH line, c\n",
    "                UNWIND split(line.my_entities_with_score, ',') AS dblink \n",
    "                WITH DISTINCT dblink, c, split(dblink, ': ') as elements\n",
    "                MERGE (e:Entity {uri: replace(elements[0], \"'\", \"\")})\n",
    "                MERGE (c)-[r:appear]->(e)\n",
    "                SET r.weight=elements[1]\"\"\")\n",
    "        \n",
    "        tx.run(\"\"\"LOAD CSV WITH HEADERS FROM \\\"file:///enhanced_dataset.csv\\\" AS line FIELDTERMINATOR \\\",\\\" \n",
    "                MERGE(c:Document {id: line.id})\n",
    "                WITH line, c\n",
    "                UNWIND split(line.my_keyphrases, ',') AS keyphrase \n",
    "                WITH DISTINCT keyphrase, c, split(keyphrase, ': ') as elements\n",
    "                MERGE (k:Keyphrase {name: trim(replace(elements[0], \"'\", \"\"))})\n",
    "                MERGE (c)-[r:appear]->(k)\n",
    "                SET r.weight=elements[1]\"\"\")\n",
    "                \n",
    "       \n",
    "if __name__ == '__main__':\n",
    "    db = NewDb(\"bolt://localhost:7687\", \"neo4j\", \"NEO4J_PASSWORD\")\n",
    "    db._prova_creazione()\n",
    "    db.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}