bnp-isbn-scraping.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "import time\n",
    "import random\n",
    "import re\n",
    "import xml.etree.ElementTree as ET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Lista de URL para recolha de dados\n",
    "\n",
    "#http://urn.bn.pt/isbn/mods/xml?id=9789896417529\n",
    "\n",
    "static = 'http://urn.bn.pt/isbn/mods/xml?id='\n",
    "\n",
    "isbn_list = []\n",
    "# open file and read the content in a list\n",
    "with open('isbn-list.txt', 'r') as filehandle:\n",
    "    isbn_list = [isbn.rstrip() for isbn in filehandle.readlines()]\n",
    "\n",
    "urllist = [static+str(isbn) for isbn in isbn_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=8466.0), HTML(value='')))",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "7c6588bf232e42bb8313230db41b2109"
      }
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-359-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-390-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-416-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-431-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-455-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-743-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-756-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-557-770-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-614-021-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-614-035-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-614-113-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-030-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-055-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-076-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-090-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-129-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-144-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-232-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-275-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-616-299-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-622-614-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-622-623-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-622-624-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-622-673-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-623-012-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-623-053-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-623-152-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-623-156-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-019-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-021-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-043-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-054-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-058-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-059-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-072-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-081-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-093-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-116-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-151-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-165-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-236-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-252-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-625-257-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-104-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-129-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-131-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-131-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-162-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-237-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-240-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-241-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-253-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-323-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-359-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-626-359-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-627-012-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-628-024-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-628-164-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-628-165-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-628-166-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-633-044-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-633-057-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-633-105-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-633-148-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-634-010-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-634-012-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-636-292-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-641-642-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-65-3008-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-653-014-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-653-017-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-657-474-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-657-514-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-004-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-006-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-007-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-012-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-013-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-016-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-016-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-024-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-031-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-037-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-038-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-040-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-042-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-043-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-048-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-660-401-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-668-054-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-668-055-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-199-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-199-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-229-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-257-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-259-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-671-267-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-672-111-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-672-114-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-672-142-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-673-012-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-673-075-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-175-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-177-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-339-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-340-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-377-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-378-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-415-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-434-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-472-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-477-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-679-490-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-702-211-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-703-176-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-718-060-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-722-193-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-722-216-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-724-000-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-734-133-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-749-041-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-749-061-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-014-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-017-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-017-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-020-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-092-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-756-095-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-760-087-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-772-016-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-772-069-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8013-43-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8025-93-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8027-04-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8027-09-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8027-16-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8027-17-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8027-18-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8031-03-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8048-93-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8056-29-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8056-40-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8145-11-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8145-13-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8145-25-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-05-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-07-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-08-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-09-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-10-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-11-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8166-28-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8173-33-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8173-48-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8173-49-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-12-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-39-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-48-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-49-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-55-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8205-56-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8213-53-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8214-32-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8219-11 anos-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8232-47-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8236-08-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8255-04-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8255-38-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8309-22-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8327-05-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8327-22-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8377-79-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8379-33-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8421-18-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8432-26-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8432-45-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8441-00-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8441-02-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8441-05-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8441-06-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8459-02-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8459-03-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8459-04-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8459-06-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8470-24-10 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8513-19-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8566-81-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8592-12-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8745-09-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8752-10-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8775-06-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8816-11-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8827-09-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8839-60-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-8839-61-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95093-2-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95173-3-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95209-5-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95883-1-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95902-1-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-95902-2-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-96031-9-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-96717-0-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-97029-2-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-97200-5-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-97240-0-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-97256-1-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-97256-7-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-98016-5-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-98333-7-1 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-98578-1-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-99296-7-8 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-99621-5-6 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-989-99621-6-3 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=978-9898-671-352-2 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=9787-972-0-72686-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-20-3090-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-550-367-9 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-550-371-7 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-553-235-0 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-625-015-4 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=989-950-030-5 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=INLD 7823/RLINLD/2013 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=INLD 8284/RLINLD/2014 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=LFCAB D240 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=LFCABD240 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=LFCABD246 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=Nº de registo INLD 6700/RLINLD/2010 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=Nº de registo INLD 8115/RLINLD/2014 \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "failed for http://urn.bn.pt/isbn/mods/xml?id=nan \n",
      "\n",
      "\n",
      "Já está\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                 ISBN                                          titulo  \\\n",
       "0   978-989-644-448-8                                   Outras mentes   \n",
       "0   978-989-616-811-7                                Poções e paixões   \n",
       "0   978-989-644-434-1  Nós, os micróbios e uma visão alargada da vida   \n",
       "0   978-989-641-752-9                         Preparação para a noite   \n",
       "0   978-972-23-6156-9                   <Os >contos de Beedle o bardo   \n",
       "..                ...                                             ...   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "\n",
       "                    autor          editora data_publicacao  \\\n",
       "0    Smith, Peter Godfrey  Temas e Debates            2017   \n",
       "0       André, João Paulo          Gradiva            2018   \n",
       "0                Yong, Ed  Temas e Debates            2017   \n",
       "0    Rocha, Jaime, pseud.   Relógio d'Água       cop. 2017   \n",
       "0   Rowling, J. K., 1965-         Presença            2018   \n",
       "..                    ...              ...             ...   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "\n",
       "                                              url_BNP  \n",
       "0   http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789896168117  \n",
       "0   http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789896417529  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789722361569  \n",
       "..                                                ...  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "\n",
       "[8466 rows x 6 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ISBN</th>\n      <th>titulo</th>\n      <th>autor</th>\n      <th>editora</th>\n      <th>data_publicacao</th>\n      <th>url_BNP</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>978-989-644-448-8</td>\n      <td>Outras mentes</td>\n      <td>Smith, Peter Godfrey</td>\n      <td>Temas e Debates</td>\n      <td>2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-616-811-7</td>\n      <td>Poções e paixões</td>\n      <td>André, João Paulo</td>\n      <td>Gradiva</td>\n      <td>2018</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789896168117</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-644-434-1</td>\n      <td>Nós, os micróbios e uma visão alargada da vida</td>\n      <td>Yong, Ed</td>\n      <td>Temas e Debates</td>\n      <td>2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-641-752-9</td>\n      <td>Preparação para a noite</td>\n      <td>Rocha, Jaime, pseud.</td>\n      <td>Relógio d'Água</td>\n      <td>cop. 2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789896417529</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-972-23-6156-9</td>\n      <td>&lt;Os &gt;contos de Beedle o bardo</td>\n      <td>Rowling, J. K., 1965-</td>\n      <td>Presença</td>\n      <td>2018</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789722361569</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>8466 rows × 6 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "#urllist_test = urllist[23:25]\n",
    "\n",
    "maindata = pd.DataFrame()\n",
    "\n",
    "for i in tqdm(urllist):\n",
    "    headers = requests.utils.default_headers()\n",
    "    headers.update({'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/63.0'})\n",
    "    URL = i\n",
    "\n",
    "    page = requests.get(URL)\n",
    "    \n",
    "    soup = BeautifulSoup(page.content, \"lxml\")\n",
    "\n",
    "    book = {}\n",
    "    try:\n",
    "        book[\"ISBN\"] = soup.identifier.string\n",
    "        book[\"titulo\"] = soup.title.string\n",
    "        book[\"autor\"] = soup.namepart.string\n",
    "    #book[\"id_autor\"] = soup.namepart\n",
    "        book[\"editora\"] = soup.publisher.string\n",
    "        book[\"data_publicacao\"] = soup.dateissued.string\n",
    "        book[\"url_BNP\"] = i\n",
    "    except:\n",
    "        print(\"failed for\", i, \"\\n\")\n",
    "        file = open(r\"not-working.txt\",\"a\")\n",
    "        file.write( i + \"\\n\")\n",
    "        file.close()\n",
    "\n",
    "    book_data = pd.DataFrame(book, index=[0])\n",
    "    \n",
    "    maindata = pd.concat([maindata, book_data])\n",
    "    time.sleep(random.randint(1,2))\n",
    "    \n",
    "print(\"Já está\")\n",
    "maindata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                 ISBN                                          titulo  \\\n",
       "0   978-989-644-448-8                                   Outras mentes   \n",
       "0   978-989-616-811-7                                Poções e paixões   \n",
       "0   978-989-644-434-1  Nós, os micróbios e uma visão alargada da vida   \n",
       "0   978-989-641-752-9                         Preparação para a noite   \n",
       "0   978-972-23-6156-9                   <Os >contos de Beedle o bardo   \n",
       "..                ...                                             ...   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "0                 NaN                                             NaN   \n",
       "\n",
       "                    autor          editora data_publicacao  \\\n",
       "0    Smith, Peter Godfrey  Temas e Debates            2017   \n",
       "0       André, João Paulo          Gradiva            2018   \n",
       "0                Yong, Ed  Temas e Debates            2017   \n",
       "0    Rocha, Jaime, pseud.   Relógio d'Água       cop. 2017   \n",
       "0   Rowling, J. K., 1965-         Presença            2018   \n",
       "..                    ...              ...             ...   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "0                     NaN              NaN             NaN   \n",
       "\n",
       "                                              url_BNP  \n",
       "0   http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789896168117  \n",
       "0   http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789896417529  \n",
       "0     http://urn.bn.pt/isbn/mods/xml?id=9789722361569  \n",
       "..                                                ...  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "0                                                 NaN  \n",
       "\n",
       "[8466 rows x 6 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ISBN</th>\n      <th>titulo</th>\n      <th>autor</th>\n      <th>editora</th>\n      <th>data_publicacao</th>\n      <th>url_BNP</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>978-989-644-448-8</td>\n      <td>Outras mentes</td>\n      <td>Smith, Peter Godfrey</td>\n      <td>Temas e Debates</td>\n      <td>2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-616-811-7</td>\n      <td>Poções e paixões</td>\n      <td>André, João Paulo</td>\n      <td>Gradiva</td>\n      <td>2018</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789896168117</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-644-434-1</td>\n      <td>Nós, os micróbios e uma visão alargada da vida</td>\n      <td>Yong, Ed</td>\n      <td>Temas e Debates</td>\n      <td>2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=978-989-644-...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-989-641-752-9</td>\n      <td>Preparação para a noite</td>\n      <td>Rocha, Jaime, pseud.</td>\n      <td>Relógio d'Água</td>\n      <td>cop. 2017</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789896417529</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>978-972-23-6156-9</td>\n      <td>&lt;Os &gt;contos de Beedle o bardo</td>\n      <td>Rowling, J. K., 1965-</td>\n      <td>Presença</td>\n      <td>2018</td>\n      <td>http://urn.bn.pt/isbn/mods/xml?id=9789722361569</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>8466 rows × 6 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "maindata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0           Smith, Peter Godfrey\n",
       "0              André, João Paulo\n",
       "0                       Yong, Ed\n",
       "0           Rocha, Jaime, pseud.\n",
       "0          Rowling, J. K., 1965-\n",
       "                ...             \n",
       "0    Paolini, Christopher, 1983-\n",
       "0       McGrayne, Sharon Bertsch\n",
       "0                            NaN\n",
       "0                Azevedo, Susana\n",
       "0      Botelho, Margarida, 1979-\n",
       "Name: autor, Length: 5206, dtype: object"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "autores = maindata.drop_duplicates(subset=['titulo', 'autor'], keep='first')\n",
    "#autores[\"autor\"].value_counts().to_csv(\"autor_BNP_new.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "ISBN               2682\n",
       "titulo             2682\n",
       "autor              2706\n",
       "editora            2706\n",
       "data_publicacao    2706\n",
       "url_BNP            2706\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "maindata.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#autores.to_csv(\"isbn_bnp_sem_duplicados.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array(['Smith, Peter Godfrey', 'André, João Paulo', 'Yong, Ed', ...,\n",
       "       'Paolini, Christopher, 1983-', 'McGrayne, Sharon Bertsch',\n",
       "       'Azevedo, Susana'], dtype=object)"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "autores[\"autor\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}