Scrap EMPREGOS_FINAL.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>\n",
       "code_show=true; \n",
       "function code_toggle() {\n",
       " if (code_show){\n",
       " $('div.input').hide();\n",
       " } else {\n",
       " $('div.input').show();\n",
       " }\n",
       " code_show = !code_show\n",
       "} \n",
       "$( document ).ready(code_toggle);\n",
       "</script>\n",
       "<form action=\"javascript:code_toggle()\"><input type=\"submit\" value=\"Click here to toggle on/off the raw code.\"></form>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from IPython.display import HTML\n",
    "from IPython.display import Markdown as md\n",
    "import ipywidgets as widgets\n",
    "import time\n",
    "\n",
    "BUSCA = ['Engenheiro','Engineer','Instrumentacao','Instrumentation','Automacao','Automation']\n",
    "\n",
    "HTML('''<script>\n",
    "code_show=true; \n",
    "function code_toggle() {\n",
    " if (code_show){\n",
    " $('div.input').hide();\n",
    " } else {\n",
    " $('div.input').show();\n",
    " }\n",
    " code_show = !code_show\n",
    "} \n",
    "$( document ).ready(code_toggle);\n",
    "</script>\n",
    "<form action=\"javascript:code_toggle()\"><input type=\"submit\" value=\"Click here to toggle on/off the raw code.\"></form>''')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import wget\n",
    "from IPython.display import HTML\n",
    "from IPython.display import Markdown as md\n",
    "import ipywidgets as widgets\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.common.by import By\n",
    "\n",
    "def vagas(tags):\n",
    "    paginas = []\n",
    "    for a in tags:\n",
    "        x=a.find('a', href=True)\n",
    "        link = x['href']\n",
    "        texto = x.text.rstrip(\"\\n\")\n",
    "        titulo = texto.strip()\n",
    "        descricao = a.find('p').getText().strip()\n",
    "        x = a.find(\"span\", {\"class\": \"vaga-local\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            local = x.getText().strip()\n",
    "        else:\n",
    "            local = ''\n",
    "        x = a.find(\"span\", {\"class\": \"data-publicacao\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            data = x.getText().strip()\n",
    "        else:\n",
    "            data = ''\n",
    "        paginas.append([titulo, 'https://www.vagas.com.br/'+link, local, data, descricao])\n",
    "    return paginas\n",
    "\n",
    "def gera_vagas(busca):\n",
    "    URL = 'https://www.vagas.com.br/vagas-de-'+busca+'?ordenar_por=mais_recentes'\n",
    "    page = requests.get(URL)\n",
    "    soup = BeautifulSoup(page.content, 'html.parser')\n",
    "    tags = soup.select('li[class=\"vaga odd\"]')\n",
    "    list1 = vagas(tags)\n",
    "    paginas = []\n",
    "    tags = soup.select('li[class=\"vaga even\"]')\n",
    "    list2 = vagas(tags)\n",
    "    paginas = concatena_listas(list1,list2)\n",
    "    return imprimir([row[0] for row in paginas],[row[1] for row in paginas],[row[2] for row in paginas],[row[3] for row in paginas])\n",
    "\n",
    "\n",
    "##########################################################\n",
    "def catho(tags):\n",
    "    \"\"\"global paginas\n",
    "    global link\n",
    "    global a\n",
    "    global titulo\n",
    "    global descricao\n",
    "    global local\n",
    "    global data\"\"\"\n",
    "    paginas = []\n",
    "    for a in tags:\n",
    "        #print(len(a),type(a), a.text)\n",
    "        #x = a.find(\"a\", {\"class\": \"viewVagaAction\"})\n",
    "        x = a.find(\"a\", {\"class\": \"viewVagaAction\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            link = x['href']\n",
    "        else:\n",
    "            link = '' \n",
    "        x = a.find(\"a\", {\"class\": \"viewVagaAction\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            titulo = x['title']\n",
    "        else:\n",
    "            titulo = ''\n",
    "        x = a.find(\"div\", {\"class\": \"descriptionComplete\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            descricao = x.text.strip()\n",
    "        else:\n",
    "            descricao = ''\n",
    "        local = ''\n",
    "        x = a.find(\"time\", {\"class\": \"dataAnuncio\"})\n",
    "        if not isinstance(x, (type(None))):\n",
    "            data = x.text.strip()\n",
    "        else:\n",
    "            data = ''\n",
    "        paginas.append([titulo, link, local, data, descricao])\n",
    "    return paginas\n",
    "##########################################################\n",
    "def gera_catho(busca):\n",
    "    \"\"\"global paginas\n",
    "    global i\n",
    "    global page\n",
    "    global soup\n",
    "    global tags\n",
    "    global x\"\"\"\n",
    "    paginas = []\n",
    "    \n",
    "    \n",
    "    for i in range(1,5):\n",
    "        #time.sleep(2)\n",
    "        #URL = 'https://www.catho.com.br/vagas/'+busca+'/?q='+busca+'&pais_id=31&faixa_sal_id_combinar=1&perfil_id=1&order=dataAtualizacao&where_search=1&how_search=2&page='+str(i)\n",
    "        #URL = 'https://www.catho.com.br/vagas/'+busca+'/?q='+busca+'&order=dataAtualizacao&page='+str(i)\n",
    "        #page = requests.get(URL)\n",
    "        #soup = BeautifulSoup(page.content, 'html.parser')\n",
    "        #tags = soup.select('ul[class=\"gtm-class sc-fBuWsC eXuzGH\"]')\n",
    "        #print(len(tags),type(page))\n",
    "        \n",
    "        options = webdriver.ChromeOptions();\n",
    "        options.add_argument(\"--user-data-dir=/home/eduardo/.config/google-chrome\")\n",
    "        options.add_argument('--start-maximized')\n",
    "        driver = webdriver.Chrome(executable_path='./chromedriver', options=options)\n",
    "        print(i)\n",
    "        URL = 'https://www.catho.com.br/vagas/'+busca+'/?q='+busca+'&pais_id=31&faixa_sal_id_combinar=1&perfil_id=1&order=dataAtualizacao&where_search=1&how_search=2&page='+str(i)\n",
    "        driver.get(URL)        \n",
    "        time.sleep(3)\n",
    "        soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "        tags = soup.select('ul[id=\"listagemVagas\"]')\n",
    "        x=tags[0].find_all('li')\n",
    "        paginas += catho(x)\n",
    "        #driver.close()\n",
    "    #return paginas\n",
    "    return imprimir([row[0] for row in paginas],[row[1] for row in paginas],[row[2] for row in paginas],[row[3] for row in paginas])\n",
    "    \n",
    "##########################################################\n",
    "\n",
    "def linkedin(tags):\n",
    "    paginas = []\n",
    "    for a in tags:\n",
    "        link = a.find('a', href=True)['href']\n",
    "        titulo = a.find(\"h3\", {\"class\": \"result-card__title job-result-card__title\"}).text.strip()\n",
    "        local = a.find(\"span\", {\"class\": \"job-result-card__location\"}).text.strip()\n",
    "        data = a.find(\"time\", {\"class\": \"job-result-card__listdate--new\"})\n",
    "        if isinstance(data, (type(None))):\n",
    "            data=a.find(\"time\", {\"class\": \"job-result-card__listdate\"}).text\n",
    "        else:\n",
    "            data = data.text\n",
    "        #print(\"° \"+titulo,\"|\", local,\"|\", data, '\\n'+link)\n",
    "        paginas.append([titulo, link, local, data])\n",
    "    return paginas\n",
    "\n",
    "def gera_linkedin(busca):\n",
    "    #URL = 'https://www.linkedin.com/jobs/search/?f_E=4%2C5%2C6&geoId=106057199&keywords='+busca+'%20NOT%20Programador%20NOT%20devops%20NOT%20dados&location=Brasil&sortBy=DD'\n",
    "    paginas = []\n",
    "    paginacao = [0]\n",
    "    for z in paginacao:\n",
    "        URL= 'https://www.linkedin.com/jobs/search/?geoId=106057199&keywords='+busca+'%20NOT%20Programador%20NOT%20engenhariajob%20NOT%20devops%20NOT%20dados%20NOT%20seleta%20NOT%20desidy%20NOT%20pice%20NOT%20renner%20NOT%20Foodjobs.com.br&location=Brasil&sortBy=DD&start='+str(z)\n",
    "        #print(URL)\n",
    "        page = requests.get(URL)\n",
    "        soup = BeautifulSoup(page.content, 'html.parser')\n",
    "        tags = soup.select('ul[class=\"jobs-search__results-list\"]')\n",
    "        x=tags[0].find_all('li')\n",
    "        paginas += linkedin(x)\n",
    "    return imprimir([row[0] for row in paginas],[row[1] for row in paginas],[row[2] for row in paginas],[row[3] for row in paginas])\n",
    "\n",
    "##########################################################\n",
    "\n",
    "def concatena_listas(list1,list2):\n",
    "    sums_list = []\n",
    "    if len(list1)>len(list2):\n",
    "        listA = list2\n",
    "        listB = list1\n",
    "    else:\n",
    "        listA = list1\n",
    "        listB = list2\n",
    "    x = 0\n",
    "    while x < len(listA):\n",
    "        #print(x)\n",
    "        sums_list.append(listA[x])\n",
    "        sums_list.append(listB[x])\n",
    "        x += 1\n",
    "    for i in listB[x:]:\n",
    "        sums_list.append(i)\n",
    "    return sums_list\n",
    "\n",
    "\n",
    "def imprimir(titulo, link, local, data):\n",
    "    string1 =\"<OL>\"\n",
    "    a = -1\n",
    "    for tit in titulo:\n",
    "        a=a+1\n",
    "        if 'oje' in data[a] or 'hour' in data[a] or 'minut' in data[a]:\n",
    "            data1 = '<font color=\"red\"><b>'+data[a]+'</b></font>'\n",
    "        else:\n",
    "            data1 = data[a]\n",
    "        string1 = string1 + \"<LI>\"\n",
    "        string1 = string1 + '<a href=\"'+link[a]+'\">'+tit+'</a>'+' | '+ local[a]+' | '+data1\n",
    "        string1 = string1 + \"</LI>\"\n",
    "    string1=string1+\"</OL>\"\n",
    "    return HTML(string1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pesquisa no Vagas e Linkedin\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "## Última atualização 26/07 07:51"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "now = datetime.now()\n",
    "current_time = now.strftime(\"%d/%m %H:%M\")\n",
    "#print(\"Current Time =\", current_time)\n",
    "md(\"## Última atualização %s\"%(current_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vagas Ok Linkedin Ok\n"
     ]
    }
   ],
   "source": [
    "#children3 = [widgets.HTML(value=gera_catho(name).data, placeholder='', description='',) for name in BUSCA]\n",
    "#print(\"Catho Ok\", end =\" \")\n",
    "children2 = [widgets.HTML(value=gera_linkedin(name).data, placeholder='', description='',) for name in BUSCA]\n",
    "print(\"Vagas Ok\", end =\" \")\n",
    "children1 = [widgets.HTML(value=gera_vagas(name).data, placeholder='', description='',) for name in BUSCA]\n",
    "print(\"Linkedin Ok\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2e04278547a4ac58c926c865f7d528d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tab(children=(HTML(value='<OL><LI><a href=\"https://www.vagas.com.br//vagas/v2086523/engenheiro-mecanico-master…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#children  = children1 + children2 + children3\n",
    "children  = children1\n",
    "tab = widgets.Tab()\n",
    "tab.children = children\n",
    "for i in range(len(BUSCA)):\n",
    "    tab.set_title(i, 'Vagas: '+BUSCA[i])\n",
    "#for i in range(len(BUSCA), len(BUSCA)*2):\n",
    "#    tab.set_title(i, 'Linkedin: '+BUSCA[i-len(BUSCA)])\n",
    "#for i in range(len(BUSCA)*2,len(BUSCA)*3):\n",
    "#    tab.set_title(i, 'Catho: '+BUSCA[i-len(BUSCA)*2])\n",
    "tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c5c368d0c91429fb3d4a0d10989bcbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Tab(children=(HTML(value='<OL><LI><a href=\"https://br.linkedin.com/jobs/view/engenharia-ambiental-e-sanit%C3%A…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#children  = children1 + children2 + children3\n",
    "children  = children2 \n",
    "tab = widgets.Tab()\n",
    "tab.children = children\n",
    "for i in range(len(BUSCA)):\n",
    "    tab.set_title(i, 'Linkedin: '+BUSCA[i])\n",
    "tab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "'break' outside loop (<ipython-input-7-6aaf1f276005>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-7-6aaf1f276005>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    break\u001b[0m\n\u001b[0m    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m 'break' outside loop\n"
     ]
    }
   ],
   "source": [
    "break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=tags[0].find_all('li')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for a in x:\n",
    "    print(a.find(\"time\", {\"class\": \"dataAnuncio\"})['datetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "datetime_str = '09/19/18 13:55:26'\n",
    "\n",
    "datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S')\n",
    "\n",
    "print(type(datetime_object))\n",
    "print(datetime_object)  # printed in default format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aaa = x[0].find(\"time\", {\"class\": \"dataAnuncio\"})['datetime']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gera_catho(BUSCA[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup.select('ul[id=\"listagemVagas\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = soup.select('ul[class=\"gtm-class sc-fBuWsC eXuzGH\"]')\n",
    "x=tags[0].find_all('li')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}