dev_get_BV.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "abf9890d-ad51-4fca-a922-a49ec2d10e6e",
   "metadata": {},
   "source": [
    "sujets:\n",
    "\n",
    "*mettre dans une fonction\n",
    "\n",
    "*à quoi ça sert ensuite\n",
    "\n",
    "*progression des années\n",
    "\n",
    "*verify = F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "889a9797-9715-48f9-9fc0-66990c1ad2cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests in /opt/mamba/lib/python3.9/site-packages (from -r requirements.txt (line 1)) (2.28.1)\n",
      "Requirement already satisfied: py7zr in /opt/mamba/lib/python3.9/site-packages (from -r requirements.txt (line 2)) (0.20.0)\n",
      "Requirement already satisfied: geopandas in /opt/mamba/lib/python3.9/site-packages (from -r requirements.txt (line 3)) (0.11.1)\n",
      "Requirement already satisfied: openpyxl in /opt/mamba/lib/python3.9/site-packages (from -r requirements.txt (line 4)) (3.0.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/mamba/lib/python3.9/site-packages (from requests->-r requirements.txt (line 1)) (2022.9.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/mamba/lib/python3.9/site-packages (from requests->-r requirements.txt (line 1)) (3.3)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /opt/mamba/lib/python3.9/site-packages (from requests->-r requirements.txt (line 1)) (2.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/mamba/lib/python3.9/site-packages (from requests->-r requirements.txt (line 1)) (1.26.11)\n",
      "Requirement already satisfied: inflate64>=0.3.0 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (0.3.0)\n",
      "Requirement already satisfied: pyppmd<0.19.0,>=0.18.1 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (0.18.3)\n",
      "Requirement already satisfied: pybcj>=0.6.0 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (1.0.1)\n",
      "Requirement already satisfied: pyzstd>=0.14.4 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (0.15.3)\n",
      "Requirement already satisfied: multivolumefile>=0.2.3 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (0.2.3)\n",
      "Requirement already satisfied: brotli>=1.0.9 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (1.0.9)\n",
      "Requirement already satisfied: pycryptodomex>=3.6.6 in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (3.15.0)\n",
      "Requirement already satisfied: psutil in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (5.9.2)\n",
      "Requirement already satisfied: texttable in /opt/mamba/lib/python3.9/site-packages (from py7zr->-r requirements.txt (line 2)) (1.6.4)\n",
      "Requirement already satisfied: shapely<2,>=1.7 in /opt/mamba/lib/python3.9/site-packages (from geopandas->-r requirements.txt (line 3)) (1.8.0)\n",
      "Requirement already satisfied: pandas>=1.0.0 in /opt/mamba/lib/python3.9/site-packages (from geopandas->-r requirements.txt (line 3)) (1.4.4)\n",
      "Requirement already satisfied: packaging in /opt/mamba/lib/python3.9/site-packages (from geopandas->-r requirements.txt (line 3)) (21.3)\n",
      "Requirement already satisfied: fiona>=1.8 in /opt/mamba/lib/python3.9/site-packages (from geopandas->-r requirements.txt (line 3)) (1.8.20)\n",
      "Requirement already satisfied: pyproj>=2.6.1.post1 in /opt/mamba/lib/python3.9/site-packages (from geopandas->-r requirements.txt (line 3)) (3.3.1)\n",
      "Requirement already satisfied: et-xmlfile in /opt/mamba/lib/python3.9/site-packages (from openpyxl->-r requirements.txt (line 4)) (1.1.0)\n",
      "Requirement already satisfied: attrs>=17 in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (22.1.0)\n",
      "Requirement already satisfied: click>=4.0 in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (7.1.2)\n",
      "Requirement already satisfied: cligj>=0.5 in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (0.7.2)\n",
      "Requirement already satisfied: click-plugins>=1.0 in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (1.1.1)\n",
      "Requirement already satisfied: six>=1.7 in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (1.15.0)\n",
      "Requirement already satisfied: munch in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (2.5.0)\n",
      "Requirement already satisfied: setuptools in /opt/mamba/lib/python3.9/site-packages (from fiona>=1.8->geopandas->-r requirements.txt (line 3)) (59.8.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/mamba/lib/python3.9/site-packages (from pandas>=1.0.0->geopandas->-r requirements.txt (line 3)) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/mamba/lib/python3.9/site-packages (from pandas>=1.0.0->geopandas->-r requirements.txt (line 3)) (2022.2.1)\n",
      "Requirement already satisfied: numpy>=1.18.5 in /opt/mamba/lib/python3.9/site-packages (from pandas>=1.0.0->geopandas->-r requirements.txt (line 3)) (1.19.5)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/mamba/lib/python3.9/site-packages (from packaging->geopandas->-r requirements.txt (line 3)) (3.0.9)\n"
     ]
    }
   ],
   "source": [
    "!pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c9555737-84e5-4099-9cc2-b0b28de8aa6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import ftplib\n",
    "import glob\n",
    "import re\n",
    "import typing\n",
    "import tempfile\n",
    "import yaml\n",
    "import py7zr\n",
    "import geopandas as gpd\n",
    "import requests #added\n",
    "from tqdm import tqdm\n",
    "import zipfile\n",
    "import pandas as pd\n",
    "import dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c6de750-3a48-4e35-9593-7d6a58cac057",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://www.insee.fr/fr/statistiques/fichier/2115016/BV2012_au_01-01-2022.zip'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict_open_data = dev.import_yaml_config()\n",
    "url = dict_open_data['Insee']\\\n",
    "        ['BV'][2022][\"file\"]\n",
    "url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cf9a5fcd-10c2-434d-9b10-5247dc004e4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/mamba/lib/python3.9/site-packages/urllib3/connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'www.insee.fr'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n",
      "Downloading: : 1.23MiB [00:00, 10.0MiB/s]\n"
     ]
    }
   ],
   "source": [
    "#from dev import safe_download_write\n",
    "localisation = dev.safe_download_write(\n",
    "    url,\n",
    "    location = None,\n",
    "    param_ftp =None,\n",
    "    ext = \".zip\", verify = False, force = True)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "80a853e4-a768-416c-867e-2903aff2551d",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_name = localisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f781b930-3bf5-4db9-af77-041f2c465cde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.path.exists(location)\n",
    "#os.listdir(location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8bc4b479-d427-4e58-a8ac-648c01d62d86",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = tempfile.TemporaryDirectory()\n",
    "location = tmp.name\n",
    "# unzip in location directory\n",
    "\n",
    "archive = zipfile.ZipFile(out_name, 'r')\n",
    "archive.extractall(path=location)\n",
    "archive.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8d2bde07-fecb-40b8-9534-c53e17451b45",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_excel(location+\"/BV2012_au_01-01-2022.xlsx\",sheet_name=\"Composition_communale\",skiprows=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f2fcf9ed-47e1-492d-b5ec-2d1d6c287a1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CODGEO</th>\n",
       "      <th>BV2012</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01001</td>\n",
       "      <td>01093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01002</td>\n",
       "      <td>01004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01004</td>\n",
       "      <td>01004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01005</td>\n",
       "      <td>69123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01006</td>\n",
       "      <td>01034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34933</th>\n",
       "      <td>97420</td>\n",
       "      <td>97411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34934</th>\n",
       "      <td>97421</td>\n",
       "      <td>97409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34935</th>\n",
       "      <td>97422</td>\n",
       "      <td>97416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34936</th>\n",
       "      <td>97423</td>\n",
       "      <td>97415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34937</th>\n",
       "      <td>97424</td>\n",
       "      <td>97414</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>34938 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      CODGEO BV2012\n",
       "0      01001  01093\n",
       "1      01002  01004\n",
       "2      01004  01004\n",
       "3      01005  69123\n",
       "4      01006  01034\n",
       "...      ...    ...\n",
       "34933  97420  97411\n",
       "34934  97421  97409\n",
       "34935  97422  97416\n",
       "34936  97423  97415\n",
       "34937  97424  97414\n",
       "\n",
       "[34938 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['BV2012'] != \"ZZZZZ\"][['CODGEO','BV2012']] #ZZZZZ à Mayotte"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a512df2-eeb4-4aa7-ba24-90a10863fe48",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}