mhlw_ocr.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mhlw_ocr.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMdeggffHwkDLmja5PduJ1W",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/imabari/ImabariScraping/blob/master/mhlw_ocr.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pt9O5n8Lc3gc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        },
        "outputId": "3c268afe-f695-4e52-cdca-a9bc37f82234"
      },
      "source": [
        "# データをダウンロード\n",
        "!wget https://www.mhlw.go.jp/content/10906000/000668093.pdf -O data.pdf"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-09-05 08:06:10--  https://www.mhlw.go.jp/content/10906000/000668093.pdf\n",
            "Resolving www.mhlw.go.jp (www.mhlw.go.jp)... 23.6.129.193, 2600:1407:21:2b9::3194, 2600:1407:21:28f::3194\n",
            "Connecting to www.mhlw.go.jp (www.mhlw.go.jp)|23.6.129.193|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 453107 (442K) [application/pdf]\n",
            "Saving to: ‘data.pdf’\n",
            "\n",
            "data.pdf            100%[===================>] 442.49K  --.-KB/s    in 0.05s   \n",
            "\n",
            "2020-09-05 08:06:11 (7.88 MB/s) - ‘data.pdf’ saved [453107/453107]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ugCimFfCdCBH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "4bde4e73-6450-442c-bd30-9528e210ad89"
      },
      "source": [
        "# PDFBOXをダウンロード\n",
        "!wget https://www-eu.apache.org/dist/pdfbox/2.0.21/pdfbox-app-2.0.21.jar -O pdfbox-app.jar"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-09-05 08:06:11--  https://www-eu.apache.org/dist/pdfbox/2.0.21/pdfbox-app-2.0.21.jar\n",
            "Resolving www-eu.apache.org (www-eu.apache.org)... 95.216.24.32, 2a01:4f9:2a:185f::2\n",
            "Connecting to www-eu.apache.org (www-eu.apache.org)|95.216.24.32|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://downloads.apache.org/pdfbox/2.0.21/pdfbox-app-2.0.21.jar [following]\n",
            "--2020-09-05 08:06:12--  https://downloads.apache.org/pdfbox/2.0.21/pdfbox-app-2.0.21.jar\n",
            "Resolving downloads.apache.org (downloads.apache.org)... 88.99.95.219, 2a01:4f8:10a:201a::2\n",
            "Connecting to downloads.apache.org (downloads.apache.org)|88.99.95.219|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9897381 (9.4M) [application/java-archive]\n",
            "Saving to: ‘pdfbox-app.jar’\n",
            "\n",
            "pdfbox-app.jar      100%[===================>]   9.44M  8.69MB/s    in 1.1s    \n",
            "\n",
            "2020-09-05 08:06:13 (8.69 MB/s) - ‘pdfbox-app.jar’ saved [9897381/9897381]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qL7Xuv5EM97a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# PDFから画像に変換\n",
        "!java -jar pdfbox-app.jar PDFToImage -imageType png -dpi 300 data.pdf"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "URJte09EMrCl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !java -jar pdfbox-app.jar ExtractImages data.pdf"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pk-ZtniNyVfb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d6c8f2de-c816-45ba-f203-012fac0e26ee"
      },
      "source": [
        "!add-apt-repository ppa:alex-p/tesseract-ocr -y\n",
        "\n",
        "!apt update\n",
        "\n",
        "!apt install tesseract-ocr\n",
        "!apt install libtesseract-dev\n",
        "\n",
        "!tesseract -v\n",
        "\n",
        "!apt install tesseract-ocr-jpn  tesseract-ocr-jpn-vert\n",
        "!apt install tesseract-ocr-script-jpan tesseract-ocr-script-jpan-vert\n",
        "\n",
        "!tesseract --list-langs\n",
        "!pip install pytesseract"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rGet:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [1 InRelease 2,586 B/88.7\r                                                                               \rGet:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease [3,626 B]\n",
            "Ign:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Get:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release [564 B]\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Get:8 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic InRelease [15.4 kB]\n",
            "Get:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release.gpg [833 B]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Get:11 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n",
            "Get:12 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ Packages [95.7 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:14 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [109 kB]\n",
            "Get:15 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease [15.4 kB]\n",
            "Get:16 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [1,067 kB]\n",
            "Get:17 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [892 kB]\n",
            "Get:19 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Packages [47.5 kB]\n",
            "Get:20 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 Packages [26.5 kB]\n",
            "Get:21 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [43.0 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [125 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [1,362 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1,421 kB]\n",
            "Get:25 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main Sources [1,864 kB]\n",
            "Get:26 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main amd64 Packages [900 kB]\n",
            "Fetched 8,262 kB in 5s (1,807 kB/s)\n",
            "Reading package lists... Done\n",
            "Ign:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:2 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic InRelease\n",
            "Hit:3 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:6 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:8 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:9 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Hit:10 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:11 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease\n",
            "Hit:14 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "74 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  libtesseract4 tesseract-ocr-eng tesseract-ocr-osd\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr tesseract-ocr-eng tesseract-ocr-osd\n",
            "The following packages will be upgraded:\n",
            "  libtesseract4\n",
            "1 upgraded, 3 newly installed, 0 to remove and 73 not upgraded.\n",
            "Need to get 6,161 kB of archives.\n",
            "After this operation, 16.3 MB of additional disk space will be used.\n",
            "Get:1 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 libtesseract4 amd64 4.1.1+git4247-97079fa3-1ppa1~bionic1 [1,260 kB]\n",
            "Get:2 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-eng all 1:4.0.0+git39-6572757-1ppa1~bionic1 [1,589 kB]\n",
            "Get:3 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-osd all 1:4.0.0+git39-6572757-1ppa1~bionic1 [2,992 kB]\n",
            "Get:4 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr amd64 4.1.1+git4247-97079fa3-1ppa1~bionic1 [320 kB]\n",
            "Fetched 6,161 kB in 4s (1,754 kB/s)\n",
            "(Reading database ... 144579 files and directories currently installed.)\n",
            "Preparing to unpack .../libtesseract4_4.1.1+git4247-97079fa3-1ppa1~bionic1_amd64.deb ...\n",
            "Unpacking libtesseract4:amd64 (4.1.1+git4247-97079fa3-1ppa1~bionic1) over (4.00~git2288-10f4998a-2) ...\n",
            "Selecting previously unselected package tesseract-ocr-eng.\n",
            "Preparing to unpack .../tesseract-ocr-eng_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-eng (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Selecting previously unselected package tesseract-ocr-osd.\n",
            "Preparing to unpack .../tesseract-ocr-osd_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-osd (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Selecting previously unselected package tesseract-ocr.\n",
            "Preparing to unpack .../tesseract-ocr_4.1.1+git4247-97079fa3-1ppa1~bionic1_amd64.deb ...\n",
            "Unpacking tesseract-ocr (4.1.1+git4247-97079fa3-1ppa1~bionic1) ...\n",
            "Setting up libtesseract4:amd64 (4.1.1+git4247-97079fa3-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-osd (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-eng (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr (4.1.1+git4247-97079fa3-1ppa1~bionic1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  libarchive-dev liblept5 libleptonica-dev\n",
            "The following NEW packages will be installed:\n",
            "  libarchive-dev libleptonica-dev libtesseract-dev\n",
            "The following packages will be upgraded:\n",
            "  liblept5\n",
            "1 upgraded, 3 newly installed, 0 to remove and 72 not upgraded.\n",
            "Need to get 4,327 kB of archives.\n",
            "After this operation, 15.2 MB of additional disk space will be used.\n",
            "Get:1 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 liblept5 amd64 1.78.0-1+nmu1ppa1~bionic1 [985 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libarchive-dev amd64 3.2.2-3.1ubuntu0.6 [470 kB]\n",
            "Get:3 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 libleptonica-dev amd64 1.78.0-1+nmu1ppa1~bionic1 [1,375 kB]\n",
            "Get:4 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 libtesseract-dev amd64 4.1.1+git4247-97079fa3-1ppa1~bionic1 [1,496 kB]\n",
            "Fetched 4,327 kB in 3s (1,607 kB/s)\n",
            "Selecting previously unselected package libarchive-dev:amd64.\n",
            "(Reading database ... 144631 files and directories currently installed.)\n",
            "Preparing to unpack .../libarchive-dev_3.2.2-3.1ubuntu0.6_amd64.deb ...\n",
            "Unpacking libarchive-dev:amd64 (3.2.2-3.1ubuntu0.6) ...\n",
            "Preparing to unpack .../liblept5_1.78.0-1+nmu1ppa1~bionic1_amd64.deb ...\n",
            "Unpacking liblept5 (1.78.0-1+nmu1ppa1~bionic1) over (1.75.3-3) ...\n",
            "Selecting previously unselected package libleptonica-dev.\n",
            "Preparing to unpack .../libleptonica-dev_1.78.0-1+nmu1ppa1~bionic1_amd64.deb ...\n",
            "Unpacking libleptonica-dev (1.78.0-1+nmu1ppa1~bionic1) ...\n",
            "Selecting previously unselected package libtesseract-dev:amd64.\n",
            "Preparing to unpack .../libtesseract-dev_4.1.1+git4247-97079fa3-1ppa1~bionic1_amd64.deb ...\n",
            "Unpacking libtesseract-dev:amd64 (4.1.1+git4247-97079fa3-1ppa1~bionic1) ...\n",
            "Setting up liblept5 (1.78.0-1+nmu1ppa1~bionic1) ...\n",
            "Setting up libarchive-dev:amd64 (3.2.2-3.1ubuntu0.6) ...\n",
            "Setting up libleptonica-dev (1.78.0-1+nmu1ppa1~bionic1) ...\n",
            "Setting up libtesseract-dev:amd64 (4.1.1+git4247-97079fa3-1ppa1~bionic1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "tesseract 4.1.1-rc2-25-g9707\n",
            " leptonica-1.78.0\n",
            "  libgif 5.1.4 : libjpeg 8d (libjpeg-turbo 1.5.2) : libpng 1.6.34 : libtiff 4.0.9 : zlib 1.2.11 : libwebp 0.6.1 : libopenjp2 2.3.0\n",
            " Found AVX2\n",
            " Found AVX\n",
            " Found FMA\n",
            " Found SSE\n",
            " Found libarchive 3.2.2 zlib/1.2.11 liblzma/5.2.2 bz2lib/1.0.6 liblz4/1.7.1\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr-jpn tesseract-ocr-jpn-vert\n",
            "0 upgraded, 2 newly installed, 0 to remove and 72 not upgraded.\n",
            "Need to get 3,281 kB of archives.\n",
            "After this operation, 5,539 kB of additional disk space will be used.\n",
            "Get:1 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-jpn all 1:4.0.0+git39-6572757-1ppa1~bionic1 [1,391 kB]\n",
            "Get:2 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-jpn-vert all 1:4.0.0+git39-6572757-1ppa1~bionic1 [1,890 kB]\n",
            "Fetched 3,281 kB in 2s (1,704 kB/s)\n",
            "Selecting previously unselected package tesseract-ocr-jpn.\n",
            "(Reading database ... 144759 files and directories currently installed.)\n",
            "Preparing to unpack .../tesseract-ocr-jpn_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-jpn (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Selecting previously unselected package tesseract-ocr-jpn-vert.\n",
            "Preparing to unpack .../tesseract-ocr-jpn-vert_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-jpn-vert (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-jpn (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-jpn-vert (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-440\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr-script-jpan tesseract-ocr-script-jpan-vert\n",
            "0 upgraded, 2 newly installed, 0 to remove and 72 not upgraded.\n",
            "Need to get 5,670 kB of archives.\n",
            "After this operation, 12.4 MB of additional disk space will be used.\n",
            "Get:1 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-script-jpan all 1:4.0.0+git39-6572757-1ppa1~bionic1 [2,586 kB]\n",
            "Get:2 http://ppa.launchpad.net/alex-p/tesseract-ocr/ubuntu bionic/main amd64 tesseract-ocr-script-jpan-vert all 1:4.0.0+git39-6572757-1ppa1~bionic1 [3,084 kB]\n",
            "Fetched 5,670 kB in 2s (2,812 kB/s)\n",
            "Selecting previously unselected package tesseract-ocr-script-jpan.\n",
            "(Reading database ... 144767 files and directories currently installed.)\n",
            "Preparing to unpack .../tesseract-ocr-script-jpan_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-script-jpan (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Selecting previously unselected package tesseract-ocr-script-jpan-vert.\n",
            "Preparing to unpack .../tesseract-ocr-script-jpan-vert_1%3a4.0.0+git39-6572757-1ppa1~bionic1_all.deb ...\n",
            "Unpacking tesseract-ocr-script-jpan-vert (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-script-jpan (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "Setting up tesseract-ocr-script-jpan-vert (1:4.0.0+git39-6572757-1ppa1~bionic1) ...\n",
            "List of available languages (6):\n",
            "Japanese\n",
            "Japanese_vert\n",
            "eng\n",
            "jpn\n",
            "jpn_vert\n",
            "osd\n",
            "Collecting pytesseract\n",
            "  Downloading https://files.pythonhosted.org/packages/17/4b/4dbd55388225bb6cd243d21f70e77cb3ce061e241257485936324b8e920f/pytesseract-0.3.6.tar.gz\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.6/dist-packages (from pytesseract) (7.0.0)\n",
            "Building wheels for collected packages: pytesseract\n",
            "  Building wheel for pytesseract (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pytesseract: filename=pytesseract-0.3.6-py2.py3-none-any.whl size=13629 sha256=1931850af5bd43f0c267656a51d4b29316cd87291886034c9c737967c1b8f833\n",
            "  Stored in directory: /root/.cache/pip/wheels/ee/71/72/b98430261d849ae631e283dfc7ccb456a3fb2ed2205714b63f\n",
            "Successfully built pytesseract\n",
            "Installing collected packages: pytesseract\n",
            "Successfully installed pytesseract-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "51vJQPiXi-h5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "\n",
        "import cv2\n",
        "import numpy as np\n",
        "import pytesseract"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYwr-CWwjpnZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab.patches import cv2_imshow"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sA4l5kE_9CFm",
        "colab_type": "text"
      },
      "source": [
        "参考：[OpenCVでの表のセルの認識方法](https://teratail.com/questions/151317)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N-_uunzkxhO1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img = cv2.imread(\"data1.png\")\n",
        "\n",
        "# BGR -> グレースケール\n",
        "gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "gray[gray > 220] = 255\n",
        "cv2.imwrite(\"gray.png\", gray)\n",
        "\n",
        "# エッジ抽出 (Canny)\n",
        "edges = cv2.Canny(gray, 1, 100, apertureSize=3)\n",
        "cv2.imwrite(\"edges.png\", edges)\n",
        "\n",
        "# 膨張処理\n",
        "kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))\n",
        "edges = cv2.dilate(edges, kernel)\n",
        "\n",
        "# 輪郭抽出\n",
        "contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DSH1TrpZho2Y",
        "colab": {}
      },
      "source": [
        "# 面積でフィルタリング\n",
        "rects = []\n",
        "\n",
        "test = []\n",
        "\n",
        "for cnt, hrchy in zip(contours, hierarchy[0]):\n",
        "    if cv2.contourArea(cnt) < 2700:\n",
        "        continue  # 面積が小さいものは除く\n",
        "    if hrchy[3] == -1:\n",
        "        continue  # ルートノードは除く\n",
        "\n",
        "    # 輪郭を囲む長方形を計算する。\n",
        "    rect = cv2.minAreaRect(cnt)\n",
        "    rect_points = cv2.boxPoints(rect).astype(int)\n",
        "\n",
        "    # 座標\n",
        "    x_max, y_max = np.amax(rect_points.T, axis=1)\n",
        "    x_min, y_min = np.amin(rect_points.T, axis=1)\n",
        "\n",
        "    rects.append([y_min, y_max , x_min, x_max])"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6Lb-sz5zv4UX",
        "colab": {}
      },
      "source": [
        "# x-y 順でソート\n",
        "rects = sorted(rects, key=lambda x: (x[1], x[2]))\n",
        "\n",
        "data = []\n",
        "\n",
        "for i, rect in enumerate(rects):\n",
        "\n",
        "    y_min, y_max , x_min, x_max = rect\n",
        "\n",
        "    # 各座標ごとに切り出し\n",
        "    dst = gray[y_min + 1 : y_max - 1, x_min + 5 : x_max]\n",
        "    # cv2.imwrite(f\"gray{i:03}.png\", dst)\n",
        "\n",
        "    txt = (\n",
        "        pytesseract.image_to_string(dst, lang=\"jpn\", config=\"--psm 6\")\n",
        "        .strip()\n",
        "        .replace(\",\", \"\")\n",
        "        .replace(\".\", \"\")\n",
        "        .replace(\" \", \"\")\n",
        "    )\n",
        "    data.append(txt)\n",
        "\n",
        "    # print(\"rect:\\n\", rect)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-pDDsnGJxjOi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5610468f-ccea-4aaf-fda4-0c33ca86e919"
      },
      "source": [
        "cv2.imwrite(\"img.png\", img)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uvUvwYV4m3LJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YK6yQmtS4bqz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.DataFrame(rects, columns=[\"y1\", \"y2\", \"x1\", \"x2\"])"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PEFpwuqN55Py",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"text\"] = data"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AI46zq4yK4uI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t1 = set(df[\"x1\"].unique()) & set(df[\"x1\"].unique() + 1)\n",
        "\n",
        "for i in t1:\n",
        "    df[\"x1\"] = df[\"x1\"].replace(i, i - 1)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oDOxVKrdLLNI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "t2 = set(df[\"y2\"].unique()) & set(df[\"y2\"].unique() + 1)\n",
        "\n",
        "for i in t2:\n",
        "    df[\"y2\"] = df[\"y2\"].replace(i, i - 1)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BzvDDnAa6KfC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pv = df.pivot_table(index=\"y2\", columns=\"x1\", values=\"text\", aggfunc=lambda x: \" \".join(str(v) for v in x))"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0I_5XLvcQVAD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pv.dropna(thresh=7, inplace=True)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKVcHP7sRE_c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pv.dropna(how=\"all\", axis=1, inplace=True)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sWLK1Khu7JBz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pv.to_csv(\"data.csv\", index=False, header=False)"
      ],
      "execution_count": 20,
      "outputs": []
    }
  ]
}