transforms/universal/digest/digest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4c45c3c6-e4d7-4e61-8de6-32d61f2ce695",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "%pip install data-prep-toolkit>=0.2.3\n",
    "%pip install pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebf1f782-0e61-485c-8670-81066beb734c",
   "metadata": {},
   "source": [
    "##### ***** Import required classes and modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9669273a-8fcc-4b40-9b20-8df658e2ab58",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dpk_digest.runtime import Digest"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7234563c-2924-4150-8a31-4aec98c1bf33",
   "metadata": {},
   "source": [
    "##### ***** Setup runtime parameters for this transform and invoke the transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "badafb96-64d2-4bb8-9f3e-b23713fd5c3f",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "21:46:45 INFO - pipeline id pipeline_id\n",
      "21:46:45 INFO - code location None\n",
      "21:46:45 INFO - data factory data_ is using local data access: input_folder - test-data/input output_folder - test-data/output\n",
      "21:46:45 INFO - data factory data_ max_files -1, n_sample -1\n",
      "21:46:45 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.parquet'], files to checkpoint ['.parquet']\n",
      "21:46:45 INFO - orchestrator digest started at 2025-01-07 21:46:45\n",
      "21:46:45 INFO - Number of files is 1, source profile {'max_file_size': 0.013697624206542969, 'min_file_size': 0.013697624206542969, 'total_file_size': 0.013697624206542969}\n",
      "21:46:45 INFO - Completed 1 files (100.0%) in 0.0 min\n",
      "21:46:45 INFO - Done processing 1 files, waiting for flush() completion.\n",
      "21:46:45 INFO - done flushing in 0.0 sec\n",
      "21:46:45 INFO - Completed execution in 0.0 min, execution result 0\n"
     ]
    }
   ],
   "source": [
    "%%capture\n",
    "Digest(input_folder= \"test-data/input\",\n",
    "        output_folder= \"test-data/output\",\n",
    "        digest_algorithm= \"sha256\").transform()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "845a75cf-f4a9-467d-87fa-ccbac1c9beb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document_id</th>\n",
       "      <th>contents</th>\n",
       "      <th>digest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>600fb9cb986bebc4344dbdaba34a2a212d2cd0d51ecb03...</td>\n",
       "      <td>Chapter 1.</td>\n",
       "      <td>600fb9cb986bebc4344dbdaba34a2a212d2cd0d51ecb03...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...</td>\n",
       "      <td>1</td>\n",
       "      <td>6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10699f104b8aafb34b542e9c28b9c5b4f22f585fbf22e9...</td>\n",
       "      <td>Recent news headlines are filled with reports ...</td>\n",
       "      <td>10699f104b8aafb34b542e9c28b9c5b4f22f585fbf22e9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>292293706e6df4757a4b3ddba7fffa4173aedb161acb56...</td>\n",
       "      <td>Businesses must make a serious effort to secur...</td>\n",
       "      <td>292293706e6df4757a4b3ddba7fffa4173aedb161acb56...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a796bd66e5fc1dd55abb1cf92115cbef6086ea6ad0c21c...</td>\n",
       "      <td>This chapter describes how you can secure and ...</td>\n",
       "      <td>a796bd66e5fc1dd55abb1cf92115cbef6086ea6ad0c21c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e34c9152459fe1c55a1f6bde873463739daef0924a8588...</td>\n",
       "      <td>/SM590000 Security fundamentals\\n/SM590000 Cur...</td>\n",
       "      <td>e34c9152459fe1c55a1f6bde873463739daef0924a8588...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>620c2496ae6d48ac61d4457921c56816c0605c1ad041de...</td>\n",
       "      <td>1 http://www.idtheftcenter.org</td>\n",
       "      <td>620c2496ae6d48ac61d4457921c56816c0605c1ad041de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>424462748fbd7f641237d60c2a7e4e68ab7f69e78dbf76...</td>\n",
       "      <td>2 http://www.ponemon.org /</td>\n",
       "      <td>424462748fbd7f641237d60c2a7e4e68ab7f69e78dbf76...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>33b240f4ce154d4a321f0826c92a20294ac1cb844e99c4...</td>\n",
       "      <td>' Copyright IBM Corp. 2014. All rights reserved.</td>\n",
       "      <td>33b240f4ce154d4a321f0826c92a20294ac1cb844e99c4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...</td>\n",
       "      <td>1</td>\n",
       "      <td>6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>f6f5ac281bee0775c514256bafa10bbdc2feb43e5a37ef...</td>\n",
       "      <td>Before reviewing database security techniques,...</td>\n",
       "      <td>f6f5ac281bee0775c514256bafa10bbdc2feb43e5a37ef...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>879e30e151e0dd5e14fe28388ea2ff11c14689c94bcefa...</td>\n",
       "      <td>/SM590000 First, and most important, is the de...</td>\n",
       "      <td>879e30e151e0dd5e14fe28388ea2ff11c14689c94bcefa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>fa0d8a17d198b8c8b37c1d58fbb3a80d7db91ec255aeeb...</td>\n",
       "      <td>A security policy is what defines whether the ...</td>\n",
       "      <td>fa0d8a17d198b8c8b37c1d58fbb3a80d7db91ec255aeeb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>10376eca1e378b771989536b4b345dca2ff35ee3753b3b...</td>\n",
       "      <td>/SM590000 The second fundamental in securing d...</td>\n",
       "      <td>10376eca1e378b771989536b4b345dca2ff35ee3753b3b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>c0848aa36ff34140621161a2d83637d9efc4a98324e838...</td>\n",
       "      <td>With your eyes now open to the importance of s...</td>\n",
       "      <td>c0848aa36ff34140621161a2d83637d9efc4a98324e838...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>6b81311fec7662e1a982a2bb412cd5c3cc00dc61f8988f...</td>\n",
       "      <td>Because of the inherently secure nature of IBM...</td>\n",
       "      <td>6b81311fec7662e1a982a2bb412cd5c3cc00dc61f8988f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>74e66d5a2490de1609273ad10b9ae2580bc77fdf1cc6d6...</td>\n",
       "      <td>Even more disturbing is that many IBM i client...</td>\n",
       "      <td>74e66d5a2490de1609273ad10b9ae2580bc77fdf1cc6d6...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3407d15eaf3f2de08928ad95f078552310f63856d62af7...</td>\n",
       "      <td>Traditionally, IBM i applications have employe...</td>\n",
       "      <td>3407d15eaf3f2de08928ad95f078552310f63856d62af7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f...</td>\n",
       "      <td>2</td>\n",
       "      <td>d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...</td>\n",
       "      <td>Row and Column Access Control Support in IBM D...</td>\n",
       "      <td>e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>d5c6192b84923978f203069f9dbc7ecbf6f8abb895e543...</td>\n",
       "      <td>Some clients using this default configuration ...</td>\n",
       "      <td>d5c6192b84923978f203069f9dbc7ecbf6f8abb895e543...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>c70d653f0120cb4d6afd973ae2293ca16d2ba7015003aa...</td>\n",
       "      <td>Although exit-point solutions can provide grea...</td>\n",
       "      <td>c70d653f0120cb4d6afd973ae2293ca16d2ba7015003aa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>94a433071544443ee89af9d2bb477505467d275c4fae8f...</td>\n",
       "      <td>An exit-point solution is a good option for da...</td>\n",
       "      <td>94a433071544443ee89af9d2bb477505467d275c4fae8f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>d4f3f84956d95009e65e236660283b8618b35b6e7fff34...</td>\n",
       "      <td>As described in 1.2, \"Current state of IBM i s...</td>\n",
       "      <td>d4f3f84956d95009e65e236660283b8618b35b6e7fff34...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>73a957551be639c1054fa74f08b02dbe71eac81910a857...</td>\n",
       "      <td>As shown in Figure 1-1, it is an all-or-nothin...</td>\n",
       "      <td>73a957551be639c1054fa74f08b02dbe71eac81910a857...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1fa65e948e99538b45a1efe30f4d9f075fbb010f24fcd7...</td>\n",
       "      <td>Figure 1-1   All-or-nothing access to the rows...</td>\n",
       "      <td>1fa65e948e99538b45a1efe30f4d9f075fbb010f24fcd7...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...</td>\n",
       "      <td>Chapter 1. Securing and protecting IBM DB2 data</td>\n",
       "      <td>18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>4e07408562bedb8b60ce05c1decfe3ad16b72230967de0...</td>\n",
       "      <td>3</td>\n",
       "      <td>4e07408562bedb8b60ce05c1decfe3ad16b72230967de0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328c...</td>\n",
       "      <td>4</td>\n",
       "      <td>4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...</td>\n",
       "      <td>Row and Column Access Control Support in IBM D...</td>\n",
       "      <td>e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>a9d5cf4f7c129b91df654ca180e0ede4a3859556522ba9...</td>\n",
       "      <td>Many businesses are trying to limit data acces...</td>\n",
       "      <td>a9d5cf4f7c129b91df654ca180e0ede4a3859556522ba9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>5cf93c80c646593fc74913fe54d28530065425102eac7d...</td>\n",
       "      <td>Some IBM i clients have tried augmenting the a...</td>\n",
       "      <td>5cf93c80c646593fc74913fe54d28530065425102eac7d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>153ce8fe205ee16af5261d8156f9d9247ad92a8c1b5e4d...</td>\n",
       "      <td>Using SQL views to limit access to a subset of...</td>\n",
       "      <td>153ce8fe205ee16af5261d8156f9d9247ad92a8c1b5e4d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>4eadf571d2362eb10c8f369e1cd0112df227d9b8a51237...</td>\n",
       "      <td>Even if you are willing to live with these per...</td>\n",
       "      <td>4eadf571d2362eb10c8f369e1cd0112df227d9b8a51237...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>353c07e0a64fd4d962f4795671d1f90ca8688e6ee83d3a...</td>\n",
       "      <td>Figure 1-2   Existing row and column controls</td>\n",
       "      <td>353c07e0a64fd4d962f4795671d1f90ca8688e6ee83d3a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>474e27900e1c69c9ee944e7d458505a29f1925edc06582...</td>\n",
       "      <td>Based on the challenges that are associated wi...</td>\n",
       "      <td>474e27900e1c69c9ee944e7d458505a29f1925edc06582...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>3fe6a85383792e9554a38aebec0b1e6d292be422f4b8a5...</td>\n",
       "      <td>The new DB2 RCAC support provides a method for...</td>\n",
       "      <td>3fe6a85383792e9554a38aebec0b1e6d292be422f4b8a5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...</td>\n",
       "      <td>Chapter 1. Securing and protecting IBM DB2 data</td>\n",
       "      <td>18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>ef2d127de37b942baad06145e54b0c619a1f22327b2ebb...</td>\n",
       "      <td>5</td>\n",
       "      <td>ef2d127de37b942baad06145e54b0c619a1f22327b2ebb...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          document_id  \\\n",
       "0   600fb9cb986bebc4344dbdaba34a2a212d2cd0d51ecb03...   \n",
       "1   6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...   \n",
       "2   10699f104b8aafb34b542e9c28b9c5b4f22f585fbf22e9...   \n",
       "3   292293706e6df4757a4b3ddba7fffa4173aedb161acb56...   \n",
       "4   a796bd66e5fc1dd55abb1cf92115cbef6086ea6ad0c21c...   \n",
       "5   e34c9152459fe1c55a1f6bde873463739daef0924a8588...   \n",
       "6   620c2496ae6d48ac61d4457921c56816c0605c1ad041de...   \n",
       "7   424462748fbd7f641237d60c2a7e4e68ab7f69e78dbf76...   \n",
       "8   33b240f4ce154d4a321f0826c92a20294ac1cb844e99c4...   \n",
       "9   6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...   \n",
       "10  f6f5ac281bee0775c514256bafa10bbdc2feb43e5a37ef...   \n",
       "11  879e30e151e0dd5e14fe28388ea2ff11c14689c94bcefa...   \n",
       "12  fa0d8a17d198b8c8b37c1d58fbb3a80d7db91ec255aeeb...   \n",
       "13  10376eca1e378b771989536b4b345dca2ff35ee3753b3b...   \n",
       "14  c0848aa36ff34140621161a2d83637d9efc4a98324e838...   \n",
       "15  6b81311fec7662e1a982a2bb412cd5c3cc00dc61f8988f...   \n",
       "16  74e66d5a2490de1609273ad10b9ae2580bc77fdf1cc6d6...   \n",
       "17  3407d15eaf3f2de08928ad95f078552310f63856d62af7...   \n",
       "18  d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f...   \n",
       "19  e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...   \n",
       "20  d5c6192b84923978f203069f9dbc7ecbf6f8abb895e543...   \n",
       "21  c70d653f0120cb4d6afd973ae2293ca16d2ba7015003aa...   \n",
       "22  94a433071544443ee89af9d2bb477505467d275c4fae8f...   \n",
       "23  d4f3f84956d95009e65e236660283b8618b35b6e7fff34...   \n",
       "24  73a957551be639c1054fa74f08b02dbe71eac81910a857...   \n",
       "25  1fa65e948e99538b45a1efe30f4d9f075fbb010f24fcd7...   \n",
       "26  18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...   \n",
       "27  4e07408562bedb8b60ce05c1decfe3ad16b72230967de0...   \n",
       "28  4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328c...   \n",
       "29  e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...   \n",
       "30  a9d5cf4f7c129b91df654ca180e0ede4a3859556522ba9...   \n",
       "31  5cf93c80c646593fc74913fe54d28530065425102eac7d...   \n",
       "32  153ce8fe205ee16af5261d8156f9d9247ad92a8c1b5e4d...   \n",
       "33  4eadf571d2362eb10c8f369e1cd0112df227d9b8a51237...   \n",
       "34  353c07e0a64fd4d962f4795671d1f90ca8688e6ee83d3a...   \n",
       "35  474e27900e1c69c9ee944e7d458505a29f1925edc06582...   \n",
       "36  3fe6a85383792e9554a38aebec0b1e6d292be422f4b8a5...   \n",
       "37  18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...   \n",
       "38  ef2d127de37b942baad06145e54b0c619a1f22327b2ebb...   \n",
       "\n",
       "                                             contents  \\\n",
       "0                                          Chapter 1.   \n",
       "1                                                   1   \n",
       "2   Recent news headlines are filled with reports ...   \n",
       "3   Businesses must make a serious effort to secur...   \n",
       "4   This chapter describes how you can secure and ...   \n",
       "5   /SM590000 Security fundamentals\\n/SM590000 Cur...   \n",
       "6                      1 http://www.idtheftcenter.org   \n",
       "7                          2 http://www.ponemon.org /   \n",
       "8    ' Copyright IBM Corp. 2014. All rights reserved.   \n",
       "9                                                   1   \n",
       "10  Before reviewing database security techniques,...   \n",
       "11  /SM590000 First, and most important, is the de...   \n",
       "12  A security policy is what defines whether the ...   \n",
       "13  /SM590000 The second fundamental in securing d...   \n",
       "14  With your eyes now open to the importance of s...   \n",
       "15  Because of the inherently secure nature of IBM...   \n",
       "16  Even more disturbing is that many IBM i client...   \n",
       "17  Traditionally, IBM i applications have employe...   \n",
       "18                                                  2   \n",
       "19  Row and Column Access Control Support in IBM D...   \n",
       "20  Some clients using this default configuration ...   \n",
       "21  Although exit-point solutions can provide grea...   \n",
       "22  An exit-point solution is a good option for da...   \n",
       "23  As described in 1.2, \"Current state of IBM i s...   \n",
       "24  As shown in Figure 1-1, it is an all-or-nothin...   \n",
       "25  Figure 1-1   All-or-nothing access to the rows...   \n",
       "26    Chapter 1. Securing and protecting IBM DB2 data   \n",
       "27                                                  3   \n",
       "28                                                  4   \n",
       "29  Row and Column Access Control Support in IBM D...   \n",
       "30  Many businesses are trying to limit data acces...   \n",
       "31  Some IBM i clients have tried augmenting the a...   \n",
       "32  Using SQL views to limit access to a subset of...   \n",
       "33  Even if you are willing to live with these per...   \n",
       "34      Figure 1-2   Existing row and column controls   \n",
       "35  Based on the challenges that are associated wi...   \n",
       "36  The new DB2 RCAC support provides a method for...   \n",
       "37    Chapter 1. Securing and protecting IBM DB2 data   \n",
       "38                                                  5   \n",
       "\n",
       "                                               digest  \n",
       "0   600fb9cb986bebc4344dbdaba34a2a212d2cd0d51ecb03...  \n",
       "1   6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...  \n",
       "2   10699f104b8aafb34b542e9c28b9c5b4f22f585fbf22e9...  \n",
       "3   292293706e6df4757a4b3ddba7fffa4173aedb161acb56...  \n",
       "4   a796bd66e5fc1dd55abb1cf92115cbef6086ea6ad0c21c...  \n",
       "5   e34c9152459fe1c55a1f6bde873463739daef0924a8588...  \n",
       "6   620c2496ae6d48ac61d4457921c56816c0605c1ad041de...  \n",
       "7   424462748fbd7f641237d60c2a7e4e68ab7f69e78dbf76...  \n",
       "8   33b240f4ce154d4a321f0826c92a20294ac1cb844e99c4...  \n",
       "9   6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d...  \n",
       "10  f6f5ac281bee0775c514256bafa10bbdc2feb43e5a37ef...  \n",
       "11  879e30e151e0dd5e14fe28388ea2ff11c14689c94bcefa...  \n",
       "12  fa0d8a17d198b8c8b37c1d58fbb3a80d7db91ec255aeeb...  \n",
       "13  10376eca1e378b771989536b4b345dca2ff35ee3753b3b...  \n",
       "14  c0848aa36ff34140621161a2d83637d9efc4a98324e838...  \n",
       "15  6b81311fec7662e1a982a2bb412cd5c3cc00dc61f8988f...  \n",
       "16  74e66d5a2490de1609273ad10b9ae2580bc77fdf1cc6d6...  \n",
       "17  3407d15eaf3f2de08928ad95f078552310f63856d62af7...  \n",
       "18  d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f...  \n",
       "19  e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...  \n",
       "20  d5c6192b84923978f203069f9dbc7ecbf6f8abb895e543...  \n",
       "21  c70d653f0120cb4d6afd973ae2293ca16d2ba7015003aa...  \n",
       "22  94a433071544443ee89af9d2bb477505467d275c4fae8f...  \n",
       "23  d4f3f84956d95009e65e236660283b8618b35b6e7fff34...  \n",
       "24  73a957551be639c1054fa74f08b02dbe71eac81910a857...  \n",
       "25  1fa65e948e99538b45a1efe30f4d9f075fbb010f24fcd7...  \n",
       "26  18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...  \n",
       "27  4e07408562bedb8b60ce05c1decfe3ad16b72230967de0...  \n",
       "28  4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328c...  \n",
       "29  e396bb604f27c079fdb9e80b54887c3bc2b3357f6e8e94...  \n",
       "30  a9d5cf4f7c129b91df654ca180e0ede4a3859556522ba9...  \n",
       "31  5cf93c80c646593fc74913fe54d28530065425102eac7d...  \n",
       "32  153ce8fe205ee16af5261d8156f9d9247ad92a8c1b5e4d...  \n",
       "33  4eadf571d2362eb10c8f369e1cd0112df227d9b8a51237...  \n",
       "34  353c07e0a64fd4d962f4795671d1f90ca8688e6ee83d3a...  \n",
       "35  474e27900e1c69c9ee944e7d458505a29f1925edc06582...  \n",
       "36  3fe6a85383792e9554a38aebec0b1e6d292be422f4b8a5...  \n",
       "37  18702b75dc2fad3663c09932cdc8d2d4644bb63e12e0f2...  \n",
       "38  ef2d127de37b942baad06145e54b0c619a1f22327b2ebb...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "pq.read_table('test-data/output/test1.parquet').to_pandas()[['document_id','contents','digest']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80030996-23f7-45e0-8c14-53f95bbddc3a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}