data-manipulation-with-spark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:32.039255Z",
     "iopub.status.busy": "2021-07-25T02:34:32.038759Z",
     "iopub.status.idle": "2021-07-25T02:34:39.426053Z",
     "shell.execute_reply": "2021-07-25T02:34:39.425054Z",
     "shell.execute_reply.started": "2021-07-25T02:34:32.039151Z"
    }
   },
   "outputs": [],
   "source": [
    "#pip install pyspark"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start a spark session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:39.428198Z",
     "iopub.status.busy": "2021-07-25T02:34:39.427883Z",
     "iopub.status.idle": "2021-07-25T02:34:45.306783Z",
     "shell.execute_reply": "2021-07-25T02:34:45.305445Z",
     "shell.execute_reply.started": "2021-07-25T02:34:39.428165Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "22/05/25 15:49:57 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "            <div>\n",
       "                <p><b>SparkSession - in-memory</b></p>\n",
       "                \n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://asthas-air.lan:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.2.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>ReadWriteVal</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        \n",
       "            </div>\n",
       "        "
      ],
      "text/plain": [
       "<pyspark.sql.session.SparkSession at 0x7ff601ba9e20>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pyspark \n",
    "from pyspark.sql import SparkSession\n",
    "\n",
    "spark = SparkSession.builder.appName(\"ReadWriteVal\").getOrCreate()\n",
    "\n",
    "spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:45.312301Z",
     "iopub.status.busy": "2021-07-25T02:34:45.311883Z",
     "iopub.status.idle": "2021-07-25T02:34:45.349193Z",
     "shell.execute_reply": "2021-07-25T02:34:45.347963Z",
     "shell.execute_reply.started": "2021-07-25T02:34:45.312255Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cores = spark._jsc.sc().getExecutorMemoryStatus().keySet().size()\n",
    "cores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dataset source: https://www.kaggle.com/satpreetmakhija/netflix-movies-and-tv-shows-2021?select=netflixData.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing netflix data in the form of csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:45.352712Z",
     "iopub.status.busy": "2021-07-25T02:34:45.351879Z",
     "iopub.status.idle": "2021-07-25T02:34:51.703860Z",
     "shell.execute_reply": "2021-07-25T02:34:51.702661Z",
     "shell.execute_reply.started": "2021-07-25T02:34:45.352657Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Director</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Cast</th>\n",
       "      <th>Production Country</th>\n",
       "      <th>Release Date</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Imdb Score</th>\n",
       "      <th>Content Type</th>\n",
       "      <th>Date Added</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cc1b6ed9-cf9e-4057-8303-34577fb54477</td>\n",
       "      <td>(Un)Well</td>\n",
       "      <td>This docuseries takes a deep dive into the luc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Reality TV</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e2ef4e91-fb25-42ab-b485-be8e3b23dedb</td>\n",
       "      <td>#Alive</td>\n",
       "      <td>As a grisly virus rampages a city, a lone man ...</td>\n",
       "      <td>Cho Il</td>\n",
       "      <td>Horror Movies, International Movies, Thrillers</td>\n",
       "      <td>Yoo Ah-in, Park Shin-hye</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>99 min</td>\n",
       "      <td>6.2/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b01b73b7-81f6-47a7-86d8-acb63080d525</td>\n",
       "      <td>#AnneFrank - Parallel Stories</td>\n",
       "      <td>Through her diary, Anne Frank's story is retol...</td>\n",
       "      <td>Sabina Fedeli, Anna Migotto</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>Helen Mirren, Gengher Gatti</td>\n",
       "      <td>Italy</td>\n",
       "      <td>2019</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>95 min</td>\n",
       "      <td>6.4/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>July 1, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6611af0-f53c-4a08-9ffa-9716dc57eb9c</td>\n",
       "      <td>#blackAF</td>\n",
       "      <td>Kenya Barris and his family navigate relations...</td>\n",
       "      <td>None</td>\n",
       "      <td>TV Comedies</td>\n",
       "      <td>Kenya Barris, Rashida Jones, Iman Benson, Genn...</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7f2d4170-bab8-4d75-adc2-197f7124c070</td>\n",
       "      <td>#cats_the_mewvie</td>\n",
       "      <td>This pawesome documentary explores how our fel...</td>\n",
       "      <td>Michael Margolis</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>Canada</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>90 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>February 5, 2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Show Id                          Title  \\\n",
       "0  cc1b6ed9-cf9e-4057-8303-34577fb54477                       (Un)Well   \n",
       "1  e2ef4e91-fb25-42ab-b485-be8e3b23dedb                         #Alive   \n",
       "2  b01b73b7-81f6-47a7-86d8-acb63080d525  #AnneFrank - Parallel Stories   \n",
       "3  b6611af0-f53c-4a08-9ffa-9716dc57eb9c                       #blackAF   \n",
       "4  7f2d4170-bab8-4d75-adc2-197f7124c070               #cats_the_mewvie   \n",
       "\n",
       "                                         Description  \\\n",
       "0  This docuseries takes a deep dive into the luc...   \n",
       "1  As a grisly virus rampages a city, a lone man ...   \n",
       "2  Through her diary, Anne Frank's story is retol...   \n",
       "3  Kenya Barris and his family navigate relations...   \n",
       "4  This pawesome documentary explores how our fel...   \n",
       "\n",
       "                      Director  \\\n",
       "0                         None   \n",
       "1                       Cho Il   \n",
       "2  Sabina Fedeli, Anna Migotto   \n",
       "3                         None   \n",
       "4             Michael Margolis   \n",
       "\n",
       "                                           Genres  \\\n",
       "0                                      Reality TV   \n",
       "1  Horror Movies, International Movies, Thrillers   \n",
       "2             Documentaries, International Movies   \n",
       "3                                     TV Comedies   \n",
       "4             Documentaries, International Movies   \n",
       "\n",
       "                                                Cast Production Country  \\\n",
       "0                                               None      United States   \n",
       "1                           Yoo Ah-in, Park Shin-hye        South Korea   \n",
       "2                        Helen Mirren, Gengher Gatti              Italy   \n",
       "3  Kenya Barris, Rashida Jones, Iman Benson, Genn...      United States   \n",
       "4                                               None             Canada   \n",
       "\n",
       "  Release Date Rating  Duration Imdb Score Content Type         Date Added  \n",
       "0         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "1         2020  TV-MA    99 min     6.2/10        Movie  September 8, 2020  \n",
       "2         2019  TV-14    95 min     6.4/10        Movie       July 1, 2020  \n",
       "3         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "4         2020  TV-14    90 min     5.1/10        Movie   February 5, 2020  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'netflixData.csv'\n",
    "# We ask spark to infer schema and header types\n",
    "netflix_rawdata = spark.read.csv(path,inferSchema=True,header=True)\n",
    "\n",
    "# Print top 5 rows of data\n",
    "netflix_rawdata.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Writing csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:51.706804Z",
     "iopub.status.busy": "2021-07-25T02:34:51.705843Z",
     "iopub.status.idle": "2021-07-25T02:34:52.847861Z",
     "shell.execute_reply": "2021-07-25T02:34:52.846686Z",
     "shell.execute_reply.started": "2021-07-25T02:34:51.706752Z"
    }
   },
   "outputs": [],
   "source": [
    "# Here I'm just going to create a copy of the original datasource and write it to another csv\n",
    "\n",
    "netflix_rawdata.toPandas().to_csv('netflix_copy.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:52.850030Z",
     "iopub.status.busy": "2021-07-25T02:34:52.849577Z",
     "iopub.status.idle": "2021-07-25T02:34:53.675808Z",
     "shell.execute_reply": "2021-07-25T02:34:53.674956Z",
     "shell.execute_reply.started": "2021-07-25T02:34:52.849984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>_c0</th>\n",
       "      <th>Show Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Director</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Cast</th>\n",
       "      <th>Production Country</th>\n",
       "      <th>Release Date</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Imdb Score</th>\n",
       "      <th>Content Type</th>\n",
       "      <th>Date Added</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>cc1b6ed9-cf9e-4057-8303-34577fb54477</td>\n",
       "      <td>(Un)Well</td>\n",
       "      <td>This docuseries takes a deep dive into the luc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Reality TV</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>e2ef4e91-fb25-42ab-b485-be8e3b23dedb</td>\n",
       "      <td>#Alive</td>\n",
       "      <td>As a grisly virus rampages a city, a lone man ...</td>\n",
       "      <td>Cho Il</td>\n",
       "      <td>Horror Movies, International Movies, Thrillers</td>\n",
       "      <td>Yoo Ah-in, Park Shin-hye</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>99 min</td>\n",
       "      <td>6.2/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>b01b73b7-81f6-47a7-86d8-acb63080d525</td>\n",
       "      <td>#AnneFrank - Parallel Stories</td>\n",
       "      <td>Through her diary, Anne Frank's story is retol...</td>\n",
       "      <td>Sabina Fedeli, Anna Migotto</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>Helen Mirren, Gengher Gatti</td>\n",
       "      <td>Italy</td>\n",
       "      <td>2019</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>95 min</td>\n",
       "      <td>6.4/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>July 1, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>b6611af0-f53c-4a08-9ffa-9716dc57eb9c</td>\n",
       "      <td>#blackAF</td>\n",
       "      <td>Kenya Barris and his family navigate relations...</td>\n",
       "      <td>None</td>\n",
       "      <td>TV Comedies</td>\n",
       "      <td>Kenya Barris, Rashida Jones, Iman Benson, Genn...</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>7f2d4170-bab8-4d75-adc2-197f7124c070</td>\n",
       "      <td>#cats_the_mewvie</td>\n",
       "      <td>This pawesome documentary explores how our fel...</td>\n",
       "      <td>Michael Margolis</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>Canada</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>90 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>February 5, 2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   _c0                               Show Id                          Title  \\\n",
       "0    0  cc1b6ed9-cf9e-4057-8303-34577fb54477                       (Un)Well   \n",
       "1    1  e2ef4e91-fb25-42ab-b485-be8e3b23dedb                         #Alive   \n",
       "2    2  b01b73b7-81f6-47a7-86d8-acb63080d525  #AnneFrank - Parallel Stories   \n",
       "3    3  b6611af0-f53c-4a08-9ffa-9716dc57eb9c                       #blackAF   \n",
       "4    4  7f2d4170-bab8-4d75-adc2-197f7124c070               #cats_the_mewvie   \n",
       "\n",
       "                                         Description  \\\n",
       "0  This docuseries takes a deep dive into the luc...   \n",
       "1  As a grisly virus rampages a city, a lone man ...   \n",
       "2  Through her diary, Anne Frank's story is retol...   \n",
       "3  Kenya Barris and his family navigate relations...   \n",
       "4  This pawesome documentary explores how our fel...   \n",
       "\n",
       "                      Director  \\\n",
       "0                         None   \n",
       "1                       Cho Il   \n",
       "2  Sabina Fedeli, Anna Migotto   \n",
       "3                         None   \n",
       "4             Michael Margolis   \n",
       "\n",
       "                                           Genres  \\\n",
       "0                                      Reality TV   \n",
       "1  Horror Movies, International Movies, Thrillers   \n",
       "2             Documentaries, International Movies   \n",
       "3                                     TV Comedies   \n",
       "4             Documentaries, International Movies   \n",
       "\n",
       "                                                Cast Production Country  \\\n",
       "0                                               None      United States   \n",
       "1                           Yoo Ah-in, Park Shin-hye        South Korea   \n",
       "2                        Helen Mirren, Gengher Gatti              Italy   \n",
       "3  Kenya Barris, Rashida Jones, Iman Benson, Genn...      United States   \n",
       "4                                               None             Canada   \n",
       "\n",
       "  Release Date Rating  Duration Imdb Score Content Type         Date Added  \n",
       "0         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "1         2020  TV-MA    99 min     6.2/10        Movie  September 8, 2020  \n",
       "2         2019  TV-14    95 min     6.4/10        Movie       July 1, 2020  \n",
       "3         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "4         2020  TV-14    90 min     5.1/10        Movie   February 5, 2020  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = './netflix_copy.csv'\n",
    "# We ask spark to infer schema and header types\n",
    "netflix_copy_rawdata = spark.read.csv(path,inferSchema=True,header=True)\n",
    "\n",
    "# Print top 5 rows of data\n",
    "netflix_copy_rawdata.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see above that the copy was created and saved in output. It could be read as the regular data file was.\n",
    "\n",
    "We can also save/import data in parquet format. Parquet is more efficient than a csv file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Writing parquet file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:53.682642Z",
     "iopub.status.busy": "2021-07-25T02:34:53.680066Z",
     "iopub.status.idle": "2021-07-25T02:34:53.689081Z",
     "shell.execute_reply": "2021-07-25T02:34:53.688233Z",
     "shell.execute_reply.started": "2021-07-25T02:34:53.682589Z"
    }
   },
   "outputs": [],
   "source": [
    "#netflix_rawdata.write.parquet('netflix_parquet/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above code would give us an error because parquet file cannot have spaces in headers. We will need to rename the columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:53.693709Z",
     "iopub.status.busy": "2021-07-25T02:34:53.693281Z",
     "iopub.status.idle": "2021-07-25T02:34:54.061941Z",
     "shell.execute_reply": "2021-07-25T02:34:54.060648Z",
     "shell.execute_reply.started": "2021-07-25T02:34:53.693669Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show_Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Director</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Cast</th>\n",
       "      <th>Production_Country</th>\n",
       "      <th>Release_Date</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Imdb_Score</th>\n",
       "      <th>Content_Type</th>\n",
       "      <th>Date_Added</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cc1b6ed9-cf9e-4057-8303-34577fb54477</td>\n",
       "      <td>(Un)Well</td>\n",
       "      <td>This docuseries takes a deep dive into the luc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Reality TV</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e2ef4e91-fb25-42ab-b485-be8e3b23dedb</td>\n",
       "      <td>#Alive</td>\n",
       "      <td>As a grisly virus rampages a city, a lone man ...</td>\n",
       "      <td>Cho Il</td>\n",
       "      <td>Horror Movies, International Movies, Thrillers</td>\n",
       "      <td>Yoo Ah-in, Park Shin-hye</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>99 min</td>\n",
       "      <td>6.2/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Show_Id     Title  \\\n",
       "0  cc1b6ed9-cf9e-4057-8303-34577fb54477  (Un)Well   \n",
       "1  e2ef4e91-fb25-42ab-b485-be8e3b23dedb    #Alive   \n",
       "\n",
       "                                         Description Director  \\\n",
       "0  This docuseries takes a deep dive into the luc...     None   \n",
       "1  As a grisly virus rampages a city, a lone man ...   Cho Il   \n",
       "\n",
       "                                           Genres                      Cast  \\\n",
       "0                                      Reality TV                      None   \n",
       "1  Horror Movies, International Movies, Thrillers  Yoo Ah-in, Park Shin-hye   \n",
       "\n",
       "  Production_Country Release_Date Rating  Duration Imdb_Score Content_Type  \\\n",
       "0      United States         2020  TV-MA  1 Season     6.6/10      TV Show   \n",
       "1        South Korea         2020  TV-MA    99 min     6.2/10        Movie   \n",
       "\n",
       "          Date_Added  \n",
       "0               None  \n",
       "1  September 8, 2020  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "netflix_rawdata_for_parquet = netflix_rawdata.withColumnRenamed(\"Show Id\",\"Show_Id\") \\\n",
    "                                .withColumnRenamed(\"Production Country\",\"Production_Country\") \\\n",
    "                                .withColumnRenamed(\"Release Date\",\"Release_Date\") \\\n",
    "                                .withColumnRenamed(\"Imdb Score\",\"Imdb_Score\") \\\n",
    "                                .withColumnRenamed(\"Content Type\",\"Content_Type\") \\\n",
    "                                .withColumnRenamed(\"Date Added\",\"Date_Added\")\n",
    "netflix_rawdata_for_parquet.limit(2).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the the columns are renames and spaces are removed, we can write the data to a parquet file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:54.066018Z",
     "iopub.status.busy": "2021-07-25T02:34:54.065461Z",
     "iopub.status.idle": "2021-07-25T02:34:55.468322Z",
     "shell.execute_reply": "2021-07-25T02:34:55.466794Z",
     "shell.execute_reply.started": "2021-07-25T02:34:54.065962Z"
    }
   },
   "outputs": [],
   "source": [
    "netflix_rawdata_for_parquet.write.mode(\"overwrite\").parquet('netflix_parquet/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also save parquet files by partitioning them on a variable. For example, lets partition this netflix data by release date for top 10 rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:55.471160Z",
     "iopub.status.busy": "2021-07-25T02:34:55.470680Z",
     "iopub.status.idle": "2021-07-25T02:34:56.093832Z",
     "shell.execute_reply": "2021-07-25T02:34:56.092711Z",
     "shell.execute_reply.started": "2021-07-25T02:34:55.471102Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show_Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Director</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Cast</th>\n",
       "      <th>Production_Country</th>\n",
       "      <th>Release_Date</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Imdb_Score</th>\n",
       "      <th>Content_Type</th>\n",
       "      <th>Date_Added</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cc1b6ed9-cf9e-4057-8303-34577fb54477</td>\n",
       "      <td>(Un)Well</td>\n",
       "      <td>This docuseries takes a deep dive into the luc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Reality TV</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e2ef4e91-fb25-42ab-b485-be8e3b23dedb</td>\n",
       "      <td>#Alive</td>\n",
       "      <td>As a grisly virus rampages a city, a lone man ...</td>\n",
       "      <td>Cho Il</td>\n",
       "      <td>Horror Movies, International Movies, Thrillers</td>\n",
       "      <td>Yoo Ah-in, Park Shin-hye</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>99 min</td>\n",
       "      <td>6.2/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b01b73b7-81f6-47a7-86d8-acb63080d525</td>\n",
       "      <td>#AnneFrank - Parallel Stories</td>\n",
       "      <td>Through her diary, Anne Frank's story is retol...</td>\n",
       "      <td>Sabina Fedeli, Anna Migotto</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>Helen Mirren, Gengher Gatti</td>\n",
       "      <td>Italy</td>\n",
       "      <td>2019</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>95 min</td>\n",
       "      <td>6.4/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>July 1, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b6611af0-f53c-4a08-9ffa-9716dc57eb9c</td>\n",
       "      <td>#blackAF</td>\n",
       "      <td>Kenya Barris and his family navigate relations...</td>\n",
       "      <td>None</td>\n",
       "      <td>TV Comedies</td>\n",
       "      <td>Kenya Barris, Rashida Jones, Iman Benson, Genn...</td>\n",
       "      <td>United States</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7f2d4170-bab8-4d75-adc2-197f7124c070</td>\n",
       "      <td>#cats_the_mewvie</td>\n",
       "      <td>This pawesome documentary explores how our fel...</td>\n",
       "      <td>Michael Margolis</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>Canada</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>90 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>February 5, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>c293788a-41f7-49a3-a7fc-005ea33bce2b</td>\n",
       "      <td>#FriendButMarried</td>\n",
       "      <td>Pining for his high school crush for years, a ...</td>\n",
       "      <td>Rako Prijanto</td>\n",
       "      <td>Dramas, International Movies, Romantic Movies</td>\n",
       "      <td>Adipati Dolken, Vanesha Prescilla, Rendi Jhon,...</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>2018</td>\n",
       "      <td>TV-G</td>\n",
       "      <td>102 min</td>\n",
       "      <td>7.0/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>May 21, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0555e67e-f624-4a05-93e4-55c117d0056d</td>\n",
       "      <td>#FriendButMarried 2</td>\n",
       "      <td>As Ayu and Ditto finally transition from best ...</td>\n",
       "      <td>Rako Prijanto</td>\n",
       "      <td>Dramas, International Movies, Romantic Movies</td>\n",
       "      <td>Adipati Dolken, Mawar de Jongh, Sari Nila, Von...</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>2020</td>\n",
       "      <td>TV-G</td>\n",
       "      <td>104 min</td>\n",
       "      <td>7.0/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>June 28, 2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>c844460f-6178-4f87-929e-80816c74ca35</td>\n",
       "      <td>#realityhigh</td>\n",
       "      <td>When nerdy high schooler Dani finally attracts...</td>\n",
       "      <td>Fernando Lebrija</td>\n",
       "      <td>Comedies</td>\n",
       "      <td>Nesta Cooper, Kate Walsh, John Michael Higgins...</td>\n",
       "      <td>United States</td>\n",
       "      <td>2017</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>99 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8b34e0e9-7258-4e49-b799-2e7eddbd7e34</td>\n",
       "      <td>#Rucker50</td>\n",
       "      <td>This documentary celebrates the 50th anniversa...</td>\n",
       "      <td>Robert McCullough Jr.</td>\n",
       "      <td>Documentaries, Sports Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>2016</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>56 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>December 1, 2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>6da2fc83-1546-4e9d-bf2e-9b472a059c18</td>\n",
       "      <td>#Selfie</td>\n",
       "      <td>Two days before their final exams, three teen ...</td>\n",
       "      <td>Cristina Jacob</td>\n",
       "      <td>Comedies, Dramas, International Movies</td>\n",
       "      <td>Flavia Hojda, Crina Semciuc, Olimpia Melinte, ...</td>\n",
       "      <td>Romania</td>\n",
       "      <td>2014</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>125 min</td>\n",
       "      <td>5.8/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>June 21, 2021</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Show_Id                          Title  \\\n",
       "0  cc1b6ed9-cf9e-4057-8303-34577fb54477                       (Un)Well   \n",
       "1  e2ef4e91-fb25-42ab-b485-be8e3b23dedb                         #Alive   \n",
       "2  b01b73b7-81f6-47a7-86d8-acb63080d525  #AnneFrank - Parallel Stories   \n",
       "3  b6611af0-f53c-4a08-9ffa-9716dc57eb9c                       #blackAF   \n",
       "4  7f2d4170-bab8-4d75-adc2-197f7124c070               #cats_the_mewvie   \n",
       "5  c293788a-41f7-49a3-a7fc-005ea33bce2b              #FriendButMarried   \n",
       "6  0555e67e-f624-4a05-93e4-55c117d0056d            #FriendButMarried 2   \n",
       "7  c844460f-6178-4f87-929e-80816c74ca35                   #realityhigh   \n",
       "8  8b34e0e9-7258-4e49-b799-2e7eddbd7e34                      #Rucker50   \n",
       "9  6da2fc83-1546-4e9d-bf2e-9b472a059c18                        #Selfie   \n",
       "\n",
       "                                         Description  \\\n",
       "0  This docuseries takes a deep dive into the luc...   \n",
       "1  As a grisly virus rampages a city, a lone man ...   \n",
       "2  Through her diary, Anne Frank's story is retol...   \n",
       "3  Kenya Barris and his family navigate relations...   \n",
       "4  This pawesome documentary explores how our fel...   \n",
       "5  Pining for his high school crush for years, a ...   \n",
       "6  As Ayu and Ditto finally transition from best ...   \n",
       "7  When nerdy high schooler Dani finally attracts...   \n",
       "8  This documentary celebrates the 50th anniversa...   \n",
       "9  Two days before their final exams, three teen ...   \n",
       "\n",
       "                      Director  \\\n",
       "0                         None   \n",
       "1                       Cho Il   \n",
       "2  Sabina Fedeli, Anna Migotto   \n",
       "3                         None   \n",
       "4             Michael Margolis   \n",
       "5                Rako Prijanto   \n",
       "6                Rako Prijanto   \n",
       "7             Fernando Lebrija   \n",
       "8        Robert McCullough Jr.   \n",
       "9               Cristina Jacob   \n",
       "\n",
       "                                           Genres  \\\n",
       "0                                      Reality TV   \n",
       "1  Horror Movies, International Movies, Thrillers   \n",
       "2             Documentaries, International Movies   \n",
       "3                                     TV Comedies   \n",
       "4             Documentaries, International Movies   \n",
       "5   Dramas, International Movies, Romantic Movies   \n",
       "6   Dramas, International Movies, Romantic Movies   \n",
       "7                                        Comedies   \n",
       "8                    Documentaries, Sports Movies   \n",
       "9          Comedies, Dramas, International Movies   \n",
       "\n",
       "                                                Cast Production_Country  \\\n",
       "0                                               None      United States   \n",
       "1                           Yoo Ah-in, Park Shin-hye        South Korea   \n",
       "2                        Helen Mirren, Gengher Gatti              Italy   \n",
       "3  Kenya Barris, Rashida Jones, Iman Benson, Genn...      United States   \n",
       "4                                               None             Canada   \n",
       "5  Adipati Dolken, Vanesha Prescilla, Rendi Jhon,...          Indonesia   \n",
       "6  Adipati Dolken, Mawar de Jongh, Sari Nila, Von...          Indonesia   \n",
       "7  Nesta Cooper, Kate Walsh, John Michael Higgins...      United States   \n",
       "8                                               None      United States   \n",
       "9  Flavia Hojda, Crina Semciuc, Olimpia Melinte, ...            Romania   \n",
       "\n",
       "  Release_Date Rating  Duration Imdb_Score Content_Type         Date_Added  \n",
       "0         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "1         2020  TV-MA    99 min     6.2/10        Movie  September 8, 2020  \n",
       "2         2019  TV-14    95 min     6.4/10        Movie       July 1, 2020  \n",
       "3         2020  TV-MA  1 Season     6.6/10      TV Show               None  \n",
       "4         2020  TV-14    90 min     5.1/10        Movie   February 5, 2020  \n",
       "5         2018   TV-G   102 min     7.0/10        Movie       May 21, 2020  \n",
       "6         2020   TV-G   104 min     7.0/10        Movie      June 28, 2020  \n",
       "7         2017  TV-14    99 min     5.1/10        Movie  September 8, 2017  \n",
       "8         2016  TV-PG    56 min     5.1/10        Movie   December 1, 2016  \n",
       "9         2014  TV-MA   125 min     5.8/10        Movie      June 21, 2021  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "netflix_rawdata_for_partition_parquet = netflix_rawdata_for_parquet.filter(\"Release_Date is not NULL\").limit(10)\n",
    "netflix_rawdata_for_partition_parquet.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:56.096027Z",
     "iopub.status.busy": "2021-07-25T02:34:56.095418Z",
     "iopub.status.idle": "2021-07-25T02:34:57.254082Z",
     "shell.execute_reply": "2021-07-25T02:34:57.250065Z",
     "shell.execute_reply.started": "2021-07-25T02:34:56.095982Z"
    }
   },
   "outputs": [],
   "source": [
    "netflix_rawdata_for_partition_parquet.write.mode(\"overwrite\").partitionBy(\"Release_Date\").parquet('part_parquet/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading parquet file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:57.255805Z",
     "iopub.status.busy": "2021-07-25T02:34:57.255418Z",
     "iopub.status.idle": "2021-07-25T02:34:57.925260Z",
     "shell.execute_reply": "2021-07-25T02:34:57.924377Z",
     "shell.execute_reply.started": "2021-07-25T02:34:57.255762Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show_Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Director</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Cast</th>\n",
       "      <th>Production_Country</th>\n",
       "      <th>Rating</th>\n",
       "      <th>Duration</th>\n",
       "      <th>Imdb_Score</th>\n",
       "      <th>Content_Type</th>\n",
       "      <th>Date_Added</th>\n",
       "      <th>Release_Date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c844460f-6178-4f87-929e-80816c74ca35</td>\n",
       "      <td>#realityhigh</td>\n",
       "      <td>When nerdy high schooler Dani finally attracts...</td>\n",
       "      <td>Fernando Lebrija</td>\n",
       "      <td>Comedies</td>\n",
       "      <td>Nesta Cooper, Kate Walsh, John Michael Higgins...</td>\n",
       "      <td>United States</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>99 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2017</td>\n",
       "      <td>2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c293788a-41f7-49a3-a7fc-005ea33bce2b</td>\n",
       "      <td>#FriendButMarried</td>\n",
       "      <td>Pining for his high school crush for years, a ...</td>\n",
       "      <td>Rako Prijanto</td>\n",
       "      <td>Dramas, International Movies, Romantic Movies</td>\n",
       "      <td>Adipati Dolken, Vanesha Prescilla, Rendi Jhon,...</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>TV-G</td>\n",
       "      <td>102 min</td>\n",
       "      <td>7.0/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>May 21, 2020</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6da2fc83-1546-4e9d-bf2e-9b472a059c18</td>\n",
       "      <td>#Selfie</td>\n",
       "      <td>Two days before their final exams, three teen ...</td>\n",
       "      <td>Cristina Jacob</td>\n",
       "      <td>Comedies, Dramas, International Movies</td>\n",
       "      <td>Flavia Hojda, Crina Semciuc, Olimpia Melinte, ...</td>\n",
       "      <td>Romania</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>125 min</td>\n",
       "      <td>5.8/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>June 21, 2021</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b01b73b7-81f6-47a7-86d8-acb63080d525</td>\n",
       "      <td>#AnneFrank - Parallel Stories</td>\n",
       "      <td>Through her diary, Anne Frank's story is retol...</td>\n",
       "      <td>Sabina Fedeli, Anna Migotto</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>Helen Mirren, Gengher Gatti</td>\n",
       "      <td>Italy</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>95 min</td>\n",
       "      <td>6.4/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>July 1, 2020</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cc1b6ed9-cf9e-4057-8303-34577fb54477</td>\n",
       "      <td>(Un)Well</td>\n",
       "      <td>This docuseries takes a deep dive into the luc...</td>\n",
       "      <td>None</td>\n",
       "      <td>Reality TV</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e2ef4e91-fb25-42ab-b485-be8e3b23dedb</td>\n",
       "      <td>#Alive</td>\n",
       "      <td>As a grisly virus rampages a city, a lone man ...</td>\n",
       "      <td>Cho Il</td>\n",
       "      <td>Horror Movies, International Movies, Thrillers</td>\n",
       "      <td>Yoo Ah-in, Park Shin-hye</td>\n",
       "      <td>South Korea</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>99 min</td>\n",
       "      <td>6.2/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>September 8, 2020</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>b6611af0-f53c-4a08-9ffa-9716dc57eb9c</td>\n",
       "      <td>#blackAF</td>\n",
       "      <td>Kenya Barris and his family navigate relations...</td>\n",
       "      <td>None</td>\n",
       "      <td>TV Comedies</td>\n",
       "      <td>Kenya Barris, Rashida Jones, Iman Benson, Genn...</td>\n",
       "      <td>United States</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>1 Season</td>\n",
       "      <td>6.6/10</td>\n",
       "      <td>TV Show</td>\n",
       "      <td>None</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7f2d4170-bab8-4d75-adc2-197f7124c070</td>\n",
       "      <td>#cats_the_mewvie</td>\n",
       "      <td>This pawesome documentary explores how our fel...</td>\n",
       "      <td>Michael Margolis</td>\n",
       "      <td>Documentaries, International Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>Canada</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>90 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>February 5, 2020</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0555e67e-f624-4a05-93e4-55c117d0056d</td>\n",
       "      <td>#FriendButMarried 2</td>\n",
       "      <td>As Ayu and Ditto finally transition from best ...</td>\n",
       "      <td>Rako Prijanto</td>\n",
       "      <td>Dramas, International Movies, Romantic Movies</td>\n",
       "      <td>Adipati Dolken, Mawar de Jongh, Sari Nila, Von...</td>\n",
       "      <td>Indonesia</td>\n",
       "      <td>TV-G</td>\n",
       "      <td>104 min</td>\n",
       "      <td>7.0/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>June 28, 2020</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8b34e0e9-7258-4e49-b799-2e7eddbd7e34</td>\n",
       "      <td>#Rucker50</td>\n",
       "      <td>This documentary celebrates the 50th anniversa...</td>\n",
       "      <td>Robert McCullough Jr.</td>\n",
       "      <td>Documentaries, Sports Movies</td>\n",
       "      <td>None</td>\n",
       "      <td>United States</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>56 min</td>\n",
       "      <td>5.1/10</td>\n",
       "      <td>Movie</td>\n",
       "      <td>December 1, 2016</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                Show_Id                          Title  \\\n",
       "0  c844460f-6178-4f87-929e-80816c74ca35                   #realityhigh   \n",
       "1  c293788a-41f7-49a3-a7fc-005ea33bce2b              #FriendButMarried   \n",
       "2  6da2fc83-1546-4e9d-bf2e-9b472a059c18                        #Selfie   \n",
       "3  b01b73b7-81f6-47a7-86d8-acb63080d525  #AnneFrank - Parallel Stories   \n",
       "4  cc1b6ed9-cf9e-4057-8303-34577fb54477                       (Un)Well   \n",
       "5  e2ef4e91-fb25-42ab-b485-be8e3b23dedb                         #Alive   \n",
       "6  b6611af0-f53c-4a08-9ffa-9716dc57eb9c                       #blackAF   \n",
       "7  7f2d4170-bab8-4d75-adc2-197f7124c070               #cats_the_mewvie   \n",
       "8  0555e67e-f624-4a05-93e4-55c117d0056d            #FriendButMarried 2   \n",
       "9  8b34e0e9-7258-4e49-b799-2e7eddbd7e34                      #Rucker50   \n",
       "\n",
       "                                         Description  \\\n",
       "0  When nerdy high schooler Dani finally attracts...   \n",
       "1  Pining for his high school crush for years, a ...   \n",
       "2  Two days before their final exams, three teen ...   \n",
       "3  Through her diary, Anne Frank's story is retol...   \n",
       "4  This docuseries takes a deep dive into the luc...   \n",
       "5  As a grisly virus rampages a city, a lone man ...   \n",
       "6  Kenya Barris and his family navigate relations...   \n",
       "7  This pawesome documentary explores how our fel...   \n",
       "8  As Ayu and Ditto finally transition from best ...   \n",
       "9  This documentary celebrates the 50th anniversa...   \n",
       "\n",
       "                      Director  \\\n",
       "0             Fernando Lebrija   \n",
       "1                Rako Prijanto   \n",
       "2               Cristina Jacob   \n",
       "3  Sabina Fedeli, Anna Migotto   \n",
       "4                         None   \n",
       "5                       Cho Il   \n",
       "6                         None   \n",
       "7             Michael Margolis   \n",
       "8                Rako Prijanto   \n",
       "9        Robert McCullough Jr.   \n",
       "\n",
       "                                           Genres  \\\n",
       "0                                        Comedies   \n",
       "1   Dramas, International Movies, Romantic Movies   \n",
       "2          Comedies, Dramas, International Movies   \n",
       "3             Documentaries, International Movies   \n",
       "4                                      Reality TV   \n",
       "5  Horror Movies, International Movies, Thrillers   \n",
       "6                                     TV Comedies   \n",
       "7             Documentaries, International Movies   \n",
       "8   Dramas, International Movies, Romantic Movies   \n",
       "9                    Documentaries, Sports Movies   \n",
       "\n",
       "                                                Cast Production_Country  \\\n",
       "0  Nesta Cooper, Kate Walsh, John Michael Higgins...      United States   \n",
       "1  Adipati Dolken, Vanesha Prescilla, Rendi Jhon,...          Indonesia   \n",
       "2  Flavia Hojda, Crina Semciuc, Olimpia Melinte, ...            Romania   \n",
       "3                        Helen Mirren, Gengher Gatti              Italy   \n",
       "4                                               None      United States   \n",
       "5                           Yoo Ah-in, Park Shin-hye        South Korea   \n",
       "6  Kenya Barris, Rashida Jones, Iman Benson, Genn...      United States   \n",
       "7                                               None             Canada   \n",
       "8  Adipati Dolken, Mawar de Jongh, Sari Nila, Von...          Indonesia   \n",
       "9                                               None      United States   \n",
       "\n",
       "  Rating  Duration Imdb_Score Content_Type         Date_Added  Release_Date  \n",
       "0  TV-14    99 min     5.1/10        Movie  September 8, 2017          2017  \n",
       "1   TV-G   102 min     7.0/10        Movie       May 21, 2020          2018  \n",
       "2  TV-MA   125 min     5.8/10        Movie      June 21, 2021          2014  \n",
       "3  TV-14    95 min     6.4/10        Movie       July 1, 2020          2019  \n",
       "4  TV-MA  1 Season     6.6/10      TV Show               None          2020  \n",
       "5  TV-MA    99 min     6.2/10        Movie  September 8, 2020          2020  \n",
       "6  TV-MA  1 Season     6.6/10      TV Show               None          2020  \n",
       "7  TV-14    90 min     5.1/10        Movie   February 5, 2020          2020  \n",
       "8   TV-G   104 min     7.0/10        Movie      June 28, 2020          2020  \n",
       "9  TV-PG    56 min     5.1/10        Movie   December 1, 2016          2016  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"./part_parquet\"\n",
    "partitioned = spark.read.parquet(path)\n",
    "partitioned.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get type of dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:57.927415Z",
     "iopub.status.busy": "2021-07-25T02:34:57.926785Z",
     "iopub.status.idle": "2021-07-25T02:34:57.933340Z",
     "shell.execute_reply": "2021-07-25T02:34:57.932302Z",
     "shell.execute_reply.started": "2021-07-25T02:34:57.927364Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pyspark.sql.dataframe.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "print(type(partitioned))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get type of each column in dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:57.935456Z",
     "iopub.status.busy": "2021-07-25T02:34:57.934842Z",
     "iopub.status.idle": "2021-07-25T02:34:59.866513Z",
     "shell.execute_reply": "2021-07-25T02:34:59.865147Z",
     "shell.execute_reply.started": "2021-07-25T02:34:57.935409Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Show_Id: string (nullable = true)\n",
      " |-- Title: string (nullable = true)\n",
      " |-- Description: string (nullable = true)\n",
      " |-- Director: string (nullable = true)\n",
      " |-- Genres: string (nullable = true)\n",
      " |-- Cast: string (nullable = true)\n",
      " |-- Production_Country: string (nullable = true)\n",
      " |-- Rating: string (nullable = true)\n",
      " |-- Duration: string (nullable = true)\n",
      " |-- Imdb_Score: string (nullable = true)\n",
      " |-- Content_Type: string (nullable = true)\n",
      " |-- Date_Added: string (nullable = true)\n",
      " |-- Release_Date: integer (nullable = true)\n",
      "\n",
      "None\n",
      "DataFrame[summary: string, Show_Id: string, Title: string, Description: string, Director: string, Genres: string, Cast: string, Production_Country: string, Rating: string, Duration: string, Imdb_Score: string, Content_Type: string, Date_Added: string, Release_Date: string]\n"
     ]
    }
   ],
   "source": [
    "print(partitioned.printSchema())\n",
    "#------\n",
    "#--OR--\n",
    "#------\n",
    "print(partitioned.describe()) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get list of columns in dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:59.868284Z",
     "iopub.status.busy": "2021-07-25T02:34:59.867897Z",
     "iopub.status.idle": "2021-07-25T02:34:59.875629Z",
     "shell.execute_reply": "2021-07-25T02:34:59.874673Z",
     "shell.execute_reply.started": "2021-07-25T02:34:59.868244Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Show_Id', 'Title', 'Description', 'Director', 'Genres', 'Cast', 'Production_Country', 'Rating', 'Duration', 'Imdb_Score', 'Content_Type', 'Date_Added', 'Release_Date']\n"
     ]
    }
   ],
   "source": [
    "print(partitioned.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get type of just 1 column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:59.878457Z",
     "iopub.status.busy": "2021-07-25T02:34:59.877465Z",
     "iopub.status.idle": "2021-07-25T02:34:59.891469Z",
     "shell.execute_reply": "2021-07-25T02:34:59.890339Z",
     "shell.execute_reply.started": "2021-07-25T02:34:59.878408Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StringType"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partitioned.schema['Director'].dataType"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get summary statistics of 1 feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:34:59.894165Z",
     "iopub.status.busy": "2021-07-25T02:34:59.893404Z",
     "iopub.status.idle": "2021-07-25T02:35:00.399063Z",
     "shell.execute_reply": "2021-07-25T02:35:00.397764Z",
     "shell.execute_reply.started": "2021-07-25T02:34:59.894117Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+\n",
      "|summary|Imdb_Score|\n",
      "+-------+----------+\n",
      "|  count|        10|\n",
      "|   mean|      null|\n",
      "| stddev|      null|\n",
      "|    min|    5.1/10|\n",
      "|    max|    7.0/10|\n",
      "+-------+----------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "partitioned.describe(['Imdb_Score']).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get summary of multiple features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:35:00.400906Z",
     "iopub.status.busy": "2021-07-25T02:35:00.400488Z",
     "iopub.status.idle": "2021-07-25T02:35:00.951709Z",
     "shell.execute_reply": "2021-07-25T02:35:00.950399Z",
     "shell.execute_reply.started": "2021-07-25T02:35:00.400858Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+----------+--------------------+\n",
      "|summary|Imdb_Score|            Director|\n",
      "+-------+----------+--------------------+\n",
      "|  count|        10|                   8|\n",
      "|    min|    5.1/10|              Cho Il|\n",
      "|    25%|      null|                null|\n",
      "|    75%|      null|                null|\n",
      "|    max|    7.0/10|Sabina Fedeli, An...|\n",
      "+-------+----------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "partitioned.select(\"Imdb_Score\", \"Director\").summary(\"count\", \"min\", \"25%\", \"75%\", \"max\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In the above 2 lines, \n",
    "* notice that count of Director is 8 but the dataframe has 10 rows. This is because 2 of the movies have null values in Director.\n",
    "* Also notice these data types we got:\n",
    "    - Release_Date: string (should be numeric as its just year)\n",
    "    - Rating: string (should be numeric)\n",
    "    - Duration: string (should be numeric)\n",
    "    - Imdb_Score: string (should be numeric)\n",
    "    - Date_Added: string (should be date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-07-25T02:35:00.953878Z",
     "iopub.status.busy": "2021-07-25T02:35:00.953379Z",
     "iopub.status.idle": "2021-07-25T02:35:01.058513Z",
     "shell.execute_reply": "2021-07-25T02:35:01.057502Z",
     "shell.execute_reply.started": "2021-07-25T02:35:00.953826Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[Show_Id: string, Title: string, Description: string, Director: string, Genres: string, Cast: string, Production_Country: string, Rating: int, Duration: int, Imdb_Score: int, Content_Type: string, Date_Added: date, Release_Date: int]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyspark.sql.functions import * \n",
    "from pyspark.sql.types import *\n",
    "partitioned_formatted = partitioned.withColumn(\"Release_Date\", partitioned[\"Release_Date\"].cast(IntegerType())) \\\n",
    "            .withColumn(\"Rating\", partitioned[\"Rating\"].cast(IntegerType())) \\\n",
    "            .withColumn(\"Duration\", partitioned[\"Duration\"].cast(IntegerType())) \\\n",
    "            .withColumn(\"Imdb_Score\", partitioned[\"Imdb_Score\"].cast(IntegerType())) \\\n",
    "            .withColumn(\"Date_Added\", to_date(partitioned.Date_Added, 'MMMMM d, yyyy')) \n",
    "partitioned_formatted"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that the data types has been converted to the correct formats"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}