trail.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "05eb1761-8669-4f33-b13e-087896a6bda2",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_location = \"/FileStore/tables/data_final-1.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "2ff9395f-45fe-4533-92b8-9b52325105ad",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">root\n",
       "-- EXT1: string (nullable = true)\n",
       "-- EXT2: string (nullable = true)\n",
       "-- EXT3: string (nullable = true)\n",
       "-- EXT4: string (nullable = true)\n",
       "-- EXT5: string (nullable = true)\n",
       "-- EXT6: string (nullable = true)\n",
       "-- EXT7: string (nullable = true)\n",
       "-- EXT8: string (nullable = true)\n",
       "-- EXT9: string (nullable = true)\n",
       "-- EXT10: string (nullable = true)\n",
       "-- EST1: string (nullable = true)\n",
       "-- EST2: string (nullable = true)\n",
       "-- EST3: string (nullable = true)\n",
       "-- EST4: string (nullable = true)\n",
       "-- EST5: string (nullable = true)\n",
       "-- EST6: string (nullable = true)\n",
       "-- EST7: string (nullable = true)\n",
       "-- EST8: string (nullable = true)\n",
       "-- EST9: string (nullable = true)\n",
       "-- EST10: string (nullable = true)\n",
       "-- AGR1: string (nullable = true)\n",
       "-- AGR2: string (nullable = true)\n",
       "-- AGR3: string (nullable = true)\n",
       "-- AGR4: string (nullable = true)\n",
       "-- AGR5: string (nullable = true)\n",
       "-- AGR6: string (nullable = true)\n",
       "-- AGR7: string (nullable = true)\n",
       "-- AGR8: string (nullable = true)\n",
       "-- AGR9: string (nullable = true)\n",
       "-- AGR10: string (nullable = true)\n",
       "-- CSN1: string (nullable = true)\n",
       "-- CSN2: string (nullable = true)\n",
       "-- CSN3: string (nullable = true)\n",
       "-- CSN4: string (nullable = true)\n",
       "-- CSN5: string (nullable = true)\n",
       "-- CSN6: string (nullable = true)\n",
       "-- CSN7: string (nullable = true)\n",
       "-- CSN8: string (nullable = true)\n",
       "-- CSN9: string (nullable = true)\n",
       "-- CSN10: string (nullable = true)\n",
       "-- OPN1: string (nullable = true)\n",
       "-- OPN2: string (nullable = true)\n",
       "-- OPN3: string (nullable = true)\n",
       "-- OPN4: string (nullable = true)\n",
       "-- OPN5: string (nullable = true)\n",
       "-- OPN6: string (nullable = true)\n",
       "-- OPN7: string (nullable = true)\n",
       "-- OPN8: string (nullable = true)\n",
       "-- OPN9: string (nullable = true)\n",
       "-- OPN10: string (nullable = true)\n",
       "-- EXT1_E: string (nullable = true)\n",
       "-- EXT2_E: string (nullable = true)\n",
       "-- EXT3_E: string (nullable = true)\n",
       "-- EXT4_E: string (nullable = true)\n",
       "-- EXT5_E: string (nullable = true)\n",
       "-- EXT6_E: string (nullable = true)\n",
       "-- EXT7_E: string (nullable = true)\n",
       "-- EXT8_E: string (nullable = true)\n",
       "-- EXT9_E: string (nullable = true)\n",
       "-- EXT10_E: string (nullable = true)\n",
       "-- EST1_E: string (nullable = true)\n",
       "-- EST2_E: string (nullable = true)\n",
       "-- EST3_E: string (nullable = true)\n",
       "-- EST4_E: string (nullable = true)\n",
       "-- EST5_E: string (nullable = true)\n",
       "-- EST6_E: string (nullable = true)\n",
       "-- EST7_E: string (nullable = true)\n",
       "-- EST8_E: string (nullable = true)\n",
       "-- EST9_E: string (nullable = true)\n",
       "-- EST10_E: string (nullable = true)\n",
       "-- AGR1_E: string (nullable = true)\n",
       "-- AGR2_E: string (nullable = true)\n",
       "-- AGR3_E: string (nullable = true)\n",
       "-- AGR4_E: string (nullable = true)\n",
       "-- AGR5_E: string (nullable = true)\n",
       "-- AGR6_E: string (nullable = true)\n",
       "-- AGR7_E: string (nullable = true)\n",
       "-- AGR8_E: string (nullable = true)\n",
       "-- AGR9_E: string (nullable = true)\n",
       "-- AGR10_E: string (nullable = true)\n",
       "-- CSN1_E: string (nullable = true)\n",
       "-- CSN2_E: string (nullable = true)\n",
       "-- CSN3_E: string (nullable = true)\n",
       "-- CSN4_E: string (nullable = true)\n",
       "-- CSN5_E: string (nullable = true)\n",
       "-- CSN6_E: string (nullable = true)\n",
       "-- CSN7_E: string (nullable = true)\n",
       "-- CSN8_E: string (nullable = true)\n",
       "-- CSN9_E: string (nullable = true)\n",
       "-- CSN10_E: string (nullable = true)\n",
       "-- OPN1_E: string (nullable = true)\n",
       "-- OPN2_E: string (nullable = true)\n",
       "-- OPN3_E: string (nullable = true)\n",
       "-- OPN4_E: string (nullable = true)\n",
       "-- OPN5_E: string (nullable = true)\n",
       "-- OPN6_E: string (nullable = true)\n",
       "-- OPN7_E: string (nullable = true)\n",
       "-- OPN8_E: string (nullable = true)\n",
       "-- OPN9_E: string (nullable = true)\n",
       "-- OPN10_E: string (nullable = true)\n",
       "-- dateload: string (nullable = true)\n",
       "-- screenw: string (nullable = true)\n",
       "-- screenh: string (nullable = true)\n",
       "-- introelapse: string (nullable = true)\n",
       "-- testelapse: string (nullable = true)\n",
       "-- endelapse: integer (nullable = true)\n",
       "-- IPC: integer (nullable = true)\n",
       "-- country: string (nullable = true)\n",
       "-- lat_appx_lots_of_err: string (nullable = true)\n",
       "-- long_appx_lots_of_err: string (nullable = true)\n",
       "\n",
       "</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">root\n |-- EXT1: string (nullable = true)\n |-- EXT2: string (nullable = true)\n |-- EXT3: string (nullable = true)\n |-- EXT4: string (nullable = true)\n |-- EXT5: string (nullable = true)\n |-- EXT6: string (nullable = true)\n |-- EXT7: string (nullable = true)\n |-- EXT8: string (nullable = true)\n |-- EXT9: string (nullable = true)\n |-- EXT10: string (nullable = true)\n |-- EST1: string (nullable = true)\n |-- EST2: string (nullable = true)\n |-- EST3: string (nullable = true)\n |-- EST4: string (nullable = true)\n |-- EST5: string (nullable = true)\n |-- EST6: string (nullable = true)\n |-- EST7: string (nullable = true)\n |-- EST8: string (nullable = true)\n |-- EST9: string (nullable = true)\n |-- EST10: string (nullable = true)\n |-- AGR1: string (nullable = true)\n |-- AGR2: string (nullable = true)\n |-- AGR3: string (nullable = true)\n |-- AGR4: string (nullable = true)\n |-- AGR5: string (nullable = true)\n |-- AGR6: string (nullable = true)\n |-- AGR7: string (nullable = true)\n |-- AGR8: string (nullable = true)\n |-- AGR9: string (nullable = true)\n |-- AGR10: string (nullable = true)\n |-- CSN1: string (nullable = true)\n |-- CSN2: string (nullable = true)\n |-- CSN3: string (nullable = true)\n |-- CSN4: string (nullable = true)\n |-- CSN5: string (nullable = true)\n |-- CSN6: string (nullable = true)\n |-- CSN7: string (nullable = true)\n |-- CSN8: string (nullable = true)\n |-- CSN9: string (nullable = true)\n |-- CSN10: string (nullable = true)\n |-- OPN1: string (nullable = true)\n |-- OPN2: string (nullable = true)\n |-- OPN3: string (nullable = true)\n |-- OPN4: string (nullable = true)\n |-- OPN5: string (nullable = true)\n |-- OPN6: string (nullable = true)\n |-- OPN7: string (nullable = true)\n |-- OPN8: string (nullable = true)\n |-- OPN9: string (nullable = true)\n |-- OPN10: string (nullable = true)\n |-- EXT1_E: string (nullable = true)\n |-- EXT2_E: string (nullable = true)\n |-- EXT3_E: string (nullable = true)\n |-- EXT4_E: string (nullable = true)\n |-- EXT5_E: string (nullable = true)\n |-- EXT6_E: string (nullable = true)\n |-- EXT7_E: string (nullable = true)\n |-- EXT8_E: string (nullable = true)\n |-- EXT9_E: string (nullable = true)\n |-- EXT10_E: string (nullable = true)\n |-- EST1_E: string (nullable = true)\n |-- EST2_E: string (nullable = true)\n |-- EST3_E: string (nullable = true)\n |-- EST4_E: string (nullable = true)\n |-- EST5_E: string (nullable = true)\n |-- EST6_E: string (nullable = true)\n |-- EST7_E: string (nullable = true)\n |-- EST8_E: string (nullable = true)\n |-- EST9_E: string (nullable = true)\n |-- EST10_E: string (nullable = true)\n |-- AGR1_E: string (nullable = true)\n |-- AGR2_E: string (nullable = true)\n |-- AGR3_E: string (nullable = true)\n |-- AGR4_E: string (nullable = true)\n |-- AGR5_E: string (nullable = true)\n |-- AGR6_E: string (nullable = true)\n |-- AGR7_E: string (nullable = true)\n |-- AGR8_E: string (nullable = true)\n |-- AGR9_E: string (nullable = true)\n |-- AGR10_E: string (nullable = true)\n |-- CSN1_E: string (nullable = true)\n |-- CSN2_E: string (nullable = true)\n |-- CSN3_E: string (nullable = true)\n |-- CSN4_E: string (nullable = true)\n |-- CSN5_E: string (nullable = true)\n |-- CSN6_E: string (nullable = true)\n |-- CSN7_E: string (nullable = true)\n |-- CSN8_E: string (nullable = true)\n |-- CSN9_E: string (nullable = true)\n |-- CSN10_E: string (nullable = true)\n |-- OPN1_E: string (nullable = true)\n |-- OPN2_E: string (nullable = true)\n |-- OPN3_E: string (nullable = true)\n |-- OPN4_E: string (nullable = true)\n |-- OPN5_E: string (nullable = true)\n |-- OPN6_E: string (nullable = true)\n |-- OPN7_E: string (nullable = true)\n |-- OPN8_E: string (nullable = true)\n |-- OPN9_E: string (nullable = true)\n |-- OPN10_E: string (nullable = true)\n |-- dateload: string (nullable = true)\n |-- screenw: string (nullable = true)\n |-- screenh: string (nullable = true)\n |-- introelapse: string (nullable = true)\n |-- testelapse: string (nullable = true)\n |-- endelapse: integer (nullable = true)\n |-- IPC: integer (nullable = true)\n |-- country: string (nullable = true)\n |-- lat_appx_lots_of_err: string (nullable = true)\n |-- long_appx_lots_of_err: string (nullable = true)\n\n</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('ml-bank').getOrCreate()\n",
    "df = spark.read.csv(file_location, header = True, inferSchema = True,  sep=\"\\t\",)\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "ff021f33-743f-40b0-8258-41ba65c6b9d3",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "8317e28f-d66d-4b5f-b10c-389e206495d9",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "features = [\"EXT1\", \"EXT2\" ,\"EXT3\" ,\"EXT4\" ,\"EXT5\" ,\"EXT6\" ,\"EXT7\" ,\"EXT8\" ,\"EXT9\" ,\"EXT10\",\n",
    "            \"EST1\" ,\"EST2\" ,\"EST3\" ,\"EST4\" ,\"EST5\" ,\"EST6\" ,\"EST7\" ,\"EST8\" ,\"EST9\" ,\"EST10\",\n",
    "            \"AGR1\" ,\"AGR2\" ,\"AGR3\" ,\"AGR4\" ,\"AGR5\" ,\"AGR6\" ,\"AGR7\" ,\"AGR8\" ,\"AGR9\" ,\"AGR10\",\n",
    "            \"CSN1\" ,\"CSN2\" ,\"CSN3\" ,\"CSN4\" ,\"CSN5\" ,\"CSN6\" ,\"CSN7\" ,\"CSN8\" ,\"CSN9\" ,\"CSN10\",\n",
    "            \"OPN1\" ,\"OPN2\" ,\"OPN3\" ,\"OPN4\" ,\"OPN5\" ,\"OPN6\" ,\"OPN7\" ,\"OPN8\" ,\"OPN9\" ,\"OPN10\"\n",
    "           ]\n",
    "\n",
    "dropList = []\n",
    "\n",
    "for each_feature in df.columns:\n",
    "\n",
    "    if each_feature in features:\n",
    "            df = df.withColumn(each_feature, df[each_feature].cast(FloatType()))\n",
    "    elif each_feature != \"country\":\n",
    "            dropList.append(each_feature)\n",
    "\n",
    "df = df.drop(*dropList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "329a97ca-4618-45fa-af9d-fd3c164cbe83",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "features_1 = [\"EXT2\" ,\"EXT3\" ,\"EXT4\" ,\"EXT5\" ,\"EXT6\" ,\"EXT7\" ,\"EXT8\" ,\"EXT9\" ,\"EXT10\",\n",
    "            \"EST1\" ,\"EST2\" ,\"EST3\" ,\"EST4\" ,\"EST5\" ,\"EST6\" ,\"EST7\" ,\"EST8\" ,\"EST9\" ,\"EST10\",\n",
    "            \"AGR1\" ,\"AGR2\" ,\"AGR3\" ,\"AGR4\" ,\"AGR5\" ,\"AGR6\" ,\"AGR7\" ,\"AGR8\" ,\"AGR9\" ,\"AGR10\",\n",
    "            \"CSN1\" ,\"CSN2\" ,\"CSN3\" ,\"CSN4\" ,\"CSN5\" ,\"CSN6\" ,\"CSN7\" ,\"CSN8\" ,\"CSN9\" ,\"CSN10\",\n",
    "            \"OPN1\" ,\"OPN2\" ,\"OPN3\" ,\"OPN4\" ,\"OPN5\" ,\"OPN6\" ,\"OPN7\" ,\"OPN8\" ,\"OPN9\" ,\"OPN10\"\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "05257d49-69ab-4154-8de7-3a8687615036",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "vectorAssembler = VectorAssembler(inputCols=features_1, outputCol = 'features',handleInvalid = 'skip')\n",
    "new_df = vectorAssembler.transform(df)\n",
    "new_df = new_df.select(['features', 'EXT1'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "a4674f27-43df-4e31-85ee-540f5c1b5c3d",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n",
       "<span class=\"ansi-red-fg\">IllegalArgumentException</span>                  Traceback (most recent call last)\n",
       "<span class=\"ansi-green-fg\">&lt;command-4372818467197720&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n",
       "<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>new_df <span class=\"ansi-blue-fg\">=</span> vectorAssembler<span class=\"ansi-blue-fg\">.</span>setHandleInvalid<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;skip&#34;</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>new_df<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">      2</span> new_df <span class=\"ansi-blue-fg\">=</span> vectorAssembler<span class=\"ansi-blue-fg\">.</span>setHandleInvalid<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;keep&#34;</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>new_df<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>show\n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/ml/base.py</span> in <span class=\"ansi-cyan-fg\">transform</span><span class=\"ansi-blue-fg\">(self, dataset, params)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    215</span>                 <span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>copy<span class=\"ansi-blue-fg\">(</span>params<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>_transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    216</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-fg\">--&gt; 217</span><span class=\"ansi-red-fg\">                 </span><span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>_transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    218</span>         <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    219</span>             <span class=\"ansi-green-fg\">raise</span> ValueError<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;Params must be a param map but got %s.&#34;</span> <span class=\"ansi-blue-fg\">%</span> type<span class=\"ansi-blue-fg\">(</span>params<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span>\n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/ml/wrapper.py</span> in <span class=\"ansi-cyan-fg\">_transform</span><span class=\"ansi-blue-fg\">(self, dataset)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    348</span>     <span class=\"ansi-green-fg\">def</span> _transform<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    349</span>         self<span class=\"ansi-blue-fg\">.</span>_transfer_params_to_java<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-fg\">--&gt; 350</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> DataFrame<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">.</span>_java_obj<span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">.</span>_jdf<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">.</span>sql_ctx<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    351</span> \n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    352</span> \n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py</span> in <span class=\"ansi-cyan-fg\">__call__</span><span class=\"ansi-blue-fg\">(self, *args)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">   1302</span> \n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">   1303</span>         answer <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>gateway_client<span class=\"ansi-blue-fg\">.</span>send_command<span class=\"ansi-blue-fg\">(</span>command<span class=\"ansi-blue-fg\">)</span>\n",
       "<span class=\"ansi-green-fg\">-&gt; 1304</span><span class=\"ansi-red-fg\">         return_value = get_return_value(\n",
       "</span><span class=\"ansi-green-intense-fg ansi-bold\">   1305</span>             answer, self.gateway_client, self.target_id, self.name)\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">   1306</span> \n",
       "\n",
       "<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/utils.py</span> in <span class=\"ansi-cyan-fg\">deco</span><span class=\"ansi-blue-fg\">(*a, **kw)</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    114</span>                 <span class=\"ansi-red-fg\"># Hide where the exception came from that shows a non-Pythonic</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    115</span>                 <span class=\"ansi-red-fg\"># JVM exception message.</span>\n",
       "<span class=\"ansi-green-fg\">--&gt; 116</span><span class=\"ansi-red-fg\">                 </span><span class=\"ansi-green-fg\">raise</span> converted <span class=\"ansi-green-fg\">from</span> <span class=\"ansi-green-fg\">None</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    117</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n",
       "<span class=\"ansi-green-intense-fg ansi-bold\">    118</span>                 <span class=\"ansi-green-fg\">raise</span>\n",
       "\n",
       "<span class=\"ansi-red-fg\">IllegalArgumentException</span>: EXT2 does not exist. Available: features, EXT1</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "<div class=\"ansiout\"><span class=\"ansi-red-fg\">---------------------------------------------------------------------------</span>\n<span class=\"ansi-red-fg\">IllegalArgumentException</span>                  Traceback (most recent call last)\n<span class=\"ansi-green-fg\">&lt;command-4372818467197720&gt;</span> in <span class=\"ansi-cyan-fg\">&lt;module&gt;</span>\n<span class=\"ansi-green-fg\">----&gt; 1</span><span class=\"ansi-red-fg\"> </span>new_df <span class=\"ansi-blue-fg\">=</span> vectorAssembler<span class=\"ansi-blue-fg\">.</span>setHandleInvalid<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;skip&#34;</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>new_df<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">      2</span> new_df <span class=\"ansi-blue-fg\">=</span> vectorAssembler<span class=\"ansi-blue-fg\">.</span>setHandleInvalid<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;keep&#34;</span><span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>new_df<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>show\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/ml/base.py</span> in <span class=\"ansi-cyan-fg\">transform</span><span class=\"ansi-blue-fg\">(self, dataset, params)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    215</span>                 <span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>copy<span class=\"ansi-blue-fg\">(</span>params<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">.</span>_transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    216</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-fg\">--&gt; 217</span><span class=\"ansi-red-fg\">                 </span><span class=\"ansi-green-fg\">return</span> self<span class=\"ansi-blue-fg\">.</span>_transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    218</span>         <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    219</span>             <span class=\"ansi-green-fg\">raise</span> ValueError<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">&#34;Params must be a param map but got %s.&#34;</span> <span class=\"ansi-blue-fg\">%</span> type<span class=\"ansi-blue-fg\">(</span>params<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">)</span>\n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/ml/wrapper.py</span> in <span class=\"ansi-cyan-fg\">_transform</span><span class=\"ansi-blue-fg\">(self, dataset)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    348</span>     <span class=\"ansi-green-fg\">def</span> _transform<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    349</span>         self<span class=\"ansi-blue-fg\">.</span>_transfer_params_to_java<span class=\"ansi-blue-fg\">(</span><span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-fg\">--&gt; 350</span><span class=\"ansi-red-fg\">         </span><span class=\"ansi-green-fg\">return</span> DataFrame<span class=\"ansi-blue-fg\">(</span>self<span class=\"ansi-blue-fg\">.</span>_java_obj<span class=\"ansi-blue-fg\">.</span>transform<span class=\"ansi-blue-fg\">(</span>dataset<span class=\"ansi-blue-fg\">.</span>_jdf<span class=\"ansi-blue-fg\">)</span><span class=\"ansi-blue-fg\">,</span> dataset<span class=\"ansi-blue-fg\">.</span>sql_ctx<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    351</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">    352</span> \n\n<span class=\"ansi-green-fg\">/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py</span> in <span class=\"ansi-cyan-fg\">__call__</span><span class=\"ansi-blue-fg\">(self, *args)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">   1302</span> \n<span class=\"ansi-green-intense-fg ansi-bold\">   1303</span>         answer <span class=\"ansi-blue-fg\">=</span> self<span class=\"ansi-blue-fg\">.</span>gateway_client<span class=\"ansi-blue-fg\">.</span>send_command<span class=\"ansi-blue-fg\">(</span>command<span class=\"ansi-blue-fg\">)</span>\n<span class=\"ansi-green-fg\">-&gt; 1304</span><span class=\"ansi-red-fg\">         return_value = get_return_value(\n</span><span class=\"ansi-green-intense-fg ansi-bold\">   1305</span>             answer, self.gateway_client, self.target_id, self.name)\n<span class=\"ansi-green-intense-fg ansi-bold\">   1306</span> \n\n<span class=\"ansi-green-fg\">/databricks/spark/python/pyspark/sql/utils.py</span> in <span class=\"ansi-cyan-fg\">deco</span><span class=\"ansi-blue-fg\">(*a, **kw)</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    114</span>                 <span class=\"ansi-red-fg\"># Hide where the exception came from that shows a non-Pythonic</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    115</span>                 <span class=\"ansi-red-fg\"># JVM exception message.</span>\n<span class=\"ansi-green-fg\">--&gt; 116</span><span class=\"ansi-red-fg\">                 </span><span class=\"ansi-green-fg\">raise</span> converted <span class=\"ansi-green-fg\">from</span> <span class=\"ansi-green-fg\">None</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    117</span>             <span class=\"ansi-green-fg\">else</span><span class=\"ansi-blue-fg\">:</span>\n<span class=\"ansi-green-intense-fg ansi-bold\">    118</span>                 <span class=\"ansi-green-fg\">raise</span>\n\n<span class=\"ansi-red-fg\">IllegalArgumentException</span>: EXT2 does not exist. Available: features, EXT1</div>",
       "errorSummary": "<span class=\"ansi-red-fg\">IllegalArgumentException</span>: EXT2 does not exist. Available: features, EXT1",
       "metadata": {},
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# new_df = vectorAssembler.setHandleInvalid(\"skip\").transform(new_df)\n",
    "# new_df = vectorAssembler.setHandleInvalid(\"keep\").transform(new_df).show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "78994f40-7fb5-4c6c-a79d-04946c5fa45f",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[37]: DataFrame[features: vector, EXT1: float]</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[37]: DataFrame[features: vector, EXT1: float]</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "new_df.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "3c52a0bc-e7a5-4a44-b1a7-01382cee7ff8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\"></div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train, test = new_df.randomSplit([0.75, 0.25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "14b3d12f-5579-469f-82ce-5ec947319100",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Out[18]: 758988</div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "arguments": {},
       "data": "<div class=\"ansiout\">Out[18]: 758988</div>",
       "datasetInfos": [],
       "metadata": {},
       "removedWidgets": [],
       "type": "html"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "inputWidgets": {},
     "nuid": "46fc6b91-ace1-4290-8ec2-e3517251541b",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 2
   },
   "notebookName": "trail",
   "notebookOrigID": 4372818467197707,
   "widgets": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}