20_machine-learning_main-runs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine learning gazetteer matching: main runs\n",
    "Notebook to use to generate our main results. We train a model with different combinations of features, but on the same random subset of training and test data per run, for directly comparable runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import random\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from gazmatch import matchutils as mu\n",
    "from gazmatch import mlrunner as ml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "### PATHS ###\n",
    "data_dir = 'data'\n",
    "path_geonames_csv = os.path.join(data_dir, 'geonames_ch_swisscoords.csv')\n",
    "path_swissnames_csv = os.path.join(data_dir, 'swissnames_new_uuids.csv')\n",
    "# with original UUIDs\n",
    "path_ground_truth_list = os.path.join(data_dir, 'ground_truth_flat.pkl')\n",
    "# modified record-unique IDs\n",
    "path_ground_truth_dict_new_uuids = os.path.join(data_dir, 'ground_truth_new_uuids.pkl')\n",
    "path_ground_truth_list_new_uuids = os.path.join(data_dir, 'ground_truth_flat_new_uuids.pkl')\n",
    "# landcover data\n",
    "land_cover_dir = 'ArealStatistik\\\\arealstatistik_nolc_2004\\\\rawdata\\\\stand_20130918_endstand'\n",
    "land_cover_csv = 'AREA_NOLC04_27_130918.csv'\n",
    "\n",
    "# candidate selection: matches per train and test record\n",
    "M_train = 30\n",
    "M_test = 30\n",
    "# train to test split\n",
    "train_portion = .75\n",
    "# number of loops using these settings (e.g. 10)\n",
    "num_loops = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-time data preparation\n",
    "- Prepare the landcover data\n",
    "- Read in the two gazetteers\n",
    "- Read in ground truth data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4128498, 13)\n"
     ]
    }
   ],
   "source": [
    "# land cover data\n",
    "df_areal = pd.read_csv(os.path.join(data_dir, land_cover_dir, land_cover_csv), sep=';', encoding='utf-8', low_memory=False)\n",
    "print(df_areal.shape)\n",
    "# prepare for Nearest Neighbour searches (this takes a while!)\n",
    "neighbour_obj = mu.prepare_neighbours(df_areal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 313562 records in SwissNames3D in 21 columns.\n",
      "We have 67796 records in GeoNames in 21 columns.\n",
      "Our ground truth dict has 400 GeoNames records.\n"
     ]
    }
   ],
   "source": [
    "# load SwissNames3D augmented with lat, lons\n",
    "swissnames = pd.read_csv(path_swissnames_csv, sep='\\t', encoding='utf-8', low_memory=False)\n",
    "print(\"We have %s records in SwissNames3D in %s columns.\" %(swissnames.shape[0], swissnames.shape[1]))\n",
    "\n",
    "### OPTIONAL: prepare swissnames for spatial filtering to use a hard distance filter in candidate selection\n",
    "#neighbour_sn = mu.prepare_neighbours_swissnames(swissnames)\n",
    "\n",
    "# load GeoNames with all WGS84 lat, lons projected to swiss coordinates\n",
    "geonames = pd.read_csv(path_geonames_csv, sep='\\t', encoding='utf-8', low_memory=False)\n",
    "print(\"We have %s records in GeoNames in %s columns.\" %(geonames.shape[0], geonames.shape[1]))\n",
    "\n",
    "# ground truth (i.e. positive matches for each geonames record)\n",
    "ground_truth_dict = mu.load_data(path_ground_truth_dict_new_uuids)\n",
    "# sanity check - this should be 400\n",
    "print(\"Our ground truth dict has %s GeoNames records.\" %len(ground_truth_dict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Machine learning gazetteer matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to avoid too much repetition below\n",
    "def run_with_features(train, test, features_to_use, description, verbose=True):\n",
    "    X = train[features_to_use]\n",
    "    y = train['match_bin']\n",
    "    rfc = RandomForestClassifier(n_estimators=200)\n",
    "    rfc = rfc.fit(X, y)\n",
    "    predictions = rfc.predict(test[features_to_use])\n",
    "    # results\n",
    "    if verbose:\n",
    "        print(\"\\n# %s, results:\" %description)\n",
    "    pairs = mu.get_positives_as_pairs(test, predictions)\n",
    "    pairs_old = [(item[0], mu.new_uuid_to_old(item[1])) for item in pairs]\n",
    "    precision,recall,f1 = mu.evaluate_results_deep(pairs_old, ground_truth_filtered, verbose=True)\n",
    "    return precision,recall,f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "### Run number 1 ###\n",
      "We selected 300 geonames features to be in our training set.\n",
      "We have 100 geonames features left for our test set.\n",
      "Training: We have 371 positive matches and 12 records with no match\n",
      "Testing: We have 129 positive matches and 2 records with no match\n",
      "Processing record 0...\n",
      "Processing record 50...\n",
      "Processing record 100...\n",
      "Processing record 150...\n",
      "Processing record 200...\n",
      "Processing record 250...\n",
      "We had 330 positive and 8647 negative pairs retained in candidate selection.\n",
      "We kept 330 out of a possible 371 positive pairs, i.e. 0.889\n",
      "The actual ratio of neg to pos is 23.302\n",
      "Calculating point-to-point distance between matches...\n",
      "Calculating Levenshtein distance between names...\n",
      "Calculating Levenshtein distance between de-commaed names...\n",
      "Calculating min Levenshtein distance on alternate names...\n",
      "Getting the min Levenshtein distance overall...\n",
      "Calculating the normalized Levenshtein-Damerau distance...\n",
      "Calculating Jaro similarity...\n",
      "Calculating Jaro-Winkler similarity...\n",
      "Getting the absolute elevation distance...\n",
      "Getting the feature types from both gazetteers...\n",
      "Getting the dummy variables for feature types...\n",
      "Getting the landcover classes for source and target...\n",
      "Getting the dummy variables for landcover classes...\n",
      "Getting the 'mode' landcover classes for source and target...\n",
      "Getting the dummy variables for 'mode' landcover classes...\n",
      "Getting the landcover 'distance'...\n",
      "Feature computation took 78.653s\n",
      "Processing record 0...\n",
      "Processing record 50...\n",
      "We had 113 positive and 2879 negative pairs retained in candidate selection.\n",
      "We kept 113 out of a possible 129 positive pairs, i.e. 0.876 (upper bound on recall).\n",
      "Test set: the ratio of neg to pos is 25.478\n",
      "Calculating point-to-point distance between matches...\n",
      "Calculating Levenshtein distance between names...\n",
      "Calculating Levenshtein distance between de-commaed names...\n",
      "Calculating min Levenshtein distance on alternate names...\n",
      "Getting the min Levenshtein distance overall...\n",
      "Calculating the normalized Levenshtein-Damerau distance...\n",
      "Calculating Jaro similarity...\n",
      "Calculating Jaro-Winkler similarity...\n",
      "Getting the absolute elevation distance...\n",
      "Getting the feature types from both gazetteers...\n",
      "Getting the dummy variables for feature types...\n",
      "Getting the landcover classes for source and target...\n",
      "Getting the dummy variables for landcover classes...\n",
      "Getting the 'mode' landcover classes for source and target...\n",
      "Getting the dummy variables for 'mode' landcover classes...\n",
      "Getting the landcover 'distance'...\n",
      "Feature computation took 27.611s\n",
      "feature columns: ['distance', 'leven_min']\n",
      "\n",
      "# basic, results:\n",
      "  0.867 precision\n",
      "  0.845 recall\n",
      "  0.856 f1\n",
      "feature columns: ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler']\n",
      "\n",
      "# str, results:\n",
      "  0.932 precision\n",
      "  0.828 recall\n",
      "  0.877 f1\n",
      "feature columns: ['distance', 'leven_min', 'gn_GLCR', 'gn_HLL', 'gn_LK', 'gn_MT', 'gn_PASS', 'gn_PK', 'gn_STM', 'gn_VAL', 'sn_Alpiner Gipfel', 'sn_Campingplatzareal', 'sn_Deponieareal', 'sn_Felskopf', 'sn_Fliessgewaesser', 'sn_Flurname swisstopo', 'sn_Gebiet', 'sn_Gipfel', 'sn_Gletscher', 'sn_Graben', 'sn_Grat', 'sn_Haltestelle Bahn', 'sn_Haltestelle Bus', 'sn_Hauptgipfel', 'sn_Haupthuegel', 'sn_Haupttal', 'sn_Huegel', 'sn_Huegelzug', 'sn_Landschaftsname', 'sn_Lokalname swisstopo', 'sn_Massiv', 'sn_Ort', 'sn_Pass', 'sn_Personenfaehre ohne Seil', 'sn_Quartier', 'sn_Quartierteil', 'sn_See', 'sn_Seeteil', 'sn_Sesselbahn', 'sn_Standplatzareal', 'sn_Strasse', 'sn_Strassenpass', 'sn_Tal', 'sn_Turm', 'sn_Verzweigung', 'sn_Wehr', 'sn_Gondelbahn', 'sn_Haltestelle Schiff', 'sn_Skilift', 'sn_Uebrige Bahnen']\n",
      "\n",
      "# basic-type, results:\n",
      "  0.919 precision\n",
      "  0.879 recall\n",
      "  0.899 f1\n",
      "feature columns: ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler', 'gn_GLCR', 'gn_HLL', 'gn_LK', 'gn_MT', 'gn_PASS', 'gn_PK', 'gn_STM', 'gn_VAL', 'sn_Alpiner Gipfel', 'sn_Campingplatzareal', 'sn_Deponieareal', 'sn_Felskopf', 'sn_Fliessgewaesser', 'sn_Flurname swisstopo', 'sn_Gebiet', 'sn_Gipfel', 'sn_Gletscher', 'sn_Graben', 'sn_Grat', 'sn_Haltestelle Bahn', 'sn_Haltestelle Bus', 'sn_Hauptgipfel', 'sn_Haupthuegel', 'sn_Haupttal', 'sn_Huegel', 'sn_Huegelzug', 'sn_Landschaftsname', 'sn_Lokalname swisstopo', 'sn_Massiv', 'sn_Ort', 'sn_Pass', 'sn_Personenfaehre ohne Seil', 'sn_Quartier', 'sn_Quartierteil', 'sn_See', 'sn_Seeteil', 'sn_Sesselbahn', 'sn_Standplatzareal', 'sn_Strasse', 'sn_Strassenpass', 'sn_Tal', 'sn_Turm', 'sn_Verzweigung', 'sn_Wehr', 'sn_Gondelbahn', 'sn_Haltestelle Schiff', 'sn_Skilift', 'sn_Uebrige Bahnen']\n",
      "\n",
      "# str-type, results:\n",
      "  0.920 precision\n",
      "  0.888 recall\n",
      "  0.904 f1\n",
      "feature columns: ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler', 'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', 'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', 'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', 'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60', 'elev_dist', 'lc_distance']\n",
      "\n",
      "# str-elev-lc, results:\n",
      "  0.872 precision\n",
      "  0.879 recall\n",
      "  0.876 f1\n",
      "feature columns: ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler', 'gn_GLCR', 'gn_HLL', 'gn_LK', 'gn_MT', 'gn_PASS', 'gn_PK', 'gn_STM', 'gn_VAL', 'sn_Alpiner Gipfel', 'sn_Campingplatzareal', 'sn_Deponieareal', 'sn_Felskopf', 'sn_Fliessgewaesser', 'sn_Flurname swisstopo', 'sn_Gebiet', 'sn_Gipfel', 'sn_Gletscher', 'sn_Graben', 'sn_Grat', 'sn_Haltestelle Bahn', 'sn_Haltestelle Bus', 'sn_Hauptgipfel', 'sn_Haupthuegel', 'sn_Haupttal', 'sn_Huegel', 'sn_Huegelzug', 'sn_Landschaftsname', 'sn_Lokalname swisstopo', 'sn_Massiv', 'sn_Ort', 'sn_Pass', 'sn_Personenfaehre ohne Seil', 'sn_Quartier', 'sn_Quartierteil', 'sn_See', 'sn_Seeteil', 'sn_Sesselbahn', 'sn_Standplatzareal', 'sn_Strasse', 'sn_Strassenpass', 'sn_Tal', 'sn_Turm', 'sn_Verzweigung', 'sn_Wehr', 'lc_distance', 'sn_Gondelbahn', 'sn_Haltestelle Schiff', 'sn_Skilift', 'sn_Uebrige Bahnen']\n",
      "\n",
      "# str-type-lcd, results:\n",
      "  0.937 precision\n",
      "  0.897 recall\n",
      "  0.916 f1\n",
      "feature columns: ['distance', 'leven_min', 'elev_dist', 'gn_GLCR', 'gn_HLL', 'gn_LK', 'gn_MT', 'gn_PASS', 'gn_PK', 'gn_STM', 'gn_VAL', 'sn_Alpiner Gipfel', 'sn_Campingplatzareal', 'sn_Deponieareal', 'sn_Felskopf', 'sn_Fliessgewaesser', 'sn_Flurname swisstopo', 'sn_Gebiet', 'sn_Gipfel', 'sn_Gletscher', 'sn_Graben', 'sn_Grat', 'sn_Haltestelle Bahn', 'sn_Haltestelle Bus', 'sn_Hauptgipfel', 'sn_Haupthuegel', 'sn_Haupttal', 'sn_Huegel', 'sn_Huegelzug', 'sn_Landschaftsname', 'sn_Lokalname swisstopo', 'sn_Massiv', 'sn_Ort', 'sn_Pass', 'sn_Personenfaehre ohne Seil', 'sn_Quartier', 'sn_Quartierteil', 'sn_See', 'sn_Seeteil', 'sn_Sesselbahn', 'sn_Standplatzareal', 'sn_Strasse', 'sn_Strassenpass', 'sn_Tal', 'sn_Turm', 'sn_Verzweigung', 'sn_Wehr', 'lc_distance', 'sn_Gondelbahn', 'sn_Haltestelle Schiff', 'sn_Skilift', 'sn_Uebrige Bahnen']\n",
      "\n",
      "# all-min, results:\n",
      "  0.945 precision\n",
      "  0.888 recall\n",
      "  0.916 f1\n",
      "feature columns: ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler', 'elev_dist', 'gn_GLCR', 'gn_HLL', 'gn_LK', 'gn_MT', 'gn_PASS', 'gn_PK', 'gn_STM', 'gn_VAL', 'sn_Alpiner Gipfel', 'sn_Campingplatzareal', 'sn_Deponieareal', 'sn_Felskopf', 'sn_Fliessgewaesser', 'sn_Flurname swisstopo', 'sn_Gebiet', 'sn_Gipfel', 'sn_Gletscher', 'sn_Graben', 'sn_Grat', 'sn_Haltestelle Bahn', 'sn_Haltestelle Bus', 'sn_Hauptgipfel', 'sn_Haupthuegel', 'sn_Haupttal', 'sn_Huegel', 'sn_Huegelzug', 'sn_Landschaftsname', 'sn_Lokalname swisstopo', 'sn_Massiv', 'sn_Ort', 'sn_Pass', 'sn_Personenfaehre ohne Seil', 'sn_Quartier', 'sn_Quartierteil', 'sn_See', 'sn_Seeteil', 'sn_Sesselbahn', 'sn_Standplatzareal', 'sn_Strasse', 'sn_Strassenpass', 'sn_Tal', 'sn_Turm', 'sn_Verzweigung', 'sn_Wehr', 'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', 'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', 'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', 'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60', 'lc_distance', 'sn_Gondelbahn', 'sn_Haltestelle Schiff', 'sn_Skilift', 'sn_Uebrige Bahnen']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "# all, results:\n",
      "  0.912 precision\n",
      "  0.897 recall\n",
      "  0.904 f1\n",
      "\n",
      "### Run number 2 ###\n",
      "We selected 300 geonames features to be in our training set.\n",
      "We have 100 geonames features left for our test set.\n",
      "Training: We have 381 positive matches and 8 records with no match\n",
      "Testing: We have 119 positive matches and 6 records with no match\n",
      "Processing record 0...\n",
      "Processing record 50...\n",
      "Processing record 100...\n",
      "Processing record 150...\n",
      "Processing record 200...\n",
      "Processing record 250...\n",
      "We had 335 positive and 8636 negative pairs retained in candidate selection.\n",
      "We kept 335 out of a possible 381 positive pairs, i.e. 0.879\n",
      "The actual ratio of neg to pos is 22.661\n",
      "Calculating point-to-point distance between matches...\n",
      "Calculating Levenshtein distance between names...\n",
      "Calculating Levenshtein distance between de-commaed names...\n",
      "Calculating min Levenshtein distance on alternate names...\n",
      "Getting the min Levenshtein distance overall...\n",
      "Calculating the normalized Levenshtein-Damerau distance...\n",
      "Calculating Jaro similarity...\n",
      "Calculating Jaro-Winkler similarity...\n",
      "Getting the absolute elevation distance...\n",
      "Getting the feature types from both gazetteers...\n",
      "Getting the dummy variables for feature types...\n",
      "Getting the landcover classes for source and target...\n",
      "Getting the dummy variables for landcover classes...\n",
      "Getting the 'mode' landcover classes for source and target...\n",
      "Getting the dummy variables for 'mode' landcover classes...\n",
      "Getting the landcover 'distance'...\n",
      "Feature computation took 92.852s\n",
      "Processing record 0...\n",
      "Processing record 50...\n",
      "We had 108 positive and 2890 negative pairs retained in candidate selection.\n",
      "We kept 108 out of a possible 119 positive pairs, i.e. 0.908 (upper bound on recall).\n",
      "Test set: the ratio of neg to pos is 26.759\n",
      "Calculating point-to-point distance between matches...\n",
      "Calculating Levenshtein distance between names...\n",
      "Calculating Levenshtein distance between de-commaed names...\n",
      "Calculating min Levenshtein distance on alternate names...\n",
      "Getting the min Levenshtein distance overall...\n",
      "Calculating the normalized Levenshtein-Damerau distance...\n",
      "Calculating Jaro similarity...\n",
      "Calculating Jaro-Winkler similarity...\n",
      "Getting the absolute elevation distance...\n",
      "Getting the feature types from both gazetteers...\n",
      "Getting the dummy variables for feature types...\n",
      "Getting the landcover classes for source and target...\n",
      "Getting the dummy variables for landcover classes...\n",
      "Getting the 'mode' landcover classes for source and target...\n",
      "Getting the dummy variables for 'mode' landcover classes...\n",
      "Getting the landcover 'distance'...\n",
      "Feature computation took 29.838s\n",
      "\n",
      "# basic, results:\n",
      "  0.809 precision\n",
      "  0.781 recall\n",
      "  0.795 f1\n",
      "\n",
      "# str, results:\n",
      "  0.862 precision\n",
      "  0.825 recall\n",
      "  0.843 f1\n",
      "\n",
      "# basic-type, results:\n",
      "  0.951 precision\n",
      "  0.851 recall\n",
      "  0.898 f1\n",
      "\n",
      "# str-type, results:\n",
      "  0.917 precision\n",
      "  0.877 recall\n",
      "  0.897 f1\n",
      "\n",
      "# str-elev-lc, results:\n",
      "  0.874 precision\n",
      "  0.851 recall\n",
      "  0.862 f1\n",
      "\n",
      "# str-type-lcd, results:\n",
      "  0.935 precision\n",
      "  0.886 recall\n",
      "  0.910 f1\n",
      "\n",
      "# all-min, results:\n",
      "  0.950 precision\n",
      "  0.842 recall\n",
      "  0.893 f1\n",
      "\n",
      "# all, results:\n",
      "  0.926 precision\n",
      "  0.877 recall\n",
      "  0.901 f1\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "\n",
    "for i in range(num_loops):\n",
    "    print(\"\\n### Run number %s ###\" %(i+1))\n",
    "    \n",
    "    ### SPLIT GEONAMES IDS INTO TRAIN:TEST AND PREPARE POSITIVES ###\n",
    "    N_train = int(len(ground_truth_dict)*train_portion)\n",
    "    train_gn_records = random.sample(ground_truth_dict.keys(), N_train)\n",
    "    test_gn_records = [k for k in ground_truth_dict.keys() if k not in train_gn_records]\n",
    "    print(\"We selected %s geonames features to be in our training set.\" %len(train_gn_records))\n",
    "    print(\"We have %s geonames features left for our test set.\" %len(test_gn_records))\n",
    "    df_pos_train, df_pos_test = mu.prepare_positives(train_gn_records, test_gn_records, ground_truth_dict, verbose=True)\n",
    "    \n",
    "    ### TRAINING DATA PREPARATION ###\n",
    "    geonames_train = geonames[geonames['geonameid'].isin(train_gn_records)]\n",
    "    df_geonames_train = geonames_train.set_index('geonameid')\n",
    "    # candidate selection\n",
    "    candidate_selecter = ml.CandidateSelecter(gazetteer_source=df_geonames_train, \n",
    "                                              gazetteer_target=swissnames)\n",
    "    matches_train = candidate_selecter.get_candidates_geoleven(id_list=train_gn_records, \n",
    "                                                                         matches_per_record=M_train)\n",
    "    ### OPTIONAL: candidate selection with a hard spatial filter (max_distance), to speed things up\n",
    "    #candidate_selecter = ml.CandidateSelecter(gazetteer_source=df_geonames_train, \n",
    "                                              #gazetteer_target=swissnames, neigh_obj=neighbour_sn)\n",
    "    #matches_train = candidate_selecter.get_candidates_geoleven_prefilter(id_list=train_gn_records, matches_per_record=M_train, \n",
    "                                                                         #max_distance=50000, filter_by_type=True)\n",
    "    # turn matches dict into list of pos (if any) and neg matches\n",
    "    pos_train, neg_train = mu.split_pos_neg(matches_train, ground_truth_dict)\n",
    "    pos_train_uniques = list(set(pos_train))\n",
    "    print(\"We had %s positive and %s negative pairs retained in candidate selection.\" \n",
    "          %(len(pos_train_uniques), len(neg_train)))\n",
    "    print(\"We kept %s out of a possible %s positive pairs, i.e. %.3f\" \n",
    "          %(len(pos_train_uniques), df_pos_train.shape[0], len(pos_train_uniques)/df_pos_train.shape[0]))\n",
    "    # prepare negatives from the ones mined\n",
    "    df_neg_train_all = pd.DataFrame.from_records(neg_train, columns=['geonamesid', 'swissnamesid', 'match'])\n",
    "    df_neg_train = df_neg_train_all.drop_duplicates()\n",
    "    # take all positives from ground truth (not just ones mined) and combine with negs\n",
    "    df_train = df_pos_train.append(df_neg_train)\n",
    "    # actual ratio of neg to pos in final training set\n",
    "    vc = df_train['match'].value_counts()\n",
    "    print(\"The actual ratio of neg to pos is %.3f\" %(vc[mu.NO_MATCH_STRING]/vc[mu.MATCH_STRING]))\n",
    "    \n",
    "    ### MATCHING FEATURES FOR TRAINING DATA ###\n",
    "    swissnames_train = swissnames[swissnames['UUID'].isin(df_train['swissnamesid'].tolist())]\n",
    "    df_swissnames_train = swissnames_train.set_index(['UUID'])\n",
    "    # compute matching features\n",
    "    fc = ml.FeatureComputer(matches=df_train, gazetteer_source=df_geonames_train, \n",
    "                            gazetteer_target=df_swissnames_train, landcover_data=df_areal, neigh_obj=neighbour_obj)\n",
    "    fc.compute_features_all(verbose=True)\n",
    "    df_train_all = fc.df.copy()\n",
    "    df_train_all['match_bin'] = pd.factorize(df_train_all['match'])[0]\n",
    "    \n",
    "    ### TEST DATA PREPARATION ###\n",
    "    geonames_test = geonames[geonames['geonameid'].isin(test_gn_records)]\n",
    "    df_geonames_test = geonames_test.set_index('geonameid')\n",
    "    # candidate selection: again here other options available e.g. a hard spatial filter\n",
    "    candidate_selecter = ml.CandidateSelecter(gazetteer_source=df_geonames_test, gazetteer_target=swissnames)\n",
    "    matches_test = candidate_selecter.get_candidates_geoleven(id_list=test_gn_records, matches_per_record=M_test)\n",
    "    # turn matches dict into list of pos and neg matches\n",
    "    pos_test, neg_test = mu.split_pos_neg(matches_test, ground_truth_dict)\n",
    "    pos_test_uniques = list(set(pos_test))\n",
    "    print(\"We had %s positive and %s negative pairs retained in candidate selection.\" \n",
    "          %(len(pos_test_uniques), len(neg_test)))\n",
    "    print(\"We kept %s out of a possible %s positive pairs, i.e. %.3f (upper bound on recall).\" \n",
    "          %(len(pos_test_uniques), df_pos_test.shape[0], len(pos_test_uniques)/df_pos_test.shape[0]))\n",
    "    # recall upper bound proper, with original uuids\n",
    "    recall_upper_bound = mu.calculate_recall_upper_bound(pos_test_uniques,df_pos_test)\n",
    "    # prepare negatives as dataframe\n",
    "    df_neg_test_all = pd.DataFrame.from_records(neg_test, columns=['geonamesid', 'swissnamesid', 'match'])\n",
    "    df_neg_test = df_neg_test_all.drop_duplicates()\n",
    "    # prepare the test matches using all (pos+neg) matches obtained from candidate selection\n",
    "    df_test_pos = pd.DataFrame.from_records(pos_test_uniques, columns=['geonamesid', 'swissnamesid', 'match'])\n",
    "    df_test = df_test_pos.append(df_neg_test)\n",
    "    vc = df_test['match'].value_counts()\n",
    "    print(\"Test set: the ratio of neg to pos is %.3f\" %(vc[mu.NO_MATCH_STRING]/vc[mu.MATCH_STRING]))\n",
    "    \n",
    "    ### MATCHING FEATURES FOR TEST DATA ###\n",
    "    swissnames_test = swissnames[swissnames['UUID'].isin(df_test['swissnamesid'].tolist())]\n",
    "    df_swissnames_test = swissnames_test.set_index(['UUID'])\n",
    "    fc = ml.FeatureComputer(matches=df_test, gazetteer_source=df_geonames_test, \n",
    "                             gazetteer_target=df_swissnames_test, landcover_data=df_areal, neigh_obj=neighbour_obj)\n",
    "    fc.compute_features_all(verbose=True)\n",
    "    df_test_all = fc.df.copy()\n",
    "    df_test_all['match_bin'] = pd.factorize(df_test_all['match'])[0]\n",
    "    for colname in df_test_all.columns:\n",
    "        if colname not in df_train_all.columns:\n",
    "            df_train_all[colname] = 0\n",
    "    for colname in df_train_all.columns:\n",
    "        if colname not in df_test_all.columns:\n",
    "            df_test_all[colname] = 0\n",
    "    \n",
    "    ### EVALUATION PRELIMINARIES ###\n",
    "    # prepare and filter ground truth for new UUIDs...\n",
    "    ground_truth_pairs_new_uuids = mu.load_data(path_ground_truth_list_new_uuids)\n",
    "    ground_truth_new_uuids_filtered = mu.filter_ground_truth_for_test(ground_truth_pairs_new_uuids, test_gn_records)\n",
    "    # ...and old UUIDs for our results\n",
    "    ground_truth_pairs = mu.load_data(path_ground_truth_list)\n",
    "    ground_truth_filtered = mu.filter_ground_truth_for_test(ground_truth_pairs, test_gn_records)\n",
    "    \n",
    "    ### RANDOM FOREST ###\n",
    "    # 1) basic\n",
    "    desc = \"basic\"\n",
    "    feature_columns = ['distance', 'leven_min']\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 2) str\n",
    "    desc = \"str\"\n",
    "    feature_columns = ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler']    \n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 3) basic-type\n",
    "    desc = \"basic-type\"\n",
    "    cols_to_exclude = ['geonamesid', 'swissnamesid', 'match', 'match_bin', 'leven', 'leven_comma', 'leven_alt', \n",
    "                       'leven_dam_norm', 'jaro', 'jaro_winkler', 'elev_dist', \n",
    "                       'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', \n",
    "                       'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', \n",
    "                       'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', \n",
    "                       'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60', \n",
    "                       'lc_distance']\n",
    "    feature_columns = [colname for colname in list(df_train_all.columns) if colname not in cols_to_exclude]\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 4) str-type\n",
    "    desc = \"str-type\"\n",
    "    cols_to_exclude = ['geonamesid', 'swissnamesid', 'match', 'match_bin', \n",
    "                       'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', \n",
    "                       'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', \n",
    "                       'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', \n",
    "                       'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60', \n",
    "                       'elev_dist', 'lc_distance']\n",
    "    feature_columns = [colname for colname in list(df_train_all.columns) if colname not in cols_to_exclude]\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "      \n",
    "    # 5) str-elev-lc\n",
    "    desc = \"str-elev-lc\"\n",
    "    feature_columns = ['distance', 'leven', 'leven_comma', 'leven_alt', 'leven_min', 'leven_dam_norm', 'jaro', 'jaro_winkler',\n",
    "                       'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', \n",
    "                       'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', \n",
    "                       'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', \n",
    "                       'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60',\n",
    "                       'elev_dist', 'lc_distance']\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 6) str-type-lcd\n",
    "    desc = \"str-type-lcd\"\n",
    "    cols_to_exclude = ['geonamesid', 'swissnamesid', 'match', 'match_bin', \n",
    "                       'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', \n",
    "                       'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', \n",
    "                       'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', \n",
    "                       'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60',\n",
    "                       'elev_dist']\n",
    "    feature_columns = [colname for colname in list(df_train_all.columns) if colname not in cols_to_exclude]\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 7) all-min\n",
    "    desc = \"all-min\"\n",
    "    cols_to_exclude = ['geonamesid', 'swissnamesid', 'match', 'match_bin', 'leven', 'leven_comma', 'leven_alt', \n",
    "                       'leven_dam_norm', 'jaro', 'jaro_winkler',  \n",
    "                       'gn_10', 'gn_20', 'gn_30', 'gn_40', 'gn_50', 'gn_60', \n",
    "                       'sn_10', 'sn_20', 'sn_30', 'sn_40', 'sn_50', 'sn_60', \n",
    "                       'gn_mode_10', 'gn_mode_20', 'gn_mode_30', 'gn_mode_40', 'gn_mode_50', 'gn_mode_60', \n",
    "                       'sn_mode_10', 'sn_mode_20', 'sn_mode_30', 'sn_mode_40', 'sn_mode_50', 'sn_mode_60']\n",
    "    feature_columns = [colname for colname in list(df_train_all.columns) if colname not in cols_to_exclude]\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))\n",
    "    \n",
    "    # 8) all\n",
    "    desc = \"all\"\n",
    "    cols_to_exclude = ['geonamesid', 'swissnamesid', 'match', 'match_bin']\n",
    "    feature_columns = [colname for colname in list(df_train_all.columns) if colname not in cols_to_exclude]\n",
    "    if i == 0:\n",
    "        print(\"feature columns: %s\" %feature_columns)\n",
    "    p,r,f = run_with_features(df_train_all, df_test_all, feature_columns, desc)\n",
    "    results.append((p,r,f,recall_upper_bound))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision\trecall\t\tf1 \t\tmax_recall\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "0.8673   \t0.8448   \t0.8559   \t0.9397\n",
      "0.9320   \t0.8276   \t0.8767   \t0.9397\n",
      "0.9189   \t0.8793   \t0.8987   \t0.9397\n",
      "0.9196   \t0.8879   \t0.9035   \t0.9397\n",
      "0.8718   \t0.8793   \t0.8755   \t0.9397\n",
      "0.9369   \t0.8966   \t0.9163   \t0.9397\n",
      "0.9450   \t0.8879   \t0.9156   \t0.9397\n",
      "0.9123   \t0.8966   \t0.9043   \t0.9397\n",
      "0.8091   \t0.7807   \t0.7946   \t0.9211\n",
      "0.8624   \t0.8246   \t0.8430   \t0.9211\n",
      "0.9510   \t0.8509   \t0.8981   \t0.9211\n",
      "0.9174   \t0.8772   \t0.8969   \t0.9211\n",
      "0.8739   \t0.8509   \t0.8622   \t0.9211\n",
      "0.9352   \t0.8860   \t0.9099   \t0.9211\n",
      "0.9505   \t0.8421   \t0.8930   \t0.9211\n",
      "0.9259   \t0.8772   \t0.9009   \t0.9211\n"
     ]
    }
   ],
   "source": [
    "print(\"precision\\trecall\\t\\tf1 \\t\\tmax_recall\")\n",
    "print(\"- - - - - - - - - - - - - - - - - - - - - - - - - - \")\n",
    "for p,r,f,rec_ub in results:\n",
    "    print(\"%.4f   \\t%.4f   \\t%.4f   \\t%.4f\" %(p,r,f,rec_ub))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}