data_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vgf3011/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:144: FutureWarning:\n",
      "\n",
      "The sklearn.neighbors.unsupervised module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import re, numpy as np, os, sys, pandas\n",
    "from pymatgen import Composition\n",
    "import data_utils\n",
    "from pymatgen import Composition\n",
    "from matminer.featurizers.base import MultipleFeaturizer\n",
    "from matminer.featurizers import composition as cf\n",
    "from matminer.utils.conversions import str_to_composition\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "oqmd_data_path = 'sample_train_set.csv'\n",
    "oqmd_data = pandas.read_csv(oqmd_data_path)\n",
    "#oqmd_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>H</th>\n",
       "      <th>Li</th>\n",
       "      <th>Be</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>N</th>\n",
       "      <th>O</th>\n",
       "      <th>F</th>\n",
       "      <th>Na</th>\n",
       "      <th>Mg</th>\n",
       "      <th>...</th>\n",
       "      <th>Tl</th>\n",
       "      <th>Pb</th>\n",
       "      <th>Bi</th>\n",
       "      <th>Ac</th>\n",
       "      <th>Th</th>\n",
       "      <th>Pa</th>\n",
       "      <th>U</th>\n",
       "      <th>Np</th>\n",
       "      <th>Pu</th>\n",
       "      <th>formation_energy_peratom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.00000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.0</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.0</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.014502</td>\n",
       "      <td>0.003289</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>0.004009</td>\n",
       "      <td>0.008340</td>\n",
       "      <td>0.020356</td>\n",
       "      <td>0.113170</td>\n",
       "      <td>0.010391</td>\n",
       "      <td>0.002525</td>\n",
       "      <td>0.005897</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006633</td>\n",
       "      <td>0.00630</td>\n",
       "      <td>0.024078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004398</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003827</td>\n",
       "      <td>0.000443</td>\n",
       "      <td>0.000739</td>\n",
       "      <td>-0.973250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.073815</td>\n",
       "      <td>0.035862</td>\n",
       "      <td>0.023544</td>\n",
       "      <td>0.030741</td>\n",
       "      <td>0.049105</td>\n",
       "      <td>0.086824</td>\n",
       "      <td>0.213342</td>\n",
       "      <td>0.078004</td>\n",
       "      <td>0.027131</td>\n",
       "      <td>0.041537</td>\n",
       "      <td>...</td>\n",
       "      <td>0.048883</td>\n",
       "      <td>0.04414</td>\n",
       "      <td>0.086991</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.034864</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.031177</td>\n",
       "      <td>0.009418</td>\n",
       "      <td>0.015696</td>\n",
       "      <td>0.810400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3.268740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.457170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.738420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.359045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.40000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.648630</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 87 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                H          Li          Be           B           C           N  \\\n",
       "count  451.000000  451.000000  451.000000  451.000000  451.000000  451.000000   \n",
       "mean     0.014502    0.003289    0.001109    0.004009    0.008340    0.020356   \n",
       "std      0.073815    0.035862    0.023544    0.030741    0.049105    0.086824   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "50%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "75%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "max      0.800000    0.500000    0.500000    0.333333    0.428571    0.666667   \n",
       "\n",
       "                O           F          Na          Mg  ...          Tl  \\\n",
       "count  451.000000  451.000000  451.000000  451.000000  ...  451.000000   \n",
       "mean     0.113170    0.010391    0.002525    0.005897  ...    0.006633   \n",
       "std      0.213342    0.078004    0.027131    0.041537  ...    0.048883   \n",
       "min      0.000000    0.000000    0.000000    0.000000  ...    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000  ...    0.000000   \n",
       "50%      0.000000    0.000000    0.000000    0.000000  ...    0.000000   \n",
       "75%      0.000000    0.000000    0.000000    0.000000  ...    0.000000   \n",
       "max      0.750000    0.800000    0.333333    0.384615  ...    0.666667   \n",
       "\n",
       "              Pb          Bi     Ac          Th     Pa           U  \\\n",
       "count  451.00000  451.000000  451.0  451.000000  451.0  451.000000   \n",
       "mean     0.00630    0.024078    0.0    0.004398    0.0    0.003827   \n",
       "std      0.04414    0.086991    0.0    0.034864    0.0    0.031177   \n",
       "min      0.00000    0.000000    0.0    0.000000    0.0    0.000000   \n",
       "25%      0.00000    0.000000    0.0    0.000000    0.0    0.000000   \n",
       "50%      0.00000    0.000000    0.0    0.000000    0.0    0.000000   \n",
       "75%      0.00000    0.000000    0.0    0.000000    0.0    0.000000   \n",
       "max      0.40000    0.666667    0.0    0.333333    0.0    0.333333   \n",
       "\n",
       "               Np          Pu  formation_energy_peratom  \n",
       "count  451.000000  451.000000                451.000000  \n",
       "mean     0.000443    0.000739                 -0.973250  \n",
       "std      0.009418    0.015696                  0.810400  \n",
       "min      0.000000    0.000000                 -3.268740  \n",
       "25%      0.000000    0.000000                 -1.457170  \n",
       "50%      0.000000    0.000000                 -0.738420  \n",
       "75%      0.000000    0.000000                 -0.359045  \n",
       "max      0.200000    0.333333                  0.648630  \n",
       "\n",
       "[8 rows x 87 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 13.6 ms, sys: 0 ns, total: 13.6 ms\n",
      "Wall time: 13.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oqmd_data['comp_obj'] = oqmd_data['pretty_comp'].apply(lambda x: Composition(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 34.6 ms, sys: 0 ns, total: 34.6 ms\n",
      "Wall time: 34.2 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oqmd_data['composition'] = oqmd_data['comp_obj'].apply(lambda x: x.reduced_formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "oqmd_data['nelems'] = oqmd_data['comp_obj'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pretty_comp</th>\n",
       "      <th>H</th>\n",
       "      <th>Li</th>\n",
       "      <th>Be</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>N</th>\n",
       "      <th>O</th>\n",
       "      <th>F</th>\n",
       "      <th>Na</th>\n",
       "      <th>...</th>\n",
       "      <th>Bi</th>\n",
       "      <th>Ac</th>\n",
       "      <th>Th</th>\n",
       "      <th>Pa</th>\n",
       "      <th>U</th>\n",
       "      <th>Np</th>\n",
       "      <th>Pu</th>\n",
       "      <th>formation_energy_peratom</th>\n",
       "      <th>comp_obj</th>\n",
       "      <th>composition</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nelems</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>...</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>...</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>...</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        pretty_comp    H   Li   Be    B    C    N    O    F   Na  ...   Bi  \\\n",
       "nelems                                                            ...        \n",
       "2               176  176  176  176  176  176  176  176  176  176  ...  176   \n",
       "3               245  245  245  245  245  245  245  245  245  245  ...  245   \n",
       "4                29   29   29   29   29   29   29   29   29   29  ...   29   \n",
       "5                 1    1    1    1    1    1    1    1    1    1  ...    1   \n",
       "\n",
       "         Ac   Th   Pa    U   Np   Pu  formation_energy_peratom  comp_obj  \\\n",
       "nelems                                                                     \n",
       "2       176  176  176  176  176  176                       176       176   \n",
       "3       245  245  245  245  245  245                       245       245   \n",
       "4        29   29   29   29   29   29                        29        29   \n",
       "5         1    1    1    1    1    1                         1         1   \n",
       "\n",
       "        composition  \n",
       "nelems               \n",
       "2               176  \n",
       "3               245  \n",
       "4                29  \n",
       "5                 1  \n",
       "\n",
       "[4 rows x 90 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data.groupby('nelems').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        pretty_comp    H   Li   Be    B    C    N    O    F   Na  ...   Bi  \\\n",
      "nelems                                                            ...        \n",
      "2               176  176  176  176  176  176  176  176  176  176  ...  176   \n",
      "3               245  245  245  245  245  245  245  245  245  245  ...  245   \n",
      "4                29   29   29   29   29   29   29   29   29   29  ...   29   \n",
      "5                 1    1    1    1    1    1    1    1    1    1  ...    1   \n",
      "\n",
      "         Ac   Th   Pa    U   Np   Pu  formation_energy_peratom  comp_obj  \\\n",
      "nelems                                                                     \n",
      "2       176  176  176  176  176  176                       176       176   \n",
      "3       245  245  245  245  245  245                       245       245   \n",
      "4        29   29   29   29   29   29                        29        29   \n",
      "5         1    1    1    1    1    1                         1         1   \n",
      "\n",
      "        composition  \n",
      "nelems               \n",
      "2               176  \n",
      "3               245  \n",
      "4                29  \n",
      "5                 1  \n",
      "\n",
      "[4 rows x 90 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(451, 91)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data.query('nelems > 1', inplace=True)\n",
    "print (oqmd_data.groupby('nelems').count())\n",
    "oqmd_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 8.02 ms, sys: 0 ns, total: 8.02 ms\n",
      "Wall time: 7.81 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oqmd_data['comp_dict'] = oqmd_data['composition'].apply(lambda x: data_utils.parse_formula(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Tl', 'Na', 'Re', 'Cu', 'Pt', 'Al', 'Au', 'Nd', 'Fe', 'Sb', 'Si', 'Pu', 'Sr', 'Ge', 'Tm', 'Lu', 'Ni', 'Kr', 'Sc', 'Zr', 'Ho', 'Sn', 'Ir', 'Co', 'Hg', 'Os', 'Ce', 'Mo', 'Li', 'Np', 'Ti', 'V', 'U', 'Y', 'S', 'Br', 'In', 'Cl', 'Yb', 'Sm', 'O', 'As', 'N', 'Ca', 'Mg', 'Rb', 'F', 'Th', 'Pr', 'P', 'Er', 'Rh', 'Ba', 'Pb', 'H', 'Bi', 'Tb', 'W', 'Be', 'Ag', 'Pd', 'Ta', 'Hf', 'Dy', 'Nb', 'Ga', 'Cd', 'Zn', 'Tc', 'B', 'Te', 'Cr', 'C', 'Se', 'Mn', 'I', 'K'}\n",
      "CPU times: user 52.4 ms, sys: 669 µs, total: 53.1 ms\n",
      "Wall time: 52 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "elements_present = set()\n",
    "for i, row in oqmd_data.iterrows():\n",
    "    elements_present |= set(row['comp_dict'].keys())\n",
    "print (elements_present)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "elements = ['H', 'He', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Ne', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'Cl', 'Ar', 'K', \n",
    "            'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Kr', 'Rb',\n",
    "            'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd', 'In', 'Sn', 'Sb', 'Te', 'I', 'Xe', 'Cs', \n",
    "            'Ba', 'La', 'Ce', 'Pr', 'Nd', 'Pm', 'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu', 'Hf', 'Ta',\n",
    "            'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi', 'Po', 'At', 'Rn', 'Fr', 'Ra', 'Ac', 'Th', 'Pa',\n",
    "            'U', 'Np', 'Pu', 'Am', 'Cm', 'Bk', 'Cf', 'Es', 'Fm', 'Md', 'No', 'Lr', 'Rf', 'Db', 'Sg', 'Bh', 'Hs', 'Mt',\n",
    "            'Ds', 'Rg', 'Cn']\n",
    "\n",
    "\n",
    "elements_tl = ['H', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'Cl', 'K',\n",
    " 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se',\n",
    " 'Br', 'Kr', 'Rb', 'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd', 'In',\n",
    " 'Sn', 'Sb', 'Te', 'I', 'Xe', 'Cs', 'Ba', 'La', 'Ce', 'Pr', 'Nd', 'Pm', 'Sm', 'Eu', 'Gd',\n",
    " 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu', 'Hf', 'Ta', 'W', 'Re', 'Os', 'Ir', 'Pt', 'Au',\n",
    " 'Hg', 'Tl', 'Pb', 'Bi', 'Ac','Th', 'Pa', 'U', 'Np', 'Pu']\n",
    "\n",
    "elem_pos = dict()\n",
    "i=0\n",
    "for el in elements:\n",
    "   elem_pos[el] = i\n",
    "   i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Tl', 'Na', 'Re', 'Cu', 'Pt', 'Al', 'Au', 'Nd', 'Fe', 'Sb', 'Si', 'Pu', 'Sr', 'Ge', 'Tm', 'Lu', 'Ni', 'Kr', 'Sc', 'Zr', 'Ho', 'Sn', 'Ir', 'Co', 'Hg', 'Os', 'Ce', 'Mo', 'Li', 'Np', 'Ti', 'V', 'U', 'Y', 'S', 'Br', 'In', 'Cl', 'Yb', 'Sm', 'O', 'As', 'N', 'Ca', 'Mg', 'Rb', 'F', 'Th', 'Pr', 'P', 'Er', 'Rh', 'Ba', 'Pb', 'H', 'Bi', 'Tb', 'W', 'Be', 'Ag', 'Pd', 'Ta', 'Hf', 'Dy', 'Nb', 'Ga', 'Cd', 'Zn', 'Tc', 'B', 'Te', 'Cr', 'C', 'Se', 'Mn', 'I', 'K'} 77\n",
      "[]\n",
      "['Ru', 'Xe', 'Cs', 'La', 'Pm', 'Eu', 'Gd', 'Ac', 'Pa']\n"
     ]
    }
   ],
   "source": [
    "print (elements_present, len(elements_present))\n",
    "print ([e for e in elements_present if e not in elements_tl])\n",
    "print ([e for e in elements_tl if e not in elements_present])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "oqmd_data['comp_fractions'] = oqmd_data['comp_dict'].apply(lambda x: data_utils.get_fractions(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  pretty_comp    H   Li   Be    B    C    N    O    F   Na  ...   Pa    U  \\\n",
      "0      ZrGeTe  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...  0.0  0.0   \n",
      "\n",
      "    Np   Pu  formation_energy_peratom      comp_obj  composition  nelems  \\\n",
      "0  0.0  0.0                   -0.8014  (Zr, Ge, Te)       ZrGeTe       3   \n",
      "\n",
      "                                           comp_dict  \\\n",
      "0  {'Zr': 0.3333333333333333, 'Ge': 0.33333333333...   \n",
      "\n",
      "                                      comp_fractions  \n",
      "0  [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...  \n",
      "\n",
      "[1 rows x 93 columns]\n",
      "(451, 93)\n",
      "(451, 93)\n"
     ]
    }
   ],
   "source": [
    "print (oqmd_data[:1])\n",
    "#oqmd_data['solution_term'] = oqmd_data['comp_obj'].apply(lambda x: compute_mixing_term(x))\n",
    "print (oqmd_data.shape)\n",
    "oqmd_data = oqmd_data[~oqmd_data['comp_fractions'].isnull()]\n",
    "print (oqmd_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(451, 93)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i,e in enumerate(elements_tl):\n",
    "    oqmd_data[e] = [ x[i] for x in oqmd_data['comp_fractions']]\n",
    "oqmd_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['pretty_comp', 'H', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Na', 'Mg',\n",
       "       'Al', 'Si', 'P', 'S', 'Cl', 'K', 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn',\n",
       "       'Fe', 'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Kr', 'Rb',\n",
       "       'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd', 'In',\n",
       "       'Sn', 'Sb', 'Te', 'I', 'Xe', 'Cs', 'Ba', 'La', 'Ce', 'Pr', 'Nd', 'Pm',\n",
       "       'Sm', 'Eu', 'Gd', 'Tb', 'Dy', 'Ho', 'Er', 'Tm', 'Yb', 'Lu', 'Hf', 'Ta',\n",
       "       'W', 'Re', 'Os', 'Ir', 'Pt', 'Au', 'Hg', 'Tl', 'Pb', 'Bi', 'Ac', 'Th',\n",
       "       'Pa', 'U', 'Np', 'Pu', 'formation_energy_peratom', 'comp_obj',\n",
       "       'composition', 'nelems', 'comp_dict', 'comp_fractions'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_calculators = MultipleFeaturizer([cf.Stoichiometry(), cf.ElementProperty.from_preset(\"magpie\"),\n",
    "                                          cf.ValenceOrbital(props=['avg']), cf.IonProperty(fast=True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_labels = feature_calculators.feature_labels()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['0-norm', '2-norm', '3-norm', '5-norm', '7-norm', '10-norm', 'MagpieData minimum Number', 'MagpieData maximum Number', 'MagpieData range Number', 'MagpieData mean Number', 'MagpieData avg_dev Number', 'MagpieData mode Number', 'MagpieData minimum MendeleevNumber', 'MagpieData maximum MendeleevNumber', 'MagpieData range MendeleevNumber', 'MagpieData mean MendeleevNumber', 'MagpieData avg_dev MendeleevNumber', 'MagpieData mode MendeleevNumber', 'MagpieData minimum AtomicWeight', 'MagpieData maximum AtomicWeight', 'MagpieData range AtomicWeight', 'MagpieData mean AtomicWeight', 'MagpieData avg_dev AtomicWeight', 'MagpieData mode AtomicWeight', 'MagpieData minimum MeltingT', 'MagpieData maximum MeltingT', 'MagpieData range MeltingT', 'MagpieData mean MeltingT', 'MagpieData avg_dev MeltingT', 'MagpieData mode MeltingT', 'MagpieData minimum Column', 'MagpieData maximum Column', 'MagpieData range Column', 'MagpieData mean Column', 'MagpieData avg_dev Column', 'MagpieData mode Column', 'MagpieData minimum Row', 'MagpieData maximum Row', 'MagpieData range Row', 'MagpieData mean Row', 'MagpieData avg_dev Row', 'MagpieData mode Row', 'MagpieData minimum CovalentRadius', 'MagpieData maximum CovalentRadius', 'MagpieData range CovalentRadius', 'MagpieData mean CovalentRadius', 'MagpieData avg_dev CovalentRadius', 'MagpieData mode CovalentRadius', 'MagpieData minimum Electronegativity', 'MagpieData maximum Electronegativity', 'MagpieData range Electronegativity', 'MagpieData mean Electronegativity', 'MagpieData avg_dev Electronegativity', 'MagpieData mode Electronegativity', 'MagpieData minimum NsValence', 'MagpieData maximum NsValence', 'MagpieData range NsValence', 'MagpieData mean NsValence', 'MagpieData avg_dev NsValence', 'MagpieData mode NsValence', 'MagpieData minimum NpValence', 'MagpieData maximum NpValence', 'MagpieData range NpValence', 'MagpieData mean NpValence', 'MagpieData avg_dev NpValence', 'MagpieData mode NpValence', 'MagpieData minimum NdValence', 'MagpieData maximum NdValence', 'MagpieData range NdValence', 'MagpieData mean NdValence', 'MagpieData avg_dev NdValence', 'MagpieData mode NdValence', 'MagpieData minimum NfValence', 'MagpieData maximum NfValence', 'MagpieData range NfValence', 'MagpieData mean NfValence', 'MagpieData avg_dev NfValence', 'MagpieData mode NfValence', 'MagpieData minimum NValence', 'MagpieData maximum NValence', 'MagpieData range NValence', 'MagpieData mean NValence', 'MagpieData avg_dev NValence', 'MagpieData mode NValence', 'MagpieData minimum NsUnfilled', 'MagpieData maximum NsUnfilled', 'MagpieData range NsUnfilled', 'MagpieData mean NsUnfilled', 'MagpieData avg_dev NsUnfilled', 'MagpieData mode NsUnfilled', 'MagpieData minimum NpUnfilled', 'MagpieData maximum NpUnfilled', 'MagpieData range NpUnfilled', 'MagpieData mean NpUnfilled', 'MagpieData avg_dev NpUnfilled', 'MagpieData mode NpUnfilled', 'MagpieData minimum NdUnfilled', 'MagpieData maximum NdUnfilled', 'MagpieData range NdUnfilled', 'MagpieData mean NdUnfilled', 'MagpieData avg_dev NdUnfilled', 'MagpieData mode NdUnfilled', 'MagpieData minimum NfUnfilled', 'MagpieData maximum NfUnfilled', 'MagpieData range NfUnfilled', 'MagpieData mean NfUnfilled', 'MagpieData avg_dev NfUnfilled', 'MagpieData mode NfUnfilled', 'MagpieData minimum NUnfilled', 'MagpieData maximum NUnfilled', 'MagpieData range NUnfilled', 'MagpieData mean NUnfilled', 'MagpieData avg_dev NUnfilled', 'MagpieData mode NUnfilled', 'MagpieData minimum GSvolume_pa', 'MagpieData maximum GSvolume_pa', 'MagpieData range GSvolume_pa', 'MagpieData mean GSvolume_pa', 'MagpieData avg_dev GSvolume_pa', 'MagpieData mode GSvolume_pa', 'MagpieData minimum GSbandgap', 'MagpieData maximum GSbandgap', 'MagpieData range GSbandgap', 'MagpieData mean GSbandgap', 'MagpieData avg_dev GSbandgap', 'MagpieData mode GSbandgap', 'MagpieData minimum GSmagmom', 'MagpieData maximum GSmagmom', 'MagpieData range GSmagmom', 'MagpieData mean GSmagmom', 'MagpieData avg_dev GSmagmom', 'MagpieData mode GSmagmom', 'MagpieData minimum SpaceGroupNumber', 'MagpieData maximum SpaceGroupNumber', 'MagpieData range SpaceGroupNumber', 'MagpieData mean SpaceGroupNumber', 'MagpieData avg_dev SpaceGroupNumber', 'MagpieData mode SpaceGroupNumber', 'avg s valence electrons', 'avg p valence electrons', 'avg d valence electrons', 'avg f valence electrons', 'compound possible', 'max ionic char', 'avg ionic char'] 145\n"
     ]
    }
   ],
   "source": [
    "print (feature_labels, len(feature_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc9bfe100bd14a10992e0fe875fd8666",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='MultipleFeaturizer', max=451.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "CPU times: user 359 ms, sys: 276 ms, total: 635 ms\n",
      "Wall time: 678 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oqmd_data = feature_calculators.featurize_dataframe(oqmd_data, col_id='comp_obj');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(451, 145)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data[feature_labels].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(451, 238)\n"
     ]
    }
   ],
   "source": [
    "oqmd_data = oqmd_data[~oqmd_data[feature_labels].isnull().any(axis=1)]\n",
    "print (oqmd_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>H</th>\n",
       "      <th>Li</th>\n",
       "      <th>Be</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>N</th>\n",
       "      <th>O</th>\n",
       "      <th>F</th>\n",
       "      <th>Na</th>\n",
       "      <th>Mg</th>\n",
       "      <th>...</th>\n",
       "      <th>MagpieData range SpaceGroupNumber</th>\n",
       "      <th>MagpieData mean SpaceGroupNumber</th>\n",
       "      <th>MagpieData avg_dev SpaceGroupNumber</th>\n",
       "      <th>MagpieData mode SpaceGroupNumber</th>\n",
       "      <th>avg s valence electrons</th>\n",
       "      <th>avg p valence electrons</th>\n",
       "      <th>avg d valence electrons</th>\n",
       "      <th>avg f valence electrons</th>\n",
       "      <th>max ionic char</th>\n",
       "      <th>avg ionic char</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "      <td>451.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.014502</td>\n",
       "      <td>0.003289</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>0.004009</td>\n",
       "      <td>0.008340</td>\n",
       "      <td>0.020356</td>\n",
       "      <td>0.113170</td>\n",
       "      <td>0.010391</td>\n",
       "      <td>0.002525</td>\n",
       "      <td>0.005897</td>\n",
       "      <td>...</td>\n",
       "      <td>147.995565</td>\n",
       "      <td>108.522974</td>\n",
       "      <td>57.563808</td>\n",
       "      <td>61.567627</td>\n",
       "      <td>1.906335</td>\n",
       "      <td>2.966925</td>\n",
       "      <td>4.620782</td>\n",
       "      <td>1.372457</td>\n",
       "      <td>0.341089</td>\n",
       "      <td>0.064993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.073815</td>\n",
       "      <td>0.035862</td>\n",
       "      <td>0.023544</td>\n",
       "      <td>0.030741</td>\n",
       "      <td>0.049105</td>\n",
       "      <td>0.086824</td>\n",
       "      <td>0.213342</td>\n",
       "      <td>0.078004</td>\n",
       "      <td>0.027131</td>\n",
       "      <td>0.041537</td>\n",
       "      <td>...</td>\n",
       "      <td>60.629072</td>\n",
       "      <td>43.493470</td>\n",
       "      <td>25.110998</td>\n",
       "      <td>56.334373</td>\n",
       "      <td>0.171937</td>\n",
       "      <td>0.816071</td>\n",
       "      <td>3.313917</td>\n",
       "      <td>2.192619</td>\n",
       "      <td>0.238241</td>\n",
       "      <td>0.046884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.111111</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>124.000000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>38.500000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>1.833333</td>\n",
       "      <td>2.571429</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.137763</td>\n",
       "      <td>0.023898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>161.000000</td>\n",
       "      <td>104.250000</td>\n",
       "      <td>59.644970</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.319141</td>\n",
       "      <td>0.057395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>211.000000</td>\n",
       "      <td>133.142857</td>\n",
       "      <td>73.162943</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.519231</td>\n",
       "      <td>7.500000</td>\n",
       "      <td>2.800000</td>\n",
       "      <td>0.532919</td>\n",
       "      <td>0.100665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.428571</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>...</td>\n",
       "      <td>227.000000</td>\n",
       "      <td>227.000000</td>\n",
       "      <td>112.500000</td>\n",
       "      <td>229.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>0.917619</td>\n",
       "      <td>0.200477</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 232 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                H          Li          Be           B           C           N  \\\n",
       "count  451.000000  451.000000  451.000000  451.000000  451.000000  451.000000   \n",
       "mean     0.014502    0.003289    0.001109    0.004009    0.008340    0.020356   \n",
       "std      0.073815    0.035862    0.023544    0.030741    0.049105    0.086824   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "50%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "75%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "max      0.800000    0.500000    0.500000    0.333333    0.428571    0.666667   \n",
       "\n",
       "                O           F          Na          Mg  ...  \\\n",
       "count  451.000000  451.000000  451.000000  451.000000  ...   \n",
       "mean     0.113170    0.010391    0.002525    0.005897  ...   \n",
       "std      0.213342    0.078004    0.027131    0.041537  ...   \n",
       "min      0.000000    0.000000    0.000000    0.000000  ...   \n",
       "25%      0.000000    0.000000    0.000000    0.000000  ...   \n",
       "50%      0.000000    0.000000    0.000000    0.000000  ...   \n",
       "75%      0.000000    0.000000    0.000000    0.000000  ...   \n",
       "max      0.750000    0.800000    0.333333    0.384615  ...   \n",
       "\n",
       "       MagpieData range SpaceGroupNumber  MagpieData mean SpaceGroupNumber  \\\n",
       "count                         451.000000                        451.000000   \n",
       "mean                          147.995565                        108.522974   \n",
       "std                            60.629072                         43.493470   \n",
       "min                             0.000000                          8.000000   \n",
       "25%                           124.000000                         81.000000   \n",
       "50%                           161.000000                        104.250000   \n",
       "75%                           211.000000                        133.142857   \n",
       "max                           227.000000                        227.000000   \n",
       "\n",
       "       MagpieData avg_dev SpaceGroupNumber  MagpieData mode SpaceGroupNumber  \\\n",
       "count                           451.000000                        451.000000   \n",
       "mean                             57.563808                         61.567627   \n",
       "std                              25.110998                         56.334373   \n",
       "min                               0.000000                          2.000000   \n",
       "25%                              38.500000                         12.000000   \n",
       "50%                              59.644970                         64.000000   \n",
       "75%                              73.162943                         70.000000   \n",
       "max                             112.500000                        229.000000   \n",
       "\n",
       "       avg s valence electrons  avg p valence electrons  \\\n",
       "count               451.000000               451.000000   \n",
       "mean                  1.906335                 2.966925   \n",
       "std                   0.171937                 0.816071   \n",
       "min                   1.111111                 0.000000   \n",
       "25%                   1.833333                 2.571429   \n",
       "50%                   2.000000                 3.000000   \n",
       "75%                   2.000000                 3.519231   \n",
       "max                   2.000000                 5.333333   \n",
       "\n",
       "       avg d valence electrons  avg f valence electrons  max ionic char  \\\n",
       "count               451.000000               451.000000      451.000000   \n",
       "mean                  4.620782                 1.372457        0.341089   \n",
       "std                   3.313917                 2.192619        0.238241   \n",
       "min                   0.000000                 0.000000        0.000025   \n",
       "25%                   1.666667                 0.000000        0.137763   \n",
       "50%                   4.000000                 0.000000        0.319141   \n",
       "75%                   7.500000                 2.800000        0.532919   \n",
       "max                  10.000000                14.000000        0.917619   \n",
       "\n",
       "       avg ionic char  \n",
       "count      451.000000  \n",
       "mean         0.064993  \n",
       "std          0.046884  \n",
       "min          0.000006  \n",
       "25%          0.023898  \n",
       "50%          0.057395  \n",
       "75%          0.100665  \n",
       "max          0.200477  \n",
       "\n",
       "[8 rows x 232 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oqmd_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Ru', 'comp_obj', 'comp_dict', 'Cs', 'La', 'pretty_comp', 'Pm', 'Gd', 'composition', 'nelems', 'Eu', 'Pa', 'comp_fractions', 'Xe', 'Ac', 'formation_energy_peratom'}\n"
     ]
    }
   ],
   "source": [
    "properties = set(oqmd_data.columns.tolist()) - (set(feature_labels) | set(elements_present))\n",
    "print (properties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(451, 235)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for col in ['comp_fractions', 'comp_dict', 'comp_obj']: oqmd_data.drop(col, axis=1, inplace=True)\n",
    "oqmd_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Ru', 'Cs', 'La', 'Pm', 'Gd', 'composition', 'nelems', 'Eu', 'Pa', 'Xe', 'Ac', 'formation_energy_peratom'}\n"
     ]
    }
   ],
   "source": [
    "properties = set(oqmd_data.columns.tolist()) - (set(feature_labels) | set(elements_present) |{'pretty_comp'})\n",
    "print (properties)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "oqmd_data = oqmd_data[['pretty_comp']+elements_tl+list(feature_labels)+list(properties)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['pretty_comp', 'H', 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Na',\n",
      "       ...\n",
      "       'La', 'Pm', 'Gd', 'composition', 'nelems', 'Eu', 'Pa', 'Xe', 'Ac',\n",
      "       'formation_energy_peratom'],\n",
      "      dtype='object', length=244)\n",
      "(451, 244)\n"
     ]
    }
   ],
   "source": [
    "print (oqmd_data.columns)\n",
    "print (oqmd_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "oqmd_data.to_csv('sample_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}