05_check_structures.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "whole-charger",
   "metadata": {},
   "source": [
    "# Check crystal structures and remove the ones with overlapping atoms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "solar-mistress",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle \n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "black-hazard",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'FIQCEN',\n",
       " 'graph_hash': '521324a26079f95c5a55e7f417c3fa7f',\n",
       " 'scaffold_hash': 'e1682de934a12384f1952d34148e3832',\n",
       " 'symmetry_hash': 'mEopNc0Bbry+C0Mp9WB7PJqumiUvMB4qfFQS9GqwjvI=225',\n",
       " 'formula': 'Cu12 H24 C72 O48',\n",
       " 'path': None,\n",
       " 'density': 0.879095741266423,\n",
       " 'has_carbon': True,\n",
       " 'has_atomic_overlaps': False,\n",
       " 'has_overcoordinated_c': False,\n",
       " 'has_overcoordinated_n': False,\n",
       " 'has_overcoordinated_h': False,\n",
       " 'has_undercoordinated_c': False,\n",
       " 'has_undercoordinated_n': False,\n",
       " 'has_undercoordinated_rare_earth': False,\n",
       " 'has_metal': True,\n",
       " 'has_lone_molecule': False,\n",
       " 'has_high_charges': True,\n",
       " 'is_porous': None,\n",
       " 'has_suspicicious_terminal_oxo': False,\n",
       " 'has_not_hydrogen': False}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MANUAL_EXCLUSIONS = list(pd.read_csv(\"data/manual_exclusions.csv\")['identifier'])\n",
    "\n",
    "with open(\"data/csd_542_mofchecker.json\", \"r\") as file:\n",
    "    mof_checker = json.load(file)\n",
    "    \n",
    "# fix\n",
    "for mof, mof_dict in mof_checker.items():\n",
    "    for structure in mof_dict:\n",
    "        mof_checker[mof][structure]['has_atomic_overlaps'] = not mof_checker[mof][structure]['has_atomic_overlaps']\n",
    "        mof_checker[mof][structure]['has_not_hydrogen'] = not mof_checker[mof][structure]['has_hydrogen']\n",
    "        del mof_checker[mof][structure]['has_hydrogen']\n",
    "\n",
    "mof_checker['FIQCEN']['asr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "missing-shock",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... after manual exclusions ...\n",
      "All identified NIST-ISODB adsorbents: 676\n",
      "All identified CSD entries: 777\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/step-08.csv\").dropna(subset=['nist_mat'])\n",
    "df = df[~df.identifier.isin(MANUAL_EXCLUSIONS)]                        \n",
    "\n",
    "mat_sel = list(set(df['nist_mat']))\n",
    "mat_sel.sort()\n",
    "mat_dict = {}\n",
    "csd_refcodes_mem = []\n",
    "for nist_name in mat_sel:\n",
    "    csd_refcodes = list(df[df['nist_mat']==nist_name]['identifier'].values)\n",
    "    csd_refcodes_mem += csd_refcodes\n",
    "    mat_dict[nist_name] = {}\n",
    "    for csd_refcode in csd_refcodes:       \n",
    "        mat_dict[nist_name][csd_refcode] = df[df['identifier']==csd_refcode]['sgh_asr'].values[0]\n",
    "        \n",
    "with open(\"data/step-09.yml\", \"w\") as f:\n",
    "    yaml.dump(mat_dict, f)\n",
    "# step-09.yml: { nist_adsorbent1: [csd1: SGH, csd2: SGH, ...], ... }\n",
    "    \n",
    "print(\"... after manual exclusions ...\")    \n",
    "print(f'All identified NIST-ISODB adsorbents: {len(mat_dict)}')\n",
    "print(f'All identified CSD entries: {len(csd_refcodes_mem)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "liquid-mounting",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DIHVIB': '521324a26079f95c5a55e7f417c3fa7f',\n",
       " 'DOTSOV42': '521324a26079f95c5a55e7f417c3fa7f',\n",
       " 'FIQCEN': '521324a26079f95c5a55e7f417c3fa7f',\n",
       " 'LUDLED': '521324a26079f95c5a55e7f417c3fa7f'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"data/step-09.yml\", \"r\") as file:\n",
    "    mat_dict = yaml.safe_load(file)\n",
    "mat_dict['CuBTC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "blocked-pricing",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cu-MOF-74\n",
      "{'COKNIB01': '9aa2c07385f5f2341ea925427725a3c4', 'COKNIB02': 'MISSING_COKNIB02', 'COKNIB03': '9aa2c07385f5f2341ea925427725a3c4'}\n",
      "MOF-HTB'\n",
      "{'NIBHUC': 'MISSING_NIBHUC', 'NIBJAK': '40173a48e39019e3fa20d50edc7e9133'}\n",
      "Mn-MOF-74\n",
      "{'COKNUN01': '4c1a36d71cb621668c336926a7773534', 'COKNUN02': 'MISSING_COKNUN02', 'COKNUN03': '4c1a36d71cb621668c336926a7773534', 'ORIWET': '4c1a36d71cb621668c336926a7773534'}\n",
      "PCN-12\n",
      "{'HOGLEV': '7ba11b2d99e473be8262cad965df7332', 'HOGLEV01': 'MISSING_HOGLEV01'}\n",
      "\n",
      "NOTE: these left are ok, possibly primitive/conventional cells \n",
      "\n",
      "Unique NIST - CSD matches: 623\n",
      "NIST - multiple CSD matches: 53\n",
      "NIST - CSD matches, total: 676\n",
      "NIST - CSD matches to check because of different graphs: 4\n"
     ]
    }
   ],
   "source": [
    "# check the ones that have same nist_mat but different graph\n",
    "# - used to verify and create manual_exclusions.csv\n",
    "# - some are left because by visual inspection they are the same\n",
    "\n",
    "with open(\"data/step-09.yml\", \"r\") as file:\n",
    "    mat_dict = yaml.safe_load(file)\n",
    "\n",
    "count_unique, count_multiple, count_tocheck = 0, 0, 0\n",
    "to_check = []\n",
    "for mat_name, d in mat_dict.items():\n",
    "    if len(d)==0:\n",
    "           print('ERROR', mat_name)\n",
    "    elif len(d)==1:\n",
    "        count_unique+=1\n",
    "    #elif len(d)<30:\n",
    "    else:\n",
    "        count_multiple+=1\n",
    "        graph_values = list(d.values())\n",
    "        if not all( [ x==graph_values[0] for x in graph_values]):\n",
    "            print(mat_name)\n",
    "            print(d)\n",
    "            to_check+=list(d.keys())\n",
    "            count_tocheck+=1\n",
    "            \n",
    "print()\n",
    "print('NOTE: these left are ok, possibly primitive/conventional cells ')\n",
    "print()\n",
    "print(\"Unique NIST - CSD matches:\", count_unique)\n",
    "print(\"NIST - multiple CSD matches:\", count_multiple)\n",
    "print(\"NIST - CSD matches, total:\", count_unique+count_multiple)\n",
    "print(\"NIST - CSD matches to check because of different graphs:\", count_tocheck)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "special-witness",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "has_not_hydrogen 47\n",
      "has_atomic_overlaps 38\n",
      "has_lone_molecule 80\n",
      "missing 29\n",
      "total 777\n"
     ]
    }
   ],
   "source": [
    "# Inspection\n",
    "\n",
    "with open(\"data/step-09.yml\", \"r\") as file:\n",
    "    mat_dict = yaml.safe_load(file)\n",
    "    \n",
    "count = {\n",
    "    'has_not_hydrogen': [],\n",
    "    'has_atomic_overlaps': [],\n",
    "    'has_lone_molecule': [],    \n",
    "}\n",
    "\n",
    "missing = []\n",
    "for csd in csd_refcodes_mem:\n",
    "    if csd not in mof_checker:\n",
    "        missing.append(csd)\n",
    "        continue\n",
    "    for k in count:\n",
    "        if mof_checker[csd]['asr'][k]:\n",
    "            count[k].append(csd)\n",
    "\n",
    "count['missing'] = missing\n",
    "count['total'] = csd_refcodes_mem\n",
    "\n",
    "for k in count:\n",
    "    print(k, len(count[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "italian-share",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NIST adsorbents left after cleaning: 569\n",
      "CSD entries adsorbents left after cleaning: 666\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/step-09.yml\", \"r\") as file:\n",
    "    mat_dict = yaml.safe_load(file)\n",
    "    \n",
    "mat_dict_new = {}\n",
    "for nist_mat, nist_dict in mat_dict.items():\n",
    "    nist_dict_new = {}\n",
    "    for csd_mat in nist_dict:  \n",
    "        if not csd_mat in count['has_atomic_overlaps'] + count['has_lone_molecule']:\n",
    "            nist_dict_new[csd_mat] = nist_dict[csd_mat]\n",
    "    if len(nist_dict_new)>0:\n",
    "        mat_dict_new[nist_mat] = nist_dict_new\n",
    "        \n",
    "with open(\"data/step-10.yaml\", \"w\") as f: # Same as -40 but without CSD entries with problems (overlaps)\n",
    "    yaml.dump(mat_dict_new, f)  \n",
    "    \n",
    "print(\"NIST adsorbents left after cleaning:\", len(mat_dict_new) ) \n",
    "print(\"CSD entries adsorbents left after cleaning:\", sum([ len(v) for v in mat_dict_new.values()]) )   "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:isodb_csd]",
   "language": "python",
   "name": "conda-env-isodb_csd-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}