create_gdx.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import windc_data as wd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    \"bea_gsp\":wd.BeaGsp,\n",
    "    \"bea_pce\":wd.BeaPce,\n",
    "    \"bea_supply_det\":wd.BeaSupplyDet,\n",
    "    \"bea_supply\" : wd.BeaSupply,\n",
    "    \"bea_use_det\" : wd.BeaUseDet,\n",
    "    \"bea_use\" : wd.BeaUse,\n",
    "    \"census_sgf\" : wd.CensusSGF,\n",
    "    \"cfs\" : wd.CFS,\n",
    "    \"eia_crude_price\" : wd.EiaCrude,\n",
    "    \"eia_emissions\" : wd.EiaEmissions,\n",
    "    \"eia_heatrate\" : wd.EiaHeatrate,\n",
    "    \"eia_seds\" : wd.EiaSeds,\n",
    "    \"nass\" : wd.Nass,\n",
    "    \"state_exim\" : wd.StateExIm,\n",
    "    \"emission_rate\":wd.EmissionRate\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = r\"C:\\Users\\mphillipson\\Documents\\WiNDC\\windc_raw_data\\windc_3_0_0\"\n",
    "\n",
    "w1 = wd.WindcEnvironment(data_dir,\n",
    "                         load = datasets,\n",
    "                         verbose = False)\n",
    "\n",
    "#w1.to_gdx(r\"C:\\Users\\mphillipson\\Documents\\WiNDC\")\n",
    "\n",
    "w1.to_gdx(r\"C:\\Users\\mphillipson\\Documents\\WiNDC\\buildstream_julia\\base_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['r', 'sr', 'i', 'fd', 'ts', 'va', 'i_det', 'sector_map', 'seds_src', 'yr', 'version', 'supply_units', 'use_units'])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>windc_label</th>\n",
       "      <th>description_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>pce</td>\n",
       "      <td>Personal consumption expenditures (F010)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>equipment</td>\n",
       "      <td>Nonresidential private fixed investment in equ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>intelprop</td>\n",
       "      <td>Nonresidential private fixed investment in int...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>residential</td>\n",
       "      <td>Residential private fixed investment (F02R)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>structures</td>\n",
       "      <td>Nonresidential private fixed investment in str...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>changinv</td>\n",
       "      <td>Change in private inventories (F030)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>defense</td>\n",
       "      <td>National defense: Consumption expenditures (F06C)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>def_equipment</td>\n",
       "      <td>Federal national defense: Gross investment in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>def_intelprop</td>\n",
       "      <td>Federal national defense: Gross investment in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>def_structures</td>\n",
       "      <td>Federal national defense: Gross investment in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>nondefense</td>\n",
       "      <td>Nondefense: Consumption expenditures (F07C)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>fed_equipment</td>\n",
       "      <td>Federal nondefense: Gross investment in equipm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>fed_intelprop</td>\n",
       "      <td>Federal nondefense: Gross investment in intell...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>fed_structures</td>\n",
       "      <td>Federal nondefense: Gross investment in struct...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>state_consume</td>\n",
       "      <td>State and local government consumption expendi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>state_equipment</td>\n",
       "      <td>State and local: Gross investment in equipment...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>state_intelprop</td>\n",
       "      <td>State and local: Gross investment in intellect...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>state_invest</td>\n",
       "      <td>State and local: Gross investment in structure...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         windc_label                                   description_code\n",
       "83               pce           Personal consumption expenditures (F010)\n",
       "84         equipment  Nonresidential private fixed investment in equ...\n",
       "85         intelprop  Nonresidential private fixed investment in int...\n",
       "86       residential        Residential private fixed investment (F02R)\n",
       "87        structures  Nonresidential private fixed investment in str...\n",
       "88          changinv               Change in private inventories (F030)\n",
       "90           defense  National defense: Consumption expenditures (F06C)\n",
       "91     def_equipment  Federal national defense: Gross investment in ...\n",
       "92     def_intelprop  Federal national defense: Gross investment in ...\n",
       "93    def_structures  Federal national defense: Gross investment in ...\n",
       "94        nondefense        Nondefense: Consumption expenditures (F07C)\n",
       "95     fed_equipment  Federal nondefense: Gross investment in equipm...\n",
       "96     fed_intelprop  Federal nondefense: Gross investment in intell...\n",
       "97    fed_structures  Federal nondefense: Gross investment in struct...\n",
       "98     state_consume  State and local government consumption expendi...\n",
       "99   state_equipment  State and local: Gross investment in equipment...\n",
       "100  state_intelprop  State and local: Gross investment in intellect...\n",
       "101     state_invest  State and local: Gross investment in structure..."
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1[\"fd\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>windc_label</th>\n",
       "      <th>description_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>agr</td>\n",
       "      <td>Farms (111CA)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fof</td>\n",
       "      <td>Forestry, fishing, and related activities (113FF)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>oil</td>\n",
       "      <td>Oil and gas extraction (211)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>min</td>\n",
       "      <td>Mining, except oil and gas (212)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>smn</td>\n",
       "      <td>Support activities for mining (213)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>fen</td>\n",
       "      <td>Federal government enterprises (GFE)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>slg</td>\n",
       "      <td>State and local general government (GSLG)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>sle</td>\n",
       "      <td>State and local government enterprises (GSLE)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>oth</td>\n",
       "      <td>Noncomparable imports and rest-of-the-world ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>use</td>\n",
       "      <td>Scrap, used and secondhand goods (Used)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   windc_label                                   description_code\n",
       "0          agr                                      Farms (111CA)\n",
       "1          fof  Forestry, fishing, and related activities (113FF)\n",
       "2          oil                       Oil and gas extraction (211)\n",
       "3          min                   Mining, except oil and gas (212)\n",
       "4          smn                Support activities for mining (213)\n",
       "..         ...                                                ...\n",
       "68         fen               Federal government enterprises (GFE)\n",
       "69         slg          State and local general government (GSLG)\n",
       "70         sle      State and local government enterprises (GSLE)\n",
       "71         oth  Noncomparable imports and rest-of-the-world ad...\n",
       "72         use            Scrap, used and secondhand goods (Used)\n",
       "\n",
       "[73 rows x 2 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w1[\"i\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gamstransfer as gt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Warning: The GAMS version (41.1.0) differs from the API version (41.0.0).\n",
      "--- Warning: The GAMS version (41.1.0) differs from the API version (41.0.0).\n"
     ]
    }
   ],
   "source": [
    "X = gt.Container(r\"C:\\Users\\mphillipson\\Documents\\WiNDC\\windc_build\\core\\windc_base.gdx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uni_0</th>\n",
       "      <th>uni_1</th>\n",
       "      <th>uni_2</th>\n",
       "      <th>uni_3</th>\n",
       "      <th>uni_4</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>104</td>\n",
       "      <td>104</td>\n",
       "      <td>212</td>\n",
       "      <td>10</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>2.204431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>104</td>\n",
       "      <td>104</td>\n",
       "      <td>212</td>\n",
       "      <td>11</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>3.736687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>104</td>\n",
       "      <td>104</td>\n",
       "      <td>212</td>\n",
       "      <td>19</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>0.351043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>104</td>\n",
       "      <td>104</td>\n",
       "      <td>212</td>\n",
       "      <td>31</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>0.542113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>104</td>\n",
       "      <td>104</td>\n",
       "      <td>212</td>\n",
       "      <td>34</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>3.618401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287204</th>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>331</td>\n",
       "      <td>40</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>1341.953532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287205</th>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>331</td>\n",
       "      <td>41</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>311.817305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287206</th>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>331</td>\n",
       "      <td>43</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>6.737957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287207</th>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>331</td>\n",
       "      <td>13</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>0.149248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287208</th>\n",
       "      <td>99999</td>\n",
       "      <td>99999</td>\n",
       "      <td>331</td>\n",
       "      <td>36</td>\n",
       "      <td>millions of us dollars (USD)</td>\n",
       "      <td>821.540596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>287209 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        uni_0  uni_1 uni_2 uni_3                         uni_4        value\n",
       "0         104    104   212    10  millions of us dollars (USD)     2.204431\n",
       "1         104    104   212    11  millions of us dollars (USD)     3.736687\n",
       "2         104    104   212    19  millions of us dollars (USD)     0.351043\n",
       "3         104    104   212    31  millions of us dollars (USD)     0.542113\n",
       "4         104    104   212    34  millions of us dollars (USD)     3.618401\n",
       "...       ...    ...   ...   ...                           ...          ...\n",
       "287204  99999  99999   331    40  millions of us dollars (USD)  1341.953532\n",
       "287205  99999  99999   331    41  millions of us dollars (USD)   311.817305\n",
       "287206  99999  99999   331    43  millions of us dollars (USD)     6.737957\n",
       "287207  99999  99999   331    13  millions of us dollars (USD)     0.149248\n",
       "287208  99999  99999   331    36  millions of us dollars (USD)   821.540596\n",
       "\n",
       "[287209 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[\"cfsdata_ma_units\"].records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdx_container = gt.Container()\n",
    "\n",
    "old_gdx_sets = w1._gdx_sets()\n",
    "\n",
    "sets_to_keep = [\"yr\"]\n",
    "\n",
    "gdx_sets = {e:old_gdx_sets[e] for e in sets_to_keep}\n",
    "\n",
    "\n",
    "\n",
    "for key,d in gdx_sets.items():\n",
    "\n",
    "    if key not in gdx_container.data:\n",
    "        num_domain = 1\n",
    "        if type(d['elements']) == type(pd.DataFrame()):\n",
    "            num_domain = len(d['elements'].columns)-1\n",
    "        gdx_container.addSet(key,[\"*\"]*num_domain,records = d['elements'],description = d['text'])\n",
    "\n",
    "\n",
    "gdx_container.write(\"test.gdx\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "old_gdx_sets.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdx_container[\"yr\"].records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gdx_container[\"r\"]\n",
    "\n",
    "gdx_container.write(\"test.gdx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key in gdx_container.data:\n",
    "    print(gdx_container[key].hasDuplicateRecords())\n",
    "    print(gdx_container[key].hasDomainViolations())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdx_container[\"i\"].records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdx_container[\"i_det\"].records.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdx_container.data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_gdx(self,output_dir,output_name = \"windc.gdx\"):\n",
    "    import gamstransfer as gt\n",
    "    \n",
    "    gdx_container = gt.Container()\n",
    "    \n",
    "    gdx_sets = self._gdx_sets()\n",
    "    \n",
    "    for key,d in gdx_sets.items():\n",
    "    #    for key,d in [(k,d) for (k,d) in gdx_dict.items() if d['type'] == 'set']:\n",
    "        if key not in gdx_container.data:\n",
    "            num_domain = 1\n",
    "            if type(d['elements']) == type(pd.DataFrame()):\n",
    "                num_domain = len(d['elements'].columns)-1\n",
    "            gdx_container.addSet(key,[\"*\"]*num_domain,records = d['elements'],description = d['text'])\n",
    "    \n",
    "    \n",
    "    for key in self.data:\n",
    "        self.data[key].build_gdx(gdx_container)\n",
    "        \n",
    "    gdx_container.write(os.path.join(output_dir,output_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "years = (1997,2018)\n",
    "\n",
    "pd.DataFrame(zip(list(range(years[0],years[1]+1)),list(range(years[0],years[1]+1))),columns = ['yr',\"description\"]),"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gams41",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "2d8148a8a31ef43057a38d74919842836b9e63bf1d1d4aa18fbf63959bc65a17"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}