Merge efedcf5 into 7bc7e19

src/pudl/constants.py

@@ -1521,7 +1521,7 @@
     },
     'epacems': {
         'years': tuple(range(1995, 2020)),
-        'states': cems_states.keys()},
+        'states': tuple(cems_states.keys())},
     'ferc1': {
         'years': tuple(range(1994, 2020))
     },
@@ -1968,7 +1968,7 @@
     'other'
 ]
 
-ESTIMATED_OR_ACTUAL = {'E': 'Estimated', 'A': 'Actual'}
+ESTIMATED_OR_ACTUAL = {'E': 'estimated', 'A': 'actual'}
 
 TRANSIT_TYPE_DICT = {
     'CV': 'conveyer',

src/pudl/convert/datapkg_to_sqlite.py

@@ -150,8 +150,13 @@ def main():
     logger.info(f"pudl_in={pudl_settings['pudl_in']}")
     logger.info(f"pudl_out={pudl_settings['pudl_out']}")
 
-    # Check if there's already a PUDL SQLite DB that we should not clobber:
-    if not args.clobber and pathlib.Path(pudl_settings["pudl_db"]).exists():
+    # Check if there's already a PUDL SQLite DB that we should not clobber
+    # Need to remove the sqlite:/// prefix from the SQLAlchemy URL since
+    # what we're checking against is a file path, not a URL.
+    if (
+        not args.clobber
+        and pathlib.Path(pudl_settings["pudl_db"].replace("sqlite:///", "").exists())
+    ):
         raise FileExistsError(
             f"SQLite DB at {pudl_settings['pudl_db']} exists and clobber is False.")
 

src/pudl/output/ferc714.py

@@ -531,7 +531,7 @@ def georef_counties(self, update=False):
         """
         if update or self._counties_gdf is None:
             census_counties = pudl.analysis.service_territory.get_census2010_gdf(
-                pudl_settings=self.pudl_settings, layer="county")
+                pudl_settings=self.pudl_settings, layer="county", ds=self.pudl_out.ds)
             self._counties_gdf = (
                 pudl.analysis.service_territory.add_geometries(
                     self.fipsify(update=update), census_gdf=census_counties)

src/pudl/output/pudltabl.py

@@ -28,7 +28,7 @@
 """
 
 import logging
-import pathlib
+from pathlib import Path
 
 # Useful high-level external modules.
 import pandas as pd
@@ -48,9 +48,17 @@
 class PudlTabl(object):
     """A class for compiling common useful tabular outputs from the PUDL DB."""
 
-    def __init__(self, pudl_engine, ds=None, freq=None, start_date=None,
-                 end_date=None, fill_fuel_cost=False, roll_fuel_cost=False,
-                 fill_net_gen=False):
+    def __init__(
+        self,
+        pudl_engine,
+        ds=None,
+        freq=None,
+        start_date=None,
+        end_date=None,
+        fill_fuel_cost=False,
+        roll_fuel_cost=False,
+        fill_net_gen=False
+    ):
         """
         Initialize the PUDL output object.
 
@@ -86,6 +94,11 @@ def __init__(self, pudl_engine, ds=None, freq=None, start_date=None,
         # We need datastore access because some data is not yet integrated into the
         # PUDL DB. See the etl_eia861 method.
         self.ds = ds
+        if self.ds is None:
+            pudl_in = Path(pudl.workspace.setup.get_defaults()["pudl_in"])
+            self.ds = pudl.workspace.datastore.Datastore(
+                local_cache_path=pudl_in / "data"
+            )
 
         # grab all working eia dates to use to set start and end dates if they
         # are not set
@@ -226,15 +239,7 @@ def etl_eia861(self, update=False):
         """
         if update or self._dfs["balancing_authority_eia861"] is None:
             logger.warning(
-                "Running the interim EIA 861 ETL process! (~2 minutes)")
-
-            if self.ds is None:
-                pudl_in = pathlib.Path(
-                    pudl.workspace.setup.get_defaults()["pudl_in"])
-                self.ds = pudl.workspace.datastore.Datastore(
-                    pudl_in=pudl_in,
-                    sandbox=False,
-                )
+                "Running the interim EIA 861 ETL process!")
 
             eia861_raw_dfs = (
                 pudl.extract.eia861.Extractor(self.ds)
@@ -362,8 +367,9 @@ def etl_ferc714(self, update=False):
         """
         if update or self._dfs["respondent_id_ferc714"] is None:
             logger.warning(
-                "Running the interim FERC 714 ETL process! (~11 minutes)")
-            ferc714_raw_dfs = pudl.extract.ferc714.extract()
+                "Running the interim FERC 714 ETL process!")
+
+            ferc714_raw_dfs = pudl.extract.ferc714.extract(ds=self.ds)
             ferc714_tfr_dfs = pudl.transform.ferc714.transform(ferc714_raw_dfs)
             for table in ferc714_tfr_dfs:
                 self._dfs[table] = ferc714_tfr_dfs[table]

test/fast_output_test.py

@@ -9,11 +9,16 @@
 
 
 @pytest.fixture(scope="module")
-def fast_out(pudl_engine):
+def fast_out(pudl_engine, pudl_datastore_fixture):
     """A PUDL output object for use with Travis CI."""
     return pudl.output.pudltabl.PudlTabl(
-        pudl_engine, freq="MS", fill_fuel_cost=True,
-        roll_fuel_cost=True, fill_net_gen=True)
+        pudl_engine,
+        ds=pudl_datastore_fixture,
+        freq="MS",
+        fill_fuel_cost=True,
+        roll_fuel_cost=True,
+        fill_net_gen=True
+    )
 
 
 def test_fuel_ferc1(fast_out):
@@ -63,3 +68,26 @@ def test_mcoe(fast_out):
     logger.info("Calculating MCOE.")
     mcoe_df = fast_out.mcoe()
     logger.info(f"Generated {len(mcoe_df)} MCOE records.")
+
+
+def test_eia861_etl(fast_out):
+    """Make sure that the EIA 861 Extract-Transform steps work."""
+    fast_out.etl_eia861()
+
+
+def test_ferc714_etl(fast_out):
+    """Make sure that the FERC 714 Extract-Transform steps work."""
+    fast_out.etl_ferc714()
+
+
+def test_ferc714_respondents(fast_out, pudl_settings_fixture):
+    """Test the FERC 714 Respondent & Service Territory outputs."""
+    ferc714_out = pudl.output.ferc714.Respondents(
+        fast_out,
+        pudl_settings=pudl_settings_fixture,
+    )
+    _ = ferc714_out.annualize()
+    _ = ferc714_out.categorize()
+    _ = ferc714_out.summarize_demand()
+    _ = ferc714_out.fipsify()
+    _ = ferc714_out.georef_counties()

test/glue_test.py

@@ -125,7 +125,7 @@ def test_unmapped_plants_eia(pudl_settings_fixture, pudl_engine):
         )
 
 
-def test_unmapped_utils_eia(pudl_settings_fixture, pudl_engine):
+def test_unmapped_utils_eia(pudl_settings_fixture, pudl_datastore_fixture, pudl_engine):
     """
     Check for unmapped EIA Utilities.
 
@@ -137,7 +137,11 @@ def test_unmapped_utils_eia(pudl_settings_fixture, pudl_engine):
     IDs.
 
     """
-    pudl_raw = pudl.output.pudltabl.PudlTabl(pudl_engine, freq=None)
+    pudl_raw = pudl.output.pudltabl.PudlTabl(
+        pudl_engine,
+        ds=pudl_datastore_fixture,
+        freq=None
+    )
     frc_eia923 = pudl_raw.frc_eia923()
     gf_eia923 = pudl_raw.gf_eia923()
     gen_eia923 = pudl_raw.gen_eia923()

test/interim_etl_test.py

@@ -7,24 +7,6 @@
 logger = logging.getLogger(__name__)
 
 
-def test_interim_eia861_etl(pudl_settings_fixture, pudl_datastore_fixture):
-    """Make sure that the EIA 861 Extract-Transform steps work."""
-    logger.info("Running the interim EIA 861 ETL process! (~2 minutes)")
-    _ = pudl.transform.eia861.transform(
-        pudl.extract.eia861.Extractor(pudl_datastore_fixture)
-        .extract(year=pudl.constants.working_partitions['eia861']['years'])
-    )
-
-
-def test_interim_ferc714_etl(pudl_settings_fixture, pudl_datastore_fixture):
-    """Make sure that the EIA 861 Extract-Transform steps work."""
-    logger.info("Running the interim FERC 714 ETL process!")
-    _ = pudl.transform.ferc714.transform(
-        pudl.extract.ferc714.extract(
-            pudl_settings=pudl_settings_fixture,
-            ds=pudl_datastore_fixture))
-
-
 def test_interim_get_census2010_gdf(pudl_settings_fixture, pudl_datastore_fixture):
     """Make sure that service_territory.get_census."""
     logger.info("Running pudl.analysis.service_territory.get_census2010_gdf.")