Bugfixes for states=[ALL] and SQLite DB clobber check

src/pudl/constants.py

@@ -1521,7 +1521,7 @@
     },
     'epacems': {
         'years': tuple(range(1995, 2020)),
-        'states': cems_states.keys()},
+        'states': tuple(cems_states.keys())},
     'ferc1': {
         'years': tuple(range(1994, 2020))
     },
@@ -1968,7 +1968,7 @@
     'other'
 ]
 
-ESTIMATED_OR_ACTUAL = {'E': 'Estimated', 'A': 'Actual'}
+ESTIMATED_OR_ACTUAL = {'E': 'estimated', 'A': 'actual'}
 
 TRANSIT_TYPE_DICT = {
     'CV': 'conveyer',

src/pudl/convert/datapkg_to_sqlite.py

@@ -151,7 +151,10 @@ def main():
     logger.info(f"pudl_out={pudl_settings['pudl_out']}")
 
     # Check if there's already a PUDL SQLite DB that we should not clobber:
-    if not args.clobber and pathlib.Path(pudl_settings["pudl_db"]).exists():
+    if (
+        not args.clobber
+        and pathlib.Path(pudl_settings["pudl_db"].replace("sqlite:///", "").exists())
+    ):
         raise FileExistsError(
             f"SQLite DB at {pudl_settings['pudl_db']} exists and clobber is False.")
 

src/pudl/output/ferc714.py

@@ -1,4 +1,5 @@
 """Functions & classes for compiling derived aspects of the FERC Form 714 data."""
+import pathlib
 from functools import cached_property
 from typing import Any, Dict, List
 
@@ -154,6 +155,12 @@ def __init__(
     ):
         """Set respondent compilation parameters."""
         self.pudl_out = pudl_out
+        if self.pudl_out.ds is None:
+            pudl_in = pathlib.Path(
+                pudl.workspace.setup.get_defaults()["pudl_in"])
+            self.pudl_out.ds = pudl.workspace.datastore.Datastore(
+                local_cache_path=pathlib.Path(pudl_in, "data")
+            )
 
         if pudl_settings is None:
             pudl_settings = pudl.workspace.setup.get_defaults()
@@ -531,7 +538,7 @@ def georef_counties(self, update=False):
         """
         if update or self._counties_gdf is None:
             census_counties = pudl.analysis.service_territory.get_census2010_gdf(
-                pudl_settings=self.pudl_settings, layer="county")
+                pudl_settings=self.pudl_settings, layer="county", ds=self.pudl_out.ds)
             self._counties_gdf = (
                 pudl.analysis.service_territory.add_geometries(
                     self.fipsify(update=update), census_gdf=census_counties)

src/pudl/output/pudltabl.py

@@ -226,14 +226,13 @@ def etl_eia861(self, update=False):
         """
         if update or self._dfs["balancing_authority_eia861"] is None:
             logger.warning(
-                "Running the interim EIA 861 ETL process! (~2 minutes)")
+                "Running the interim EIA 861 ETL process!")
 
             if self.ds is None:
                 pudl_in = pathlib.Path(
                     pudl.workspace.setup.get_defaults()["pudl_in"])
                 self.ds = pudl.workspace.datastore.Datastore(
-                    pudl_in=pudl_in,
-                    sandbox=False,
+                    local_cache_path=pathlib.Path(pudl_in, "data")
                 )
 
             eia861_raw_dfs = (
@@ -362,8 +361,15 @@ def etl_ferc714(self, update=False):
         """
         if update or self._dfs["respondent_id_ferc714"] is None:
             logger.warning(
-                "Running the interim FERC 714 ETL process! (~11 minutes)")
-            ferc714_raw_dfs = pudl.extract.ferc714.extract()
+                "Running the interim FERC 714 ETL process!")
+
+            if self.ds is None:
+                pudl_in = pathlib.Path(
+                    pudl.workspace.setup.get_defaults()["pudl_in"])
+                self.ds = pudl.workspace.datastore.Datastore(
+                    local_cache_path=pathlib.Path(pudl_in, "data")
+                )
+            ferc714_raw_dfs = pudl.extract.ferc714.extract(ds=self.ds)
             ferc714_tfr_dfs = pudl.transform.ferc714.transform(ferc714_raw_dfs)
             for table in ferc714_tfr_dfs:
                 self._dfs[table] = ferc714_tfr_dfs[table]

test/fast_output_test.py

@@ -9,11 +9,16 @@
 
 
 @pytest.fixture(scope="module")
-def fast_out(pudl_engine):
+def fast_out(pudl_engine, pudl_datastore_fixture):
     """A PUDL output object for use with Travis CI."""
     return pudl.output.pudltabl.PudlTabl(
-        pudl_engine, freq="MS", fill_fuel_cost=True,
-        roll_fuel_cost=True, fill_net_gen=True)
+        pudl_engine,
+        ds=pudl_datastore_fixture,
+        freq="MS",
+        fill_fuel_cost=True,
+        roll_fuel_cost=True,
+        fill_net_gen=True
+    )
 
 
 def test_fuel_ferc1(fast_out):
@@ -63,3 +68,26 @@ def test_mcoe(fast_out):
     logger.info("Calculating MCOE.")
     mcoe_df = fast_out.mcoe()
     logger.info(f"Generated {len(mcoe_df)} MCOE records.")
+
+
+def test_eia861_etl(fast_out):
+    """Make sure that the EIA 861 Extract-Transform steps work."""
+    fast_out.etl_eia861()
+
+
+def test_ferc714_etl(fast_out):
+    """Make sure that the FERC 714 Extract-Transform steps work."""
+    fast_out.etl_ferc714()
+
+
+def test_ferc714_respondents(fast_out, pudl_settings_fixture):
+    """Test the FERC 714 Respondent & Service Territory outputs."""
+    ferc714_out = pudl.output.ferc714.Respondents(
+        fast_out,
+        pudl_settings=pudl_settings_fixture,
+    )
+    _ = ferc714_out.annualize()
+    _ = ferc714_out.categorize()
+    _ = ferc714_out.summarize_demand()
+    _ = ferc714_out.fipsify()
+    _ = ferc714_out.georef_counties()

test/interim_etl_test.py

@@ -7,24 +7,6 @@
 logger = logging.getLogger(__name__)
 
 
-def test_interim_eia861_etl(pudl_settings_fixture, pudl_datastore_fixture):
-    """Make sure that the EIA 861 Extract-Transform steps work."""
-    logger.info("Running the interim EIA 861 ETL process! (~2 minutes)")
-    _ = pudl.transform.eia861.transform(
-        pudl.extract.eia861.Extractor(pudl_datastore_fixture)
-        .extract(year=pudl.constants.working_partitions['eia861']['years'])
-    )
-
-
-def test_interim_ferc714_etl(pudl_settings_fixture, pudl_datastore_fixture):
-    """Make sure that the EIA 861 Extract-Transform steps work."""
-    logger.info("Running the interim FERC 714 ETL process!")
-    _ = pudl.transform.ferc714.transform(
-        pudl.extract.ferc714.extract(
-            pudl_settings=pudl_settings_fixture,
-            ds=pudl_datastore_fixture))
-
-
 def test_interim_get_census2010_gdf(pudl_settings_fixture, pudl_datastore_fixture):
     """Make sure that service_territory.get_census."""
     logger.info("Running pudl.analysis.service_territory.get_census2010_gdf.")