Apply best practices and Ruff

.pre-commit-config.yaml

@@ -7,15 +7,18 @@ repos:
     rev: v4.6.0
     hooks:
     -   id: check-yaml
--   repo: https://github.com/psf/black
-    rev: 24.4.2
-    hooks:
-    -   id: black
--   repo: https://github.com/PyCQA/isort
-    rev: 5.13.2
+
+# Run ruff to lint and format
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.8
     hooks:
-      - id: isort
-        args: ["--profile", "black", "--filter-files"]
+    # Run the linter.
+    -   id: ruff
+        args: [--fix]
+    # Run the formatter.
+    -   id: ruff-format
+
+# Run codespell to check spelling
 -   repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:

README.md

@@ -2,6 +2,7 @@
 
  [![pypi](https://img.shields.io/pypi/v/powerplantmatching.svg)](https://pypi.org/project/powerplantmatching/) [![conda](https://img.shields.io/conda/vn/conda-forge/powerplantmatching.svg)](https://anaconda.org/conda-forge/powerplantmatching) ![pythonversion](https://img.shields.io/pypi/pyversions/powerplantmatching) ![LICENSE](https://img.shields.io/pypi/l/powerplantmatching.svg) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3358985.svg)](https://zenodo.org/record/3358985#.XUReFPxS_MU) [![doc](https://readthedocs.org/projects/powerplantmatching/badge/?version=latest)](https://powerplantmatching.readthedocs.io/en/latest/) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/FRESNA/powerplantmatching/master.svg)](https://results.pre-commit.ci/latest/github/FRESNA/powerplantmatching/master)
 [![Stack Exchange questions](https://img.shields.io/stackexchange/stackoverflow/t/pypsa)](https://stackoverflow.com/questions/tagged/pypsa)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 
 A toolset for cleaning, standardizing and combining multiple power
 plant databases.

analysis/compare-with-entsoe-stats.py

@@ -36,8 +36,8 @@
 def parse(c):
     try:
         return client.query_installed_generation_capacity(c, **kwargs).iloc[0]
-    except:
-        print(f"Country {c} failed")
+    except Exception as e:
+        print(f"Country {c} failed with {e}")
         return np.nan
 
 

analysis/danish-powerplants.py

@@ -1,15 +1,9 @@
 import copy
 
-import cartopy.crs as ccrs
 import hvplot
 import hvplot.pandas  # noqa
 import hvplot.xarray  # noqa
-import pandas as pd
 import panel as pn
-import xarray as xr
-from holoviews import opts
-from holoviews.plotting.util import process_cmap
-from xarray import align
 
 import powerplantmatching as pm
 

analysis/german-powerplants.py

@@ -1,15 +1,9 @@
 import copy
 
-import cartopy.crs as ccrs
 import hvplot
 import hvplot.pandas  # noqa
 import hvplot.xarray  # noqa
-import pandas as pd
 import panel as pn
-import xarray as xr
-from holoviews import opts
-from holoviews.plotting.util import process_cmap
-from xarray import align
 
 import powerplantmatching as pm
 

analysis/interactive-map-with-widgets.py

@@ -1,10 +1,7 @@
 import hvplot
 import hvplot.pandas  # noqa
 import hvplot.xarray  # noqa
-import pandas as pd
 import panel as pn
-import xarray as xr
-from xarray import align
 
 import powerplantmatching as pm
 

analysis/interactive-map.py

@@ -1,10 +1,6 @@
 import hvplot
 import hvplot.pandas  # noqa
 import hvplot.xarray  # noqa
-import pandas as pd
-import panel as pn
-import xarray as xr
-from xarray import align
 
 import powerplantmatching as pm
 

doc/example.ipynb

@@ -22,8 +22,9 @@
     }
    ],
    "source": [
-    "import powerplantmatching as pm\n",
-    "import pandas as pd"
+    "import pandas as pd\n",
+    "\n",
+    "import powerplantmatching as pm"
    ]
   },
   {
@@ -844,8 +845,8 @@
     }
    ],
    "source": [
-    "print('Total capacity of GEO is: \\n  {} MW \\n'.format(geo.Capacity.sum()));\n",
-    "print('The technology types are: \\n {} '.format(geo.Technology.unique()))"
+    "print(f\"Total capacity of GEO is: \\n  {geo.Capacity.sum()} MW \\n\")\n",
+    "print(f\"The technology types are: \\n {geo.Technology.unique()} \")"
    ]
   },
   {
@@ -888,7 +889,7 @@
     }
    ],
    "source": [
-    "pm.plot.fueltype_totals_bar([geo, entsoe, stats], keys=[\"ENTSOE\", \"GEO\", 'Statistics']);"
+    "pm.plot.fueltype_totals_bar([geo, entsoe, stats], keys=[\"ENTSOE\", \"GEO\", \"Statistics\"]);"
    ]
   },
   {
@@ -1414,7 +1415,7 @@
     }
    ],
    "source": [
-    "pm.plot.fueltype_totals_bar([intersection, stats], keys=[\"Intersection\", 'Statistics']);"
+    "pm.plot.fueltype_totals_bar([intersection, stats], keys=[\"Intersection\", \"Statistics\"]);"
    ]
   },
   {
@@ -1432,7 +1433,9 @@
     }
    ],
    "source": [
-    "combined = intersection.powerplant.extend_by_non_matched(entsoe).powerplant.extend_by_non_matched(geo)"
+    "combined = intersection.powerplant.extend_by_non_matched(\n",
+    "    entsoe\n",
+    ").powerplant.extend_by_non_matched(geo)"
    ]
   },
   {
@@ -1452,7 +1455,7 @@
     }
    ],
    "source": [
-    "pm.plot.fueltype_totals_bar([combined, stats], keys=[\"Combined\", 'Statistics']);"
+    "pm.plot.fueltype_totals_bar([combined, stats], keys=[\"Combined\", \"Statistics\"]);"
    ]
   },
   {
@@ -1511,7 +1514,7 @@
     }
    ],
    "source": [
-    "m.powerplant.plot_map(figsize=(11,8));"
+    "m.powerplant.plot_map(figsize=(11, 8));"
    ]
   },
   {
@@ -1531,7 +1534,7 @@
     }
    ],
    "source": [
-    "pm.plot.fueltype_totals_bar([m, stats], keys=[\"Processed\", 'Statistics']);"
+    "pm.plot.fueltype_totals_bar([m, stats], keys=[\"Processed\", \"Statistics\"]);"
    ]
   },
   {
@@ -1569,7 +1572,7 @@
     }
    ],
    "source": [
-    "pm.plot.factor_comparison([m, stats], keys=['Processed', 'Statistics'])"
+    "pm.plot.factor_comparison([m, stats], keys=[\"Processed\", \"Statistics\"])"
    ]
   },
   {
@@ -1911,9 +1914,14 @@
     }
    ],
    "source": [
-    "pd.concat([m[m.DateIn.notnull()].groupby('Fueltype').DateIn.count(),\n",
-    "          m[m.DateIn.isna()].fillna(1).groupby('Fueltype').DateIn.count()], \n",
-    "          keys=['DateIn existent', 'DateIn missing'], axis=1)"
+    "pd.concat(\n",
+    "    [\n",
+    "        m[m.DateIn.notnull()].groupby(\"Fueltype\").DateIn.count(),\n",
+    "        m[m.DateIn.isna()].fillna(1).groupby(\"Fueltype\").DateIn.count(),\n",
+    "    ],\n",
+    "    keys=[\"DateIn existent\", \"DateIn missing\"],\n",
+    "    axis=1,\n",
+    ")"
    ]
   }
  ],

doc/release-notes.rst

@@ -13,7 +13,7 @@ Version 0.5.15 (12.05.2024)
 
 * Corrected capacity of some hydro powerplants in the United Kingdom contained
   in the ENTSOE data based on manual search. The error originated from the use
-  of the value 999 in the capacity column to represent NaN values. The chnages
+  of the value 999 in the capacity column to represent NaN values. The changes
   accumulate to a reduction of approximately 5 GW in capacity.
   (https://github.com/PyPSA/powerplantmatching/pull/141)
 

powerplantmatching/__init__.py

@@ -21,21 +21,25 @@
 power plant databases.
 """
 
-
-from __future__ import absolute_import
-
 __version__ = "0.5.15"
 __author__ = "Fabian Hofmann"
 __copyright__ = "Copyright 2017-2024 Technical University of Berlin"
 # The rough hierarchy of this package is
 # core, utils, heuristics, cleaning, matching, collection, data
 
-# from . import cleaning
-# from . import matching
-# from . import collection
-# Commonly used sub-modules. Imported here to provide end-user
-# convenience.
 from . import core, data, heuristics, plot, utils
 from .accessor import PowerPlantAccessor
 from .collection import powerplants
 from .core import get_config, package_config
+
+__all__ = [
+    "powerplants",
+    "get_config",
+    "package_config",
+    "PowerPlantAccessor",
+    "core",
+    "data",
+    "heuristics",
+    "plot",
+    "utils",
+]

powerplantmatching/accessor.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 Created on Tue Jul 16 18:09:56 2019
 

powerplantmatching/cleaning.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016-2018 Fabian Hofmann (FIAS), Jonas Hoersch (KIT, IAI) and
 # Fabian Gotzens (FZJ, IEK-STE)
 
@@ -17,11 +16,8 @@
 """
 Functions for vertically cleaning a dataset.
 """
-from __future__ import absolute_import, print_function
 
 import logging
-import os
-import re
 
 import networkx as nx
 import numpy as np
@@ -160,14 +156,13 @@ def gather_and_replace(df, mapping):
     for key, pattern in mapping.items():
         if not pattern:
             # if pattern is not given, fall back to case-insensitive key
-            pattern = r"(?i)\b%s\b" % key
+            pattern = rf"(?i)\b{key}\b"
         elif isinstance(pattern, list):
             # if pattern is a list, concat all entries in a case-insensitive regex
             pattern = r"(?i)" + "|".join([rf"\b{p}\b" for p in pattern])
         elif not isinstance(pattern, str):
             raise ValueError(f"Pattern must be string or list, not {type(pattern)}")
-        func = lambda ds: ds.str.contains(pattern)
-        where = df.astype(str).apply(func).any(axis=1)
+        where = df.astype(str).apply(lambda ds: ds.str.contains(pattern)).any(axis=1)
         res = res.where(~where, key)
     return res
 

powerplantmatching/collection.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016-2018 Fabian Hofmann (FIAS), Jonas Hoersch (KIT, IAI) and
 # Fabian Gotzens (FZJ, IEK-STE)
 
@@ -17,6 +16,7 @@
 """
 Processed datasets of merged and/or adjusted data
 """
+
 import logging
 import os
 
@@ -31,7 +31,6 @@
     parmap,
     projectID_to_dict,
     set_column_name,
-    set_uncommon_fueltypes_to_other,
     to_dict_if_string,
 )
 

powerplantmatching/core.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 """
 Created on Tue Jul 16 15:47:46 2019
 

powerplantmatching/data.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2016-2020 Fabian Hofmann (FIAS), Jonas Hoersch (KIT, IAI) and
 # Fabian Gotzens (FZJ, IEK-STE)
 
@@ -20,8 +19,6 @@
 
 import logging
 import os
-import re
-import xml.etree.ElementTree as ET
 from zipfile import ZipFile
 
 import entsoe
@@ -38,13 +35,12 @@
     gather_specifications,
     gather_technology_info,
 )
-from .core import _data_in, _package_data, get_config
+from .core import _package_data, get_config
 from .heuristics import scale_to_net_capacities
 from .utils import (
     config_filter,
     convert_to_short_name,
     correct_manually,
-    fill_geoposition,
     get_raw_file,
     set_column_name,
 )
@@ -266,7 +262,7 @@ def GEO(raw=False, update=False, config=None):
 
     def to_year(ds):
         years = pd.to_numeric(ds.dropna().astype(str).str[:4], errors="coerce")
-        year = years[lambda x: x > 1900]
+        year = years[lambda x: x > 1900]  # noqa
         return years.reindex_like(ds)
 
     fn = get_raw_file("GEO_units", update=update, config=config)
@@ -527,7 +523,7 @@ def Capacity_stats(
     else:
         source = "Capacity statistics"
 
-    fueltypes = config["target_fueltypes"]
+    fueltypes = config["target_fueltypes"]  # noqa
     df = (
         df.query("year == @year")
         .rename(columns={"technology": "Fueltype"})
@@ -579,8 +575,8 @@ def GPD(raw=False, update=False, config=None, filter_other_dbs=True):
 
     other_dbs = []
     if filter_other_dbs:
-        other_dbs = ["GEODB", "Open Power System Data", "ENTSOE"]
-    countries = config["target_countries"]
+        other_dbs = ["GEODB", "Open Power System Data", "ENTSOE"]  # noqa
+    countries = config["target_countries"]  # noqa
     return (
         df.rename(columns=lambda x: x.title())
         .drop(columns="Country")
@@ -1203,7 +1199,7 @@ def UBA(
         .str[0]
         .astype(float),
         Country="Germany",
-        projectID=["UBA{:03d}".format(i + header + 2) for i in uba.index],
+        projectID=[f"UBA{i + header + 2:03d}" for i in uba.index],
         Technology=uba.Technology.replace(RENAME_TECHNOLOGY),
     )
     uba.loc[uba.CHP.notnull(), "Set"] = "CHP"
@@ -1580,8 +1576,8 @@ def IRENASTAT(raw=False, update=False, config=None):
     df["Fueltype"] = df.Technology.map(fueltype_dict)
     df["Technology"] = df.Technology.replace(technology_dict)
 
-    l = list(set(df.columns).difference(set(["Capacity"])))
-    df = df.groupby(l, as_index=False, dropna=True).sum()
+    non_capacity_columns = list(set(df.columns).difference(set(["Capacity"])))
+    df = df.groupby(non_capacity_columns, as_index=False, dropna=True).sum()
 
     return df