From 883cfbea1dd66024f5ed73f4b0c60fe82468d72b Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Thu, 18 Jul 2024 19:10:50 -0700 Subject: [PATCH 01/22] coerce and rotate pvgis TMY data to desired tz and year - add private function `_coerce_and_rotate_pvgis()` - add `utc_offset` and `coerce_year` params to docstring for `get_pvgis_tmy` - call private function if `utc_offset` is not zero --- pvlib/iotools/pvgis.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 06986bf2e5..b4499a9731 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -390,9 +390,25 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): raise ValueError(err_msg) +def _coerce_and_rotate_pvgis(pvgis_data, tz, year): + """ + After converting TZ, rotate indices so timeseries starts at midnight + and force all indices to a common year. Only works for integer TZ. + """ + # assert tz//1 == tz + tzname = f'Etc/GMT{-tz:+d}' + pvgis_data = pd.concat( + [pvgis_data.iloc[-tz:], pvgis_data.iloc[:-tz]], + axis=0).tz_convert(tzname) + pvgis_data.index = [ + timestamp.replace(year=year) for timestamp in pvgis_data.index] + return pvgis_data + + def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, - map_variables=True, url=URL, timeout=30): + map_variables=True, url=URL, timeout=30, + utc_offset=0, coerce_year=1990): """ Get TMY data from PVGIS. @@ -424,6 +440,12 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, base url of PVGIS API, append ``tmy`` to get TMY endpoint timeout : int, default 30 time in seconds to wait for server response before timeout + utc_offset: int, default 0 + Use to specify a timezone other than the default UTC zero. Will force + year to ``coerce_year`` if not zero. + coerce_year: int, default 1990 + Use to force indices to desired year. Ignored if ``utc_offset`` is + zero. Returns ------- @@ -510,6 +532,9 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, if map_variables: data = data.rename(columns=VARIABLE_MAP) + if utc_offset != 0: + data = _coerce_and_rotate_pvgis(data, utc_offset, coerce_year) + return data, months_selected, inputs, meta From 7e6e4c0e0e0aa4f4aecc40ad1f55db590c30c634 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 00:25:07 -0700 Subject: [PATCH 02/22] test get_pvgis_tmy_coerce_year check if utc_offset and coerce_year work as expected --- pvlib/tests/iotools/test_pvgis.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 45c4cde46b..55573cc56d 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -435,6 +435,18 @@ def _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data): assert np.allclose(data[outvar], expected[outvar]) +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_pvgis_tmy_coerce_year(): + """test utc_offset and coerce_year work as expected""" + pvgis_data = get_pvgis_tmy(45, 8, utc_offset=+2) # Turin + assert pvgis_data[0].index.iloc[0] == pd.TimeStamp('1990-01-01 00:00:00', tz='Etc/GMT-2') + assert pvgis_data[0].index.iloc[-1] == pd.TimeStamp('1990-12-31 23:00:00', tz='Etc/GMT-2') + pvgis_data = get_pvgis_tmy(45, 8, utc_offset=+2, coerce_year=2021) # Turin + assert pvgis_data[0].index.iloc[0] == pd.TimeStamp('2021-01-01 00:00:00', tz='Etc/GMT-2') + assert pvgis_data[0].index.iloc[-1] == pd.TimeStamp('2021-12-31 23:00:00', tz='Etc/GMT-2') + + @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_pvgis_tmy_csv(expected, month_year_expected, inputs_expected, From 84d4695fe8561214d1eff9b944db220ec6981072 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 00:33:43 -0700 Subject: [PATCH 03/22] fix flake8 in test_pvgis_coerce_year - remove whitespace - shorter lines --- pvlib/tests/iotools/test_pvgis.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 55573cc56d..deb5988f29 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -438,13 +438,17 @@ def _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data): @pytest.mark.remote_data @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_pvgis_tmy_coerce_year(): - """test utc_offset and coerce_year work as expected""" - pvgis_data = get_pvgis_tmy(45, 8, utc_offset=+2) # Turin - assert pvgis_data[0].index.iloc[0] == pd.TimeStamp('1990-01-01 00:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index.iloc[-1] == pd.TimeStamp('1990-12-31 23:00:00', tz='Etc/GMT-2') - pvgis_data = get_pvgis_tmy(45, 8, utc_offset=+2, coerce_year=2021) # Turin - assert pvgis_data[0].index.iloc[0] == pd.TimeStamp('2021-01-01 00:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index.iloc[-1] == pd.TimeStamp('2021-12-31 23:00:00', tz='Etc/GMT-2') + """test utc_offset and coerce_year work as expected""" + pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2) # Turin + jan1_midnight1990 = pd.TimeStamp('1990-01-01 00:00:00', tz='Etc/GMT-2') + dec31_midnight1990 = pd.TimeStamp('1990-12-31 23:00:00', tz='Etc/GMT-2') + assert pvgis_data[0].index.iloc[0] == jan1_midnight1990 + assert pvgis_data[0].index.iloc[-1] == dec31_midnight1990 + pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2, coerce_year=2021) # Turin + jan1_midnight2021 = pd.TimeStamp('2021-01-01 00:00:00', tz='Etc/GMT-2') + dec31_midnight2021 = pd.TimeStamp('2021-12-31 23:00:00', tz='Etc/GMT-2') + assert pvgis_data[0].index.iloc[0] == jan1_midnight2021 + assert pvgis_data[0].index.iloc[-1] == dec31_midnight2021 @pytest.mark.remote_data From 3e4207594caf829b05b55e7e4a80fe5f7bab9ebf Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 01:01:33 -0700 Subject: [PATCH 04/22] remove iloc for index in test pvgis coerce - incorrect syntax for indices --- pvlib/tests/iotools/test_pvgis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index deb5988f29..0b217e2c70 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -442,13 +442,13 @@ def test_get_pvgis_tmy_coerce_year(): pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2) # Turin jan1_midnight1990 = pd.TimeStamp('1990-01-01 00:00:00', tz='Etc/GMT-2') dec31_midnight1990 = pd.TimeStamp('1990-12-31 23:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index.iloc[0] == jan1_midnight1990 - assert pvgis_data[0].index.iloc[-1] == dec31_midnight1990 + assert pvgis_data[0].index[0] == jan1_midnight1990 + assert pvgis_data[0].index[-1] == dec31_midnight1990 pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2, coerce_year=2021) # Turin jan1_midnight2021 = pd.TimeStamp('2021-01-01 00:00:00', tz='Etc/GMT-2') dec31_midnight2021 = pd.TimeStamp('2021-12-31 23:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index.iloc[0] == jan1_midnight2021 - assert pvgis_data[0].index.iloc[-1] == dec31_midnight2021 + assert pvgis_data[0].index[0] == jan1_midnight2021 + assert pvgis_data[0].index[-1] == dec31_midnight2021 @pytest.mark.remote_data From d0bd0a0c5863a814329e09abda1d9a32daff1ecb Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 01:27:38 -0700 Subject: [PATCH 05/22] deal with leap year in pvgis when coercing - if february is a leap year, when shifting tz, causes issues - so replace february year with non-leap year --- pvlib/iotools/pvgis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index b4499a9731..4b6021db0a 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -14,6 +14,7 @@ * `monthly radiation `_ """ +import calendar import io import json from pathlib import Path @@ -397,6 +398,13 @@ def _coerce_and_rotate_pvgis(pvgis_data, tz, year): """ # assert tz//1 == tz tzname = f'Etc/GMT{-tz:+d}' + # check if February is leap year + feb1 = pvgis_data[pvgis_data.index.month==2].index.year[0] + if calendar.isleap(feb1): + # replace Feb year with a non-leap year + pvgis_data.index = [ + timestamp.replace(year=1990) if timestamp.month==2 else ts + for timestamp in pvgis_data.index] pvgis_data = pd.concat( [pvgis_data.iloc[-tz:], pvgis_data.iloc[:-tz]], axis=0).tz_convert(tzname) From ec482ac7d5f37956f499d389b8e77b60e57c0965 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 01:31:54 -0700 Subject: [PATCH 06/22] fix space around operator in coerce pvgis - also fix use ts for timestamp when fixing feb leap year --- pvlib/iotools/pvgis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 4b6021db0a..112d1c599e 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -399,12 +399,12 @@ def _coerce_and_rotate_pvgis(pvgis_data, tz, year): # assert tz//1 == tz tzname = f'Etc/GMT{-tz:+d}' # check if February is leap year - feb1 = pvgis_data[pvgis_data.index.month==2].index.year[0] + feb1 = pvgis_data[pvgis_data.index.month == 2].index.year[0] if calendar.isleap(feb1): # replace Feb year with a non-leap year pvgis_data.index = [ - timestamp.replace(year=1990) if timestamp.month==2 else ts - for timestamp in pvgis_data.index] + ts.replace(year=1990) if ts.month == 2 else ts + for ts in pvgis_data.index] pvgis_data = pd.concat( [pvgis_data.iloc[-tz:], pvgis_data.iloc[:-tz]], axis=0).tz_convert(tzname) From 883ced8b6b857b2db171f64f3f8b920ded034357 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 01:45:57 -0700 Subject: [PATCH 07/22] fix pd.Timestamp in pvgis coerce year - lower case "s" not TimeStamp --- pvlib/tests/iotools/test_pvgis.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 0b217e2c70..0cc72ab833 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -440,13 +440,13 @@ def _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data): def test_get_pvgis_tmy_coerce_year(): """test utc_offset and coerce_year work as expected""" pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2) # Turin - jan1_midnight1990 = pd.TimeStamp('1990-01-01 00:00:00', tz='Etc/GMT-2') - dec31_midnight1990 = pd.TimeStamp('1990-12-31 23:00:00', tz='Etc/GMT-2') + jan1_midnight1990 = pd.Timestamp('1990-01-01 00:00:00', tz='Etc/GMT-2') + dec31_midnight1990 = pd.Timestamp('1990-12-31 23:00:00', tz='Etc/GMT-2') assert pvgis_data[0].index[0] == jan1_midnight1990 assert pvgis_data[0].index[-1] == dec31_midnight1990 pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2, coerce_year=2021) # Turin - jan1_midnight2021 = pd.TimeStamp('2021-01-01 00:00:00', tz='Etc/GMT-2') - dec31_midnight2021 = pd.TimeStamp('2021-12-31 23:00:00', tz='Etc/GMT-2') + jan1_midnight2021 = pd.Timestamp('2021-01-01 00:00:00', tz='Etc/GMT-2') + dec31_midnight2021 = pd.Timestamp('2021-12-31 23:00:00', tz='Etc/GMT-2') assert pvgis_data[0].index[0] == jan1_midnight2021 assert pvgis_data[0].index[-1] == dec31_midnight2021 From dd2c85f3a91673feb150a94b362849094c16f7ad Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Fri, 19 Jul 2024 18:29:10 -0700 Subject: [PATCH 08/22] Update v0.11.1 what's new for coerce pvgis tmy - add description and links to issue/pr --- docs/sphinx/source/whatsnew/v0.11.1.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.11.1.rst b/docs/sphinx/source/whatsnew/v0.11.1.rst index 4ccc9687c3..0148436154 100644 --- a/docs/sphinx/source/whatsnew/v0.11.1.rst +++ b/docs/sphinx/source/whatsnew/v0.11.1.rst @@ -16,6 +16,10 @@ Enhancements * Add new parameters for min/max absolute air mass to :py:func:`pvlib.spectrum.spectral_factor_firstsolar`. (:issue:`2086`, :pull:`2100`) +* Add ``utc_offset`` and ``coerce_year`` arguments to + :py:func:`pvlib.iotools.get_pvgis_tmy` to allow user to specify time zone, + rotate indices of TMY to begin at midnight, and force indices to desired + year. (:issue:`2139`, :pull:`2138`) Bug fixes @@ -45,4 +49,4 @@ Contributors * Leonardo Micheli (:ghuser:`lmicheli`) * Echedey Luis (:ghuser:`echedey-ls`) * Rajiv Daxini (:ghuser:`RDaxini`) - +* Mark A. Mikofski (:ghuser:`mikofski`) From 31800e922e9888a5d71b7a9ea0a4e550c9ea12b5 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 27 Jul 2024 02:46:15 -0700 Subject: [PATCH 09/22] replace year and tzinfo in pvgis_tmy coerce year - also use np.roll - also make new index and dataframe instead of altering original - removes need to sanitize original index February for leap year - remove calendar import but add numpy and pytz - code much simpler shorter, easier to read --- pvlib/iotools/pvgis.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 112d1c599e..0ccd0ca861 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -14,12 +14,13 @@ * `monthly radiation `_ """ -import calendar import io import json from pathlib import Path import requests +import numpy as np import pandas as pd +import pytz from pvlib.iotools import read_epw, parse_epw import warnings from pvlib._deprecation import pvlibDeprecationWarning @@ -396,21 +397,15 @@ def _coerce_and_rotate_pvgis(pvgis_data, tz, year): After converting TZ, rotate indices so timeseries starts at midnight and force all indices to a common year. Only works for integer TZ. """ - # assert tz//1 == tz - tzname = f'Etc/GMT{-tz:+d}' - # check if February is leap year - feb1 = pvgis_data[pvgis_data.index.month == 2].index.year[0] - if calendar.isleap(feb1): - # replace Feb year with a non-leap year - pvgis_data.index = [ - ts.replace(year=1990) if ts.month == 2 else ts - for ts in pvgis_data.index] - pvgis_data = pd.concat( - [pvgis_data.iloc[-tz:], pvgis_data.iloc[:-tz]], - axis=0).tz_convert(tzname) - pvgis_data.index = [ - timestamp.replace(year=year) for timestamp in pvgis_data.index] - return pvgis_data + tzname = pytz.timezone(f'Etc/GMT{-tz:+d}') + new_index = [ + timestamp.replace(year=year, tzinfo=tzname) + for timestamp in pvgis_data.index] + new_pvgis_data = pd.DataFrame( + np.roll(pvgis_data, tz, axis=0), + columns=pvgis_data.columns, + index=new_index) + return new_pvgis_data def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, From 72c55f67ef82e3bc0279b617ae2aabd47da64f18 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 27 Jul 2024 02:52:53 -0700 Subject: [PATCH 10/22] remove unused imports from pvgis.py for flake8 --- pvlib/iotools/pvgis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 0ccd0ca861..9bd7d8f5ac 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -22,8 +22,6 @@ import pandas as pd import pytz from pvlib.iotools import read_epw, parse_epw -import warnings -from pvlib._deprecation import pvlibDeprecationWarning URL = 'https://re.jrc.ec.europa.eu/api/' From 0735183384ed47a5f01d975d966701eb5fdcb7cd Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 27 Jul 2024 03:03:51 -0700 Subject: [PATCH 11/22] change private function name to _coerce_and_roll_pvgis_tmy --- pvlib/iotools/pvgis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 9bd7d8f5ac..6da4e15012 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -390,9 +390,9 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): raise ValueError(err_msg) -def _coerce_and_rotate_pvgis(pvgis_data, tz, year): +def _coerce_and_roll_pvgis_tmy(pvgis_data, tz, year): """ - After converting TZ, rotate indices so timeseries starts at midnight + After converting TZ, roll dataframe so timeseries starts at midnight and force all indices to a common year. Only works for integer TZ. """ tzname = pytz.timezone(f'Etc/GMT{-tz:+d}') @@ -534,7 +534,7 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, data = data.rename(columns=VARIABLE_MAP) if utc_offset != 0: - data = _coerce_and_rotate_pvgis(data, utc_offset, coerce_year) + data = _coerce_and_roll_pvgis_tmy(data, utc_offset, coerce_year) return data, months_selected, inputs, meta From 2059dc66fda30f9ec341dd9d529f217c58b21658 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Sat, 27 Jul 2024 13:47:42 -0700 Subject: [PATCH 12/22] spot check rolled pvgis TMY values after converting tz - fix Turin is actually CET which is UTC+1 - be DRY so use variables for test year and tz constants, versus WET and hardcoded - check tz unchanged if default zero utc_offset - use _ output args instead of indexing data[0] - add comments --- pvlib/tests/iotools/test_pvgis.py | 38 +++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 0cc72ab833..58def17c98 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -439,16 +439,34 @@ def _compare_pvgis_tmy_basic(expected, meta_expected, pvgis_data): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_pvgis_tmy_coerce_year(): """test utc_offset and coerce_year work as expected""" - pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2) # Turin - jan1_midnight1990 = pd.Timestamp('1990-01-01 00:00:00', tz='Etc/GMT-2') - dec31_midnight1990 = pd.Timestamp('1990-12-31 23:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index[0] == jan1_midnight1990 - assert pvgis_data[0].index[-1] == dec31_midnight1990 - pvgis_data = get_pvgis_tmy(45, 8, utc_offset=2, coerce_year=2021) # Turin - jan1_midnight2021 = pd.Timestamp('2021-01-01 00:00:00', tz='Etc/GMT-2') - dec31_midnight2021 = pd.Timestamp('2021-12-31 23:00:00', tz='Etc/GMT-2') - assert pvgis_data[0].index[0] == jan1_midnight2021 - assert pvgis_data[0].index[-1] == dec31_midnight2021 + base_case, _, _, _ = get_pvgis_tmy(45, 8) # Turin + assert str(base_case.index.tz) == 'UTC' + noon_test_data = [ + base_case[base_case.index.month == m].iloc[12] + for m in range(1, 13)] + cet_tz = 1 # Turin time is CET + cet_name = 'Etc/GMT-1' + # check indices of rolled data after converting timezone + pvgis_data, _, _, _ = get_pvgis_tmy(45, 8, utc_offset=cet_tz) + jan1_midnight = pd.Timestamp('1990-01-01 00:00:00', tz=cet_name) + dec31_midnight = pd.Timestamp('1990-12-31 23:00:00', tz=cet_name) + assert pvgis_data.index[0] == jan1_midnight + assert pvgis_data.index[-1] == dec31_midnight + # spot check rolled data matches original + for m, test_case in enumerate(noon_test_data): + expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12+cet_tz] + assert all(test_case == expected) + # repeat tests with year coerced + test_yr = 2021 + pvgis_data, _, _, _ = get_pvgis_tmy( + 45, 8, utc_offset=cet_tz, coerce_year=test_yr) + jan1_midnight = pd.Timestamp(f'{test_yr}-01-01 00:00:00', tz=cet_name) + dec31_midnight = pd.Timestamp(f'{test_yr}-12-31 23:00:00', tz=cet_name) + assert pvgis_data.index[0] == jan1_midnight + assert pvgis_data.index[-1] == dec31_midnight + for m, test_case in enumerate(noon_test_data): + expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12+cet_tz] + assert all(test_case == expected) @pytest.mark.remote_data From a38cf9aea21d5fdbb69fa84df6a2bd5a0a367c5f Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 31 Jul 2024 00:06:04 -0700 Subject: [PATCH 13/22] Update utc_offset description - explain setting utc_offset will roll data to start at Jan 1 midnight --- pvlib/iotools/pvgis.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 6da4e15012..ebab8968ce 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -442,8 +442,9 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, timeout : int, default 30 time in seconds to wait for server response before timeout utc_offset: int, default 0 - Use to specify a timezone other than the default UTC zero. Will force - year to ``coerce_year`` if not zero. + Use to specify a time zone other than the default UTC zero and roll + dataframe by ``utc_offset`` so that it starts at midnight on January + 1st. If not zero, will also force year to ``coerce_year``. coerce_year: int, default 1990 Use to force indices to desired year. Ignored if ``utc_offset`` is zero. From 440e7809f30b48cb1640b6bac7fe1f906a2675b4 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 31 Jul 2024 11:30:28 -0700 Subject: [PATCH 14/22] change coerce_year and utc_offset defaults to None in pvgis TMY - update arg docstrings - allow user to coerce year even if utc_offset is None or zero - use 1990 as default if utc_offset is not None or zero, but coerce_year was unspecified - add warning comment to be explicit and test identity to avoid unexpected implicit booleaness --- pvlib/iotools/pvgis.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index ebab8968ce..a89294ddb2 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -409,7 +409,7 @@ def _coerce_and_roll_pvgis_tmy(pvgis_data, tz, year): def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, map_variables=True, url=URL, timeout=30, - utc_offset=0, coerce_year=1990): + utc_offset=None, coerce_year=None): """ Get TMY data from PVGIS. @@ -441,13 +441,13 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, base url of PVGIS API, append ``tmy`` to get TMY endpoint timeout : int, default 30 time in seconds to wait for server response before timeout - utc_offset: int, default 0 + utc_offset: int, default None Use to specify a time zone other than the default UTC zero and roll - dataframe by ``utc_offset`` so that it starts at midnight on January - 1st. If not zero, will also force year to ``coerce_year``. - coerce_year: int, default 1990 - Use to force indices to desired year. Ignored if ``utc_offset`` is - zero. + dataframe by ``utc_offset`` so it starts at midnight on January 1st. + Ignored if ``None``, otherwise will also force year to ``coerce_year``. + coerce_year: int, default None + Use to force indices to desired year. Will default to 1990 if + ``coerce_year`` is ``None``, but ``utc_offset`` is not ``None``. Returns ------- @@ -534,7 +534,14 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, if map_variables: data = data.rename(columns=VARIABLE_MAP) - if utc_offset != 0: + if utc_offset is not None or coerce_year is not None: + # XXX: be explicit, test for identity not implicit booleaness + # utc_offset None but coerce_year isn't, set year with utc zero + if utc_offset is None: # XXX: None and zero are both False + utc_offset = 0 + # coerce_year is None but utc_off isn't, set year to 1990 + if coerce_year is None: # more explicit than (coerce_year or 1990) + coerce_year = 1990 data = _coerce_and_roll_pvgis_tmy(data, utc_offset, coerce_year) return data, months_selected, inputs, meta From 4fd05fbbd1c154b4e28f9b5ce85b871e62861728 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 31 Jul 2024 12:23:03 -0700 Subject: [PATCH 15/22] rename roll_utc_offset in get_pvgis_tmy - refactor utc_offset everywhere including comments and docstring - add additional test to coerce year even if utc offset is zero or none - change tzname to 'UTC' (versus Etc/GMT or Etc/GMT+0) if replacing with zero utc offset --- pvlib/iotools/pvgis.py | 24 +++++++++++++----------- pvlib/tests/iotools/test_pvgis.py | 13 +++++++++++-- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index a89294ddb2..43ce7449c5 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -395,7 +395,9 @@ def _coerce_and_roll_pvgis_tmy(pvgis_data, tz, year): After converting TZ, roll dataframe so timeseries starts at midnight and force all indices to a common year. Only works for integer TZ. """ - tzname = pytz.timezone(f'Etc/GMT{-tz:+d}') + tzname = ( + pytz.timezone(f'Etc/GMT{-tz:+d}') if tz != 0 + else pytz.timezone('UTC')) new_index = [ timestamp.replace(year=year, tzinfo=tzname) for timestamp in pvgis_data.index] @@ -409,7 +411,7 @@ def _coerce_and_roll_pvgis_tmy(pvgis_data, tz, year): def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, userhorizon=None, startyear=None, endyear=None, map_variables=True, url=URL, timeout=30, - utc_offset=None, coerce_year=None): + roll_utc_offset=None, coerce_year=None): """ Get TMY data from PVGIS. @@ -441,13 +443,13 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, base url of PVGIS API, append ``tmy`` to get TMY endpoint timeout : int, default 30 time in seconds to wait for server response before timeout - utc_offset: int, default None + roll_utc_offset: int, default None Use to specify a time zone other than the default UTC zero and roll - dataframe by ``utc_offset`` so it starts at midnight on January 1st. - Ignored if ``None``, otherwise will also force year to ``coerce_year``. + dataframe by ``roll_utc_offset`` so it starts at midnight on January + 1st. Ignored if ``None``, otherwise will force year to ``coerce_year``. coerce_year: int, default None Use to force indices to desired year. Will default to 1990 if - ``coerce_year`` is ``None``, but ``utc_offset`` is not ``None``. + ``coerce_year`` is ``None``, but ``roll_utc_offset`` is not ``None``. Returns ------- @@ -534,15 +536,15 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, if map_variables: data = data.rename(columns=VARIABLE_MAP) - if utc_offset is not None or coerce_year is not None: + if roll_utc_offset is not None or coerce_year is not None: # XXX: be explicit, test for identity not implicit booleaness - # utc_offset None but coerce_year isn't, set year with utc zero - if utc_offset is None: # XXX: None and zero are both False - utc_offset = 0 + # roll_utc_offset None but coerce_year isn't, set year with utc zero + if roll_utc_offset is None: # XXX: None and zero are both False + roll_utc_offset = 0 # coerce_year is None but utc_off isn't, set year to 1990 if coerce_year is None: # more explicit than (coerce_year or 1990) coerce_year = 1990 - data = _coerce_and_roll_pvgis_tmy(data, utc_offset, coerce_year) + data = _coerce_and_roll_pvgis_tmy(data, roll_utc_offset, coerce_year) return data, months_selected, inputs, meta diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index 58def17c98..d1d3ac98d1 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -447,7 +447,7 @@ def test_get_pvgis_tmy_coerce_year(): cet_tz = 1 # Turin time is CET cet_name = 'Etc/GMT-1' # check indices of rolled data after converting timezone - pvgis_data, _, _, _ = get_pvgis_tmy(45, 8, utc_offset=cet_tz) + pvgis_data, _, _, _ = get_pvgis_tmy(45, 8, roll_utc_offset=cet_tz) jan1_midnight = pd.Timestamp('1990-01-01 00:00:00', tz=cet_name) dec31_midnight = pd.Timestamp('1990-12-31 23:00:00', tz=cet_name) assert pvgis_data.index[0] == jan1_midnight @@ -459,7 +459,7 @@ def test_get_pvgis_tmy_coerce_year(): # repeat tests with year coerced test_yr = 2021 pvgis_data, _, _, _ = get_pvgis_tmy( - 45, 8, utc_offset=cet_tz, coerce_year=test_yr) + 45, 8, roll_utc_offset=cet_tz, coerce_year=test_yr) jan1_midnight = pd.Timestamp(f'{test_yr}-01-01 00:00:00', tz=cet_name) dec31_midnight = pd.Timestamp(f'{test_yr}-12-31 23:00:00', tz=cet_name) assert pvgis_data.index[0] == jan1_midnight @@ -467,6 +467,15 @@ def test_get_pvgis_tmy_coerce_year(): for m, test_case in enumerate(noon_test_data): expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12+cet_tz] assert all(test_case == expected) + # repeat tests with year coerced but utc offset none or zero + pvgis_data, _, _, _ = get_pvgis_tmy(45, 8, coerce_year=test_yr) + jan1_midnight = pd.Timestamp(f'{test_yr}-01-01 00:00:00', tz='UTC') + dec31_midnight = pd.Timestamp(f'{test_yr}-12-31 23:00:00', tz='UTC') + assert pvgis_data.index[0] == jan1_midnight + assert pvgis_data.index[-1] == dec31_midnight + for m, test_case in enumerate(noon_test_data): + expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12] + assert all(test_case == expected) @pytest.mark.remote_data From 1b92d40df4c6dda3b04cd77770e469b25f9dae78 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Thu, 1 Aug 2024 10:50:46 -0700 Subject: [PATCH 16/22] Update pvlib/iotools/pvgis.py with suggestions use "optional" vs. "default None" per #1574 Co-authored-by: Kevin Anderson --- pvlib/iotools/pvgis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 43ce7449c5..5568fe0431 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -443,13 +443,13 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, base url of PVGIS API, append ``tmy`` to get TMY endpoint timeout : int, default 30 time in seconds to wait for server response before timeout - roll_utc_offset: int, default None + roll_utc_offset: int, optional Use to specify a time zone other than the default UTC zero and roll dataframe by ``roll_utc_offset`` so it starts at midnight on January 1st. Ignored if ``None``, otherwise will force year to ``coerce_year``. - coerce_year: int, default None + coerce_year: int, optional Use to force indices to desired year. Will default to 1990 if - ``coerce_year`` is ``None``, but ``roll_utc_offset`` is not ``None``. + ``coerce_year`` is not specified, but ``roll_utc_offset`` is specified. Returns ------- From 43ce3c4a786fcc6bc1b5960b3d391f507804451b Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Thu, 1 Aug 2024 10:51:24 -0700 Subject: [PATCH 17/22] Update docs/sphinx/source/whatsnew/v0.11.1.rst rename argument "roll_utc_offset" in whatsnew Co-authored-by: Kevin Anderson --- docs/sphinx/source/whatsnew/v0.11.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/source/whatsnew/v0.11.1.rst b/docs/sphinx/source/whatsnew/v0.11.1.rst index 0148436154..41d0ebbc3c 100644 --- a/docs/sphinx/source/whatsnew/v0.11.1.rst +++ b/docs/sphinx/source/whatsnew/v0.11.1.rst @@ -16,7 +16,7 @@ Enhancements * Add new parameters for min/max absolute air mass to :py:func:`pvlib.spectrum.spectral_factor_firstsolar`. (:issue:`2086`, :pull:`2100`) -* Add ``utc_offset`` and ``coerce_year`` arguments to +* Add ``roll_utc_offset`` and ``coerce_year`` arguments to :py:func:`pvlib.iotools.get_pvgis_tmy` to allow user to specify time zone, rotate indices of TMY to begin at midnight, and force indices to desired year. (:issue:`2139`, :pull:`2138`) From 91c169e1f0ac5c51eb9f1c750685de87bb6fe6d3 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 5 Aug 2024 18:35:14 -0700 Subject: [PATCH 18/22] rename _coerce_and_roll_tmy, remove 'pvgis' --- pvlib/iotools/pvgis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 5568fe0431..7f01fa0cd0 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -390,7 +390,7 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): raise ValueError(err_msg) -def _coerce_and_roll_pvgis_tmy(pvgis_data, tz, year): +def _coerce_and_roll_tmy(pvgis_data, tz, year): """ After converting TZ, roll dataframe so timeseries starts at midnight and force all indices to a common year. Only works for integer TZ. @@ -544,7 +544,7 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, # coerce_year is None but utc_off isn't, set year to 1990 if coerce_year is None: # more explicit than (coerce_year or 1990) coerce_year = 1990 - data = _coerce_and_roll_pvgis_tmy(data, roll_utc_offset, coerce_year) + data = _coerce_and_roll_tmy(data, roll_utc_offset, coerce_year) return data, months_selected, inputs, meta From 1e9ee64186eb74457f1b2769093b406dc39af7a7 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 5 Aug 2024 19:06:15 -0700 Subject: [PATCH 19/22] rename index with new tz in coerce pvgis tmy --- pvlib/iotools/pvgis.py | 1 + pvlib/tests/iotools/test_pvgis.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 7f01fa0cd0..b8fd14c793 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -405,6 +405,7 @@ def _coerce_and_roll_tmy(pvgis_data, tz, year): np.roll(pvgis_data, tz, axis=0), columns=pvgis_data.columns, index=new_index) + new_pvgis_data.index.name = f'time({tzname})' return new_pvgis_data diff --git a/pvlib/tests/iotools/test_pvgis.py b/pvlib/tests/iotools/test_pvgis.py index d1d3ac98d1..728e6789e8 100644 --- a/pvlib/tests/iotools/test_pvgis.py +++ b/pvlib/tests/iotools/test_pvgis.py @@ -441,6 +441,7 @@ def test_get_pvgis_tmy_coerce_year(): """test utc_offset and coerce_year work as expected""" base_case, _, _, _ = get_pvgis_tmy(45, 8) # Turin assert str(base_case.index.tz) == 'UTC' + assert base_case.index.name == 'time(UTC)' noon_test_data = [ base_case[base_case.index.month == m].iloc[12] for m in range(1, 13)] @@ -452,6 +453,7 @@ def test_get_pvgis_tmy_coerce_year(): dec31_midnight = pd.Timestamp('1990-12-31 23:00:00', tz=cet_name) assert pvgis_data.index[0] == jan1_midnight assert pvgis_data.index[-1] == dec31_midnight + assert pvgis_data.index.name == f'time({cet_name})' # spot check rolled data matches original for m, test_case in enumerate(noon_test_data): expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12+cet_tz] @@ -464,6 +466,7 @@ def test_get_pvgis_tmy_coerce_year(): dec31_midnight = pd.Timestamp(f'{test_yr}-12-31 23:00:00', tz=cet_name) assert pvgis_data.index[0] == jan1_midnight assert pvgis_data.index[-1] == dec31_midnight + assert pvgis_data.index.name == f'time({cet_name})' for m, test_case in enumerate(noon_test_data): expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12+cet_tz] assert all(test_case == expected) @@ -473,6 +476,7 @@ def test_get_pvgis_tmy_coerce_year(): dec31_midnight = pd.Timestamp(f'{test_yr}-12-31 23:00:00', tz='UTC') assert pvgis_data.index[0] == jan1_midnight assert pvgis_data.index[-1] == dec31_midnight + assert pvgis_data.index.name == 'time(UTC)' for m, test_case in enumerate(noon_test_data): expected = pvgis_data[pvgis_data.index.month == m+1].iloc[12] assert all(test_case == expected) From d24e4f3c61f4f04970b674000fce6994c1db100d Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Mon, 5 Aug 2024 19:41:40 -0700 Subject: [PATCH 20/22] allow tz of None in _coerce_and_roll_tmy - treat tz=None as UTC - allows get_pvgis_tmy to be simpler - remove unnecessary comments --- pvlib/iotools/pvgis.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index b8fd14c793..360eb37e50 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -393,11 +393,14 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): def _coerce_and_roll_tmy(pvgis_data, tz, year): """ After converting TZ, roll dataframe so timeseries starts at midnight - and force all indices to a common year. Only works for integer TZ. + and force all indices to a common year. Only works for integer TZ, but + ``None`` and ``False`` are re-interpreted as zero / UTC. """ - tzname = ( - pytz.timezone(f'Etc/GMT{-tz:+d}') if tz != 0 - else pytz.timezone('UTC')) + if tz: + tzname = pytz.timezone(f'Etc/GMT{-tz:+d}') + else: + tz = 0 + tzname = pytz.timezone('UTC') new_index = [ timestamp.replace(year=year, tzinfo=tzname) for timestamp in pvgis_data.index] @@ -537,14 +540,9 @@ def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, if map_variables: data = data.rename(columns=VARIABLE_MAP) - if roll_utc_offset is not None or coerce_year is not None: - # XXX: be explicit, test for identity not implicit booleaness - # roll_utc_offset None but coerce_year isn't, set year with utc zero - if roll_utc_offset is None: # XXX: None and zero are both False - roll_utc_offset = 0 - # coerce_year is None but utc_off isn't, set year to 1990 - if coerce_year is None: # more explicit than (coerce_year or 1990) - coerce_year = 1990 + if not (roll_utc_offset is None and coerce_year is None): + # roll_utc_offset is specified, but coerce_year isn't + coerce_year = coerce_year or 1990 data = _coerce_and_roll_tmy(data, roll_utc_offset, coerce_year) return data, months_selected, inputs, meta From fb2813d14aaac36c9227478ab7be1006ffa6cc43 Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 14 Aug 2024 00:42:57 -0700 Subject: [PATCH 21/22] clarify input tmy_data is UTC... - ... in docstring of private function pvgis._coerce_and_roll_tmy() - rename tmy_data - name new_index explicitly using pd.DatetimeIndex() --- pvlib/iotools/pvgis.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index 360eb37e50..f2ad1c192d 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -390,26 +390,27 @@ def read_pvgis_hourly(filename, pvgis_format=None, map_variables=True): raise ValueError(err_msg) -def _coerce_and_roll_tmy(pvgis_data, tz, year): +def _coerce_and_roll_tmy(tmy_data, tz, year): """ - After converting TZ, roll dataframe so timeseries starts at midnight - and force all indices to a common year. Only works for integer TZ, but - ``None`` and ``False`` are re-interpreted as zero / UTC. + Assumes ``tmy_data`` input is UTC, converts from UTC to ``tz``, rolls + dataframe so timeseries starts at midnight, and forces all indices to + ``year``. Only works for integer ``tz``, but ``None`` and ``False`` are + re-interpreted as zero / UTC. """ if tz: tzname = pytz.timezone(f'Etc/GMT{-tz:+d}') else: tz = 0 tzname = pytz.timezone('UTC') - new_index = [ + new_index = pd.DatetimeIndex([ timestamp.replace(year=year, tzinfo=tzname) - for timestamp in pvgis_data.index] - new_pvgis_data = pd.DataFrame( - np.roll(pvgis_data, tz, axis=0), - columns=pvgis_data.columns, + for timestamp in tmy_data.index], + name = f'time({tzname})') + new_tmy_data = pd.DataFrame( + np.roll(tmy_data, tz, axis=0), + columns=tmy_data.columns, index=new_index) - new_pvgis_data.index.name = f'time({tzname})' - return new_pvgis_data + return new_tmy_data def get_pvgis_tmy(latitude, longitude, outputformat='json', usehorizon=True, From ac9dbb0a8cdb4e25e2aa872920ec6180016eb75f Mon Sep 17 00:00:00 2001 From: Mark Mikofski Date: Wed, 14 Aug 2024 00:46:52 -0700 Subject: [PATCH 22/22] fix flake8 in _coerce_and_roll_tmy --- pvlib/iotools/pvgis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pvlib/iotools/pvgis.py b/pvlib/iotools/pvgis.py index f2ad1c192d..fbfcf55e33 100644 --- a/pvlib/iotools/pvgis.py +++ b/pvlib/iotools/pvgis.py @@ -405,7 +405,7 @@ def _coerce_and_roll_tmy(tmy_data, tz, year): new_index = pd.DatetimeIndex([ timestamp.replace(year=year, tzinfo=tzname) for timestamp in tmy_data.index], - name = f'time({tzname})') + name=f'time({tzname})') new_tmy_data = pd.DataFrame( np.roll(tmy_data, tz, axis=0), columns=tmy_data.columns,