Merge pull request #379 from NREL/release/2.1.6

Release/2.1.6

.github/workflows/nbval.yaml

@@ -25,7 +25,7 @@ jobs:
  - name: Install notebook environment
  run: |
  python -m pip install --upgrade pip wheel
- pip install -r requirements.txt -r docs/notebook_requirements.txt .[test]
+ pip install --timeout=300 -r requirements.txt -r docs/notebook_requirements.txt .[test]
  - name: Run notebook and check output
  run: |
  # --sanitize-with: pre-process text to remove irrelevant differences (e.g. warning filepaths)

.github/workflows/pytest.yaml

@@ -40,7 +40,7 @@ jobs:
  - name: Install ${{ matrix.env }}
  run: |
  python -m pip install --upgrade pip wheel
- pip install ${{ matrix.env }}
+ pip install --timeout=300 ${{ matrix.env }}
  - name: Test with pytest ${{ matrix.env }}
  run: |
  pytest

.github/workflows/requirements.yaml

@@ -26,4 +26,4 @@ jobs:
  - name: Install notebook environment
  run: |
  python -m pip install --upgrade pip wheel
- pip install -r requirements.txt -r docs/notebook_requirements.txt
+ pip install --timeout=300 -r requirements.txt -r docs/notebook_requirements.txt

docs/notebook_requirements.txt

@@ -11,10 +11,10 @@ defusedxml==0.7.1
 entrypoints==0.2.3
 html5lib==1.0.1
 ipykernel==4.8.2
-ipython==7.16.3
+ipython==8.10.0
 ipython-genutils==0.2.0
 ipywidgets==7.3.0
-jedi==0.12.1
+jedi==0.16.0
 Jinja2==3.0.0
 jsonschema==2.6.0
 jupyter==1.0.0
@@ -32,11 +32,11 @@ nest-asyncio==1.5.5
 notebook==6.4.12
 numexpr==2.8.0
 pandocfilters==1.4.2
-parso==0.3.1
+parso==0.5.2
 pexpect==4.6.0
 pickleshare==0.7.5
 prometheus-client==0.3.0
-prompt-toolkit==3.0.27
+prompt-toolkit==3.0.30
 ptyprocess==0.6.0
 pycparser==2.20
 Pygments==2.7.4

docs/sphinx/source/changelog.rst

@@ -1,5 +1,6 @@
 RdTools Change Log
 ==================
+.. include:: changelog/v2.1.6.rst
 .. include:: changelog/v2.1.5.rst
 .. include:: changelog/v2.1.4.rst
 .. include:: changelog/v2.1.3.rst

docs/sphinx/source/changelog/v2.1.6.rst

@@ -0,0 +1,37 @@
+**********************
+v2.1.6 (July 31, 2023)
+**********************
+
+Bug Fixes
+---------
+* Fix NonExistentTimeError in :py:func:`rdtools.clearsky_temperature.get_clearsky_tamb`
+ (:issue:`372` :pull:`373`)
+
+Enhancements
+------------
+* :py:func:`rdtools.degradation.degradation_classical_decomposition` now
+ executes significantly faster. (:pull:`371`)
+
+Requirements
+------------
+* Increased the minimum versions of several dependencies: (:pull:`371`)
+
+ + pandas increased to 1.3.0 (released July 2, 2021)
+ + numpy to 1.17.3 (released October 17, 2019)
+ + statsmodels to 0.11.0 (released February 21, 2020)
+ + scipy to 1.2.0 (released December 17, 2018)
+
+* Add support for pvlib 0.10 (:pull:`378`)
+* Updated notebook requirements (:pull:`360`)
+* Bumps certifi from 2020.12.5 to 2022.12.7 (:pull:`357`)
+
+Testing
+-------
+* Extended pip timeout (:pull:`360`)
+* Updated example notebooks with new figure sizes (:pull:`360`)
+
+Contributors
+------------
+* Michael Deceglie (:ghuser:`mdeceglie`)
+* Bernat Nicolau (:ghuser:`BernatNicolau`)
+* Kevin Anderson (:ghuser:`kandersolar`)

rdtools/clearsky_temperature.py

@@ -57,7 +57,9 @@ def get_clearsky_tamb(times, latitude, longitude, window_size=40,
  freq_actual = times.freq
 
  dt_daily = pd.date_range(times.date[0] - buffer, times.date[-1] + buffer,
- freq='D', tz=times.tz)
+ freq='D')
+ dt_daily = dt_daily.tz_localize(times.tz, ambiguous='infer',
+ nonexistent='shift_forward')
 
  f = h5py.File(filepath, "r")
 

rdtools/degradation.py

@@ -126,17 +126,9 @@ def degradation_classical_decomposition(energy_normalized,
 
  # Compute yearly rolling mean to isolate trend component using
  # moving average
- it = df.iterrows()
- energy_ma = []
- for i, row in it:
- if row.years - 0.5 >= min(df.years) and \
- row.years + 0.5 <= max(df.years):
- roll = df[(df.years <= row.years + 0.5) &
- (df.years >= row.years - 0.5)]
- energy_ma.append(roll.energy_normalized.mean())
- else:
- energy_ma.append(np.nan)
-
+ energy_ma = df['energy_normalized'].rolling('365d', center=True).mean()
+ has_full_year = (df['years'] >= df['years'][0] + 0.5) & (df['years'] <= df['years'][-1] - 0.5)
+ energy_ma[~has_full_year] = np.nan
  df['energy_ma'] = energy_ma
 
  # add intercept-constant to the exogeneous variable
@@ -332,10 +324,8 @@ def _mk_test(x, alpha=0.05):
  n = len(x)
 
  # calculate S
- s = 0
- for k in range(n - 1):
- for j in range(k + 1, n):
- s += np.sign(x[j] - x[k])
+ x = np.array(x)
+ s = np.sum(np.triu(np.sign(-np.subtract.outer(x, x)), 1))
 
  # calculate the unique data
  unique_x = np.unique(x)

rdtools/test/analysis_chains_test.py

@@ -407,11 +407,11 @@ def test_srr_soiling(soiling_analysis_sensor):
  ci = srr_results['sratio_confidence_interval']
  renorm_factor = srr_results['calc_info']['renormalizing_factor']
  print(f'soiling ci:{ci}')
- assert 0.965 == pytest.approx(sratio, abs=1e-3),\
+ assert 0.965 == pytest.approx(sratio, abs=1e-3), \
  'Soiling ratio different from expected value in TrendAnalysis.srr_soiling'
- assert [0.96, 0.97] == pytest.approx(ci, abs=1e-2),\
+ assert [0.96, 0.97] == pytest.approx(ci, abs=1e-2), \
  'Soiling confidence interval different from expected value in TrendAnalysis.srr_soiling'
- assert 0.974 == pytest.approx(renorm_factor, abs=1e-3),\
+ assert 0.974 == pytest.approx(renorm_factor, abs=1e-3), \
  'Renormalization factor different from expected value in TrendAnalysis.srr_soiling'
 
 

rdtools/test/clearsky_temperature_test.py

@@ -1,6 +1,8 @@
 import pytest
+import datetime
 import pandas as pd
 
+
 from rdtools.clearsky_temperature import get_clearsky_tamb
 
 
@@ -35,3 +37,21 @@ def test_not_on_land():
  with pytest.warns(UserWarning, match='possibly invalid Lat/Lon coordinates'):
  ocean_cs_tamb = get_clearsky_tamb(dt, 40, -60)
  assert ocean_cs_tamb.isnull().all()
+
+
+def test_with_tricky_timezones():
+ # Some timezones have DST shifts at midnight, which
+ # can lead to NonExistentTimeError. This tests for the
+ # problem in issue #372
+
+ tz = 'America/Santiago'
+ start_date = datetime.datetime(2018, 8, 10, 0, 0, 0)
+ end_date = datetime.datetime(2018, 8, 14, 23, 0, 0)
+ freq = 'H'
+ lat = -24
+ lon = -70
+
+ times = pd.date_range(start=start_date, end=end_date, freq=freq)
+ times = times.tz_localize(tz=tz, ambiguous='infer',
+ nonexistent='shift_forward')
+ get_clearsky_tamb(times, lat, lon)

rdtools/test/soiling_test.py

@@ -13,17 +13,17 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
  reps = 10
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=reps)
- assert 0.964369 == pytest.approx(sr, abs=1e-6),\
+ assert 0.964369 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different from expected value'
- assert np.array([0.962540, 0.965295]) == pytest.approx(sr_ci, abs=1e-6),\
+ assert np.array([0.962540, 0.965295]) == pytest.approx(sr_ci, abs=1e-6), \
  'Confidence interval different from expected value'
- assert 0.960205 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6),\
+ assert 0.960205 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6), \
  'Exceedance level different from expected value'
- assert 0.984079 == pytest.approx(soiling_info['renormalizing_factor'], abs=1e-6),\
+ assert 0.984079 == pytest.approx(soiling_info['renormalizing_factor'], abs=1e-6), \
  'Renormalizing factor different from expected value'
- assert len(soiling_info['stochastic_soiling_profiles']) == reps,\
+ assert len(soiling_info['stochastic_soiling_profiles']) == reps, \
  'Length of soiling_info["stochastic_soiling_profiles"] different than expected'
- assert isinstance(soiling_info['stochastic_soiling_profiles'], list),\
+ assert isinstance(soiling_info['stochastic_soiling_profiles'], list), \
  'soiling_info["stochastic_soiling_profiles"] is not a list'
 
  # Check soiling_info['soiling_interval_summary']
@@ -33,12 +33,12 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
  actual_summary_columns = soiling_info['soiling_interval_summary'].columns.values
 
  for x in actual_summary_columns:
- assert x in expected_summary_columns,\
+ assert x in expected_summary_columns, \
  f"'{x}' not an expected column in soiling_info['soiling_interval_summary']"
  for x in expected_summary_columns:
- assert x in actual_summary_columns,\
+ assert x in actual_summary_columns, \
  f"'{x}' was expected as a column, but not in soiling_info['soiling_interval_summary']"
- assert isinstance(soiling_info['soiling_interval_summary'], pd.DataFrame),\
+ assert isinstance(soiling_info['soiling_interval_summary'], pd.DataFrame), \
  'soiling_info["soiling_interval_summary"] not a dataframe'
  expected_means = pd.Series({'soiling_rate': -0.002644544,
  'soiling_rate_low': -0.002847504,
@@ -57,9 +57,9 @@ def test_soiling_srr(soiling_normalized_daily, soiling_insolation, soiling_times
  pd.testing.assert_index_equal(soiling_info['soiling_ratio_perfect_clean'].index, soiling_times,
  check_names=False)
  sr_mean = soiling_info['soiling_ratio_perfect_clean'].mean()
- assert 0.968265 == pytest.approx(sr_mean, abs=1e-6),\
+ assert 0.968265 == pytest.approx(sr_mean, abs=1e-6), \
  "The mean of soiling_info['soiling_ratio_perfect_clean'] differs from expected"
- assert isinstance(soiling_info['soiling_ratio_perfect_clean'], pd.Series),\
+ assert isinstance(soiling_info['soiling_ratio_perfect_clean'], pd.Series), \
  'soiling_info["soiling_ratio_perfect_clean"] not a pandas series'
 
 
@@ -74,7 +74,7 @@ def test_soiling_srr_consecutive_invalid(soiling_normalized_daily, soiling_insol
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=reps,
  max_relative_slope_error=20.0, method=method)
- assert expected_sr == pytest.approx(sr, abs=1e-6),\
+ assert expected_sr == pytest.approx(sr, abs=1e-6), \
  f'Soiling ratio different from expected value for {method} with consecutive invalid intervals' # noqa: E501
 
 
@@ -96,7 +96,7 @@ def test_soiling_srr_with_precip(soiling_normalized_daily, soiling_insolation, s
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
  clean_criterion=clean_criterion, **kwargs)
- assert expected_sr == pytest.approx(sr, abs=1e-6),\
+ assert expected_sr == pytest.approx(sr, abs=1e-6), \
  f"Soiling ratio with clean_criterion='{clean_criterion}' different from expected"
 
 
@@ -105,9 +105,9 @@ def test_soiling_srr_confidence_levels(soiling_normalized_daily, soiling_insolat
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
  confidence_level=95, reps=10, exceedance_prob=80.0)
- assert np.array([0.959322, 0.966066]) == pytest.approx(sr_ci, abs=1e-6),\
+ assert np.array([0.959322, 0.966066]) == pytest.approx(sr_ci, abs=1e-6), \
  'Confidence interval with confidence_level=95 different than expected'
- assert 0.962691 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6),\
+ assert 0.962691 == pytest.approx(soiling_info['exceedance_level'], abs=1e-6), \
  'soiling_info["exceedance_level"] different than expected when exceedance_prob=80'
 
 
@@ -125,7 +125,7 @@ def test_soiling_srr_clean_threshold(soiling_normalized_daily, soiling_insolatio
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
  clean_threshold=0.01)
- assert 0.964369 == pytest.approx(sr, abs=1e-6),\
+ assert 0.964369 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio with specified clean_threshold different from expected value'
 
  with pytest.raises(NoValidIntervalError):
@@ -139,9 +139,9 @@ def test_soiling_srr_trim(soiling_normalized_daily, soiling_insolation):
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
  trim=True)
 
- assert 0.978093 == pytest.approx(sr, abs=1e-6),\
+ assert 0.978093 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio with trim=True different from expected value'
- assert len(soiling_info['soiling_interval_summary']) == 1,\
+ assert len(soiling_info['soiling_interval_summary']) == 1, \
  'Wrong number of soiling intervals found with trim=True'
 
 
@@ -153,7 +153,7 @@ def test_soiling_srr_method(soiling_normalized_daily, soiling_insolation, method
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
  method=method)
- assert expected_sr == pytest.approx(sr, abs=1e-6),\
+ assert expected_sr == pytest.approx(sr, abs=1e-6), \
  f'Soiling ratio with method="{method}" different from expected value'
 
 
@@ -174,9 +174,9 @@ def test_soiling_srr_recenter_false(soiling_normalized_daily, soiling_insolation
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation, reps=10,
  recenter=False)
- assert 1 == soiling_info['renormalizing_factor'],\
+ assert 1 == soiling_info['renormalizing_factor'], \
  'Renormalizing factor != 1 with recenter=False'
- assert 0.966387 == pytest.approx(sr, abs=1e-6),\
+ assert 0.966387 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different than expected when recenter=False'
 
 
@@ -188,11 +188,11 @@ def test_soiling_srr_negative_step(soiling_normalized_daily, soiling_insolation)
  with pytest.warns(UserWarning, match='20% or more of the daily data'):
  sr, sr_ci, soiling_info = soiling_srr(stepped_daily, soiling_insolation, reps=10)
 
- assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, False, True],\
+ assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, False, True], \
  'Soiling interval validity differs from expected when a large negative step\
  is incorporated into the data'
 
- assert 0.936932 == pytest.approx(sr, abs=1e-6),\
+ assert 0.936932 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different from expected when a large negative step is incorporated into the data' # noqa: E501
 
 
@@ -202,10 +202,10 @@ def test_soiling_srr_max_negative_slope_error(soiling_normalized_daily, soiling_
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily, soiling_insolation,
  reps=10, max_relative_slope_error=45.0)
 
- assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, True, False],\
+ assert list(soiling_info['soiling_interval_summary']['valid'].values) == [True, True, False], \
  'Soiling interval validity differs from expected when max_relative_slope_error=45.0'
 
- assert 0.958761 == pytest.approx(sr, abs=1e-6),\
+ assert 0.958761 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different from expected when max_relative_slope_error=45.0'
 
 
@@ -220,14 +220,14 @@ def test_soiling_srr_with_nan_interval(soiling_normalized_daily, soiling_insolat
  np.random.seed(1977)
  with pytest.warns(UserWarning, match='20% or more of the daily data'):
  sr, sr_ci, soiling_info = soiling_srr(normalized_corrupt, soiling_insolation, reps=reps)
- assert 0.948792 == pytest.approx(sr, abs=1e-6),\
+ assert 0.948792 == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different from expected value when an entire interval was NaN'
 
 
 def test_soiling_srr_outlier_factor(soiling_normalized_daily, soiling_insolation):
  _, _, info = soiling_srr(soiling_normalized_daily, soiling_insolation,
  reps=1, outlier_factor=8)
- assert len(info['soiling_interval_summary']) == 2,\
+ assert len(info['soiling_interval_summary']) == 2, \
  'Increasing the outlier_factor did not result in the expected number of soiling intervals'
 
 
@@ -256,7 +256,7 @@ def test_soiling_srr_min_interval_length_default(soiling_normalized_daily, soili
  np.random.seed(1977)
  sr, sr_ci, soiling_info = soiling_srr(soiling_normalized_daily[start:],
  soiling_insolation[start:], reps=reps)
- assert expected_sr == pytest.approx(sr, abs=1e-6),\
+ assert expected_sr == pytest.approx(sr, abs=1e-6), \
  'Soiling ratio different from expected value'
 
 

requirements-min.txt

@@ -1,10 +1,10 @@
 h5py==2.8.0
 matplotlib==3.0.0
-numpy==1.15
-pandas==0.23.2
+numpy==1.17.3
+pandas==1.3.0
 pvlib==0.7.0
-scipy==1.1.0
-statsmodels==0.9.0
+scipy==1.2.0
+statsmodels==0.11.0
 tables==3.5.1
 numexpr==2.7.1 # https://github.com/pydata/numexpr/issues/369
 plotly==4.0.0

requirements.txt

@@ -1,5 +1,5 @@
 cached-property==1.5.2
-certifi==2020.12.5
+certifi==2022.12.7
 chardet==4.0.0
 cycler==0.10.0
 fonttools==4.37.1