make test_psm3.py robust to API overuse errors (#873)

* use NREL_API_KEY environment variable when available * remove try-except clause to ensure that azure env variable is used * set environment variable from Azure's secret variables * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines within pytest call * Update azure-pipelines.yml for Azure Pipelines echo only * Update azure-pipelines.yml for Azure Pipelines with python * Update azure-pipelines.yml for Azure Pipelines all os.environ * Update azure-pipelines.yml for Azure Pipelines export variable * export Azure secret variable into environment to access from python * Update azure-pipelines.yml for Azure Pipelines fix displayName * Update azure-pipelines.yml for Azure Pipelines psm3 test only * Update azure-pipelines.yml for Azure Pipelines for real this time * Update azure-pipelines.yml for Azure Pipelines for really real this time * Update azure-pipelines.yml for Azure Pipelines single failing test * Update azure-pipelines.yml for Azure Pipelines test_get_psm3_tmy * Update azure-pipelines.yml for Azure Pipelines io_input * Update azure-pipelines.yml for Azure Pipelines test_parse_psm3 * Update azure-pipelines.yml for Azure Pipelines test_read_psm3 * Update azure-pipelines.yml for Azure Pipelines all test_psm3.py tests * Update azure-pipelines.yml for Azure Pipelines all python versions * use NREL_API_KEY environment variable when available * remove try-except clause to ensure that azure env variable is used * set environment variable from Azure's secret variables * Update azure-pipelines.yml for Azure Pipelines * Update azure-pipelines.yml for Azure Pipelines within pytest call * Update azure-pipelines.yml for Azure Pipelines echo only * Update azure-pipelines.yml for Azure Pipelines with python * Update azure-pipelines.yml for Azure Pipelines all os.environ * Update azure-pipelines.yml for Azure Pipelines export variable * export Azure secret variable into environment to access from python * temporary test_blah function and get_psm3 delayer * improve delay_get_psm3(), replace get_psm3() calls * set DEMO_KEY to environment variable if available * update azure-pipelines basic script for testing * match azure secret variable * return to full suite of testing on azure * return test_psm3() specific pytest call to general * add description to whatsnew * formatting fix * logger module implementation * more formatting * change while loop to for loop * use warnings instead of logging * fixturize DEMO_KEY * implement use of pytest-rerunfailures instead of custom loop * parametrize error tests into a signle test * fix windows environment import of NREL key * test to echo environment variable of NREL key * fix windows secret variable setting * add pytest-rerunfailures to ci requirements yml * use secret.nrelApiKey syntax * test new secret variable * try secret.api_key syntax * std syntax only psm3 * try to print other variables that arent secrets * confirm personal repo still works * bump * try all python versions * try everything * install pytest-rerunrailures in Azure * configure conda_35 tests to pull pytest_rerunfailures from pip instead of conda * add note for pytest-rerunfailures for conda_35 * change fixture from DEMO_KEY to nrel_api_key * remove debug variable echo commands * move pytest-rerunfailures to pip install * improve docstrings * use verbose name for caught error * readd leap year test
pvlib · CameronTStark · Mar 28, 2020 · Aug 8, 2019 · Aug 8, 2019 · Aug 9, 2019
commit 1200266713f7b48fb70c154732b10dcfdcc52f89
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -19,14 +19,16 @@ jobs:
       Python37:
         python.version: '3.7'
 
+
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
       pip install -e .
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 
@@ -74,6 +76,7 @@ jobs:
     displayName: 'List installed dependencies'
   - script: |
       source activate test_env
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov --cov-report=xml --cov-report=html
     displayName: 'pytest'
   # - script: |
@@ -126,6 +129,7 @@ jobs:
     displayName: 'List installed dependencies'
   - script: |
       call activate test_env
+      set NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml
     displayName: 'pytest'
   - task: PublishTestResults@2
@@ -153,8 +157,9 @@ jobs:
       versionSpec: '$(python.version)'
 
   - script: |
-      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines
+      pip install pytest pytest-cov pytest-mock pytest-timeout pytest-azurepipelines pytest-rerunfailures
       pip install -e .
+      export NREL_API_KEY=$(nrelApiKey)
       pytest pvlib --junitxml=junit/test-results.xml --cov=pvlib --cov-report=xml --cov-report=html
     displayName: 'Test with pytest'
 

diff --git a/ci/requirements-py35-min.yml b/ci/requirements-py35-min.yml
@@ -15,3 +15,4 @@ dependencies:
     - pip:
         - numpy==1.12.0
         - pandas==0.18.1
+        - pytest-rerunfailures # conda version is >3.6
diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml
@@ -26,3 +26,4 @@ dependencies:
     - pip:
         - nrel-pysam>=2.0
         - pvfactors==1.0.1
+        - pytest-rerunfailures # conda version is >3.6
diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml
@@ -16,6 +16,7 @@ dependencies:
     - pytest
     - pytest-cov
     - pytest-mock
+    - pytest-rerunfailures
     - pytest-timeout
     - python=3.6
     - pytz

diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml
@@ -17,6 +17,7 @@ dependencies:
     - pytest-cov
     - pytest-mock
     - pytest-timeout
+    - pytest-rerunfailures
     - python=3.7
     - pytz
     - requests

diff --git a/docs/sphinx/source/whatsnew/v0.7.2.rst b/docs/sphinx/source/whatsnew/v0.7.2.rst
@@ -13,6 +13,8 @@ Bug fixes
 ~~~~~~~~~
 * Fix :py:func:`~pvlib.iotools.read_tmy3` parsing when February contains
   a leap year (:pull:`866`)
+* Implement NREL Developer Network API key for consistent success with API
+  calls in :py:mod:`pvlib.tests.iotools.test_psm3` (:pull:`873`)
 
 Documentation
 ~~~~~~~~~~~~~
@@ -26,3 +28,4 @@ Contributors
 ~~~~~~~~~~~~
 * Mark Mikofski (:ghuser:`mikofski`)
 * Cliff Hansen (:ghuser:`cwhanse`)
+* Cameron T. Stark (:ghuser:`camerontstark`)
diff --git a/pvlib/tests/iotools/test_psm3.py b/pvlib/tests/iotools/test_psm3.py
@@ -2,13 +2,15 @@
 test iotools for PSM3
 """
 
+import os
 from pvlib.iotools import psm3
 from conftest import needs_pandas_0_22, DATA_DIR
 import numpy as np
 import pandas as pd
 import pytest
 from requests import HTTPError
 from io import StringIO
+import warnings
 
 TMY_TEST_DATA = DATA_DIR / 'test_psm3_tmy-2017.csv'
 YEAR_TEST_DATA = DATA_DIR / 'test_psm3_2017.csv'
@@ -21,7 +23,26 @@
     'Temperature Units', 'Pressure Units', 'Wind Direction Units',
     'Wind Speed', 'Surface Albedo Units', 'Version']
 PVLIB_EMAIL = 'pvlib-admin@googlegroups.com'
-DEMO_KEY = 'DEMO_KEY'
+
+
+@pytest.fixture(scope="module")
+def nrel_api_key():
+    """Supplies pvlib-python's NREL Developer Network API key.
+
+    Azure Pipelines CI utilizes a secret variable set to NREL_API_KEY
+    to mitigate failures associated with using the default key of
+    "DEMO_KEY". A user is capable of using their own key this way if
+    desired however the default key should suffice for testing purposes.
+    """
+    try:
+        demo_key = os.environ["NREL_API_KEY"]
+    except KeyError:
+        warnings.warn(
+            "WARNING: NREL API KEY environment variable not set! "
+            "Using DEMO_KEY instead. Unexpected failures may occur."
+        )
+        demo_key = 'DEMO_KEY'
+    return demo_key
 
 
 def assert_psm3_equal(header, data, expected):
@@ -50,47 +71,58 @@ def assert_psm3_equal(header, data, expected):
 
 
 @needs_pandas_0_22
-def test_get_psm3_tmy():
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_tmy(nrel_api_key):
     """test get_psm3 with a TMY"""
-    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='tmy-2017')
+    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
+                                 PVLIB_EMAIL, names='tmy-2017')
     expected = pd.read_csv(TMY_TEST_DATA)
     assert_psm3_equal(header, data, expected)
-    # check errors
-    with pytest.raises(HTTPError):
-        # HTTP 403 forbidden because api_key is rejected
-        psm3.get_psm3(LATITUDE, LONGITUDE, api_key='BAD', email=PVLIB_EMAIL)
-    with pytest.raises(HTTPError):
-        # coordinates were not found in the NSRDB
-        psm3.get_psm3(51, -5, DEMO_KEY, PVLIB_EMAIL)
-    with pytest.raises(HTTPError):
-        # names is not one of the available options
-        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, names='bad')
 
 
 @needs_pandas_0_22
-def test_get_psm3_singleyear():
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_singleyear(nrel_api_key):
     """test get_psm3 with a single year"""
-    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='2017', interval=30)
+    header, data = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
+                                 PVLIB_EMAIL, names='2017', interval=30)
     expected = pd.read_csv(YEAR_TEST_DATA)
     assert_psm3_equal(header, data, expected)
-    # check leap day
-    _, data_2012 = psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL,
-                                 names='2012', interval=60, leap_day=True)
-    assert len(data_2012) == (8760+24)
-    # check errors
-    with pytest.raises(HTTPError):
-        # HTTP 403 forbidden because api_key is rejected
-        psm3.get_psm3(LATITUDE, LONGITUDE, api_key='BAD', email=PVLIB_EMAIL,
-                      names='2017')
-    with pytest.raises(HTTPError):
-        # coordinates were not found in the NSRDB
-        psm3.get_psm3(51, -5, DEMO_KEY, PVLIB_EMAIL, names='2017')
-    with pytest.raises(HTTPError):
-        # intervals can only be 30 or 60 minutes
-        psm3.get_psm3(LATITUDE, LONGITUDE, DEMO_KEY, PVLIB_EMAIL, names='2017',
-                      interval=15)
+
+
+@needs_pandas_0_22
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_check_leap_day(nrel_api_key):
+    _, data_2012 = psm3.get_psm3(LATITUDE, LONGITUDE, nrel_api_key,
+                                 PVLIB_EMAIL, names="2012", interval=60,
+                                 leap_day=True)
+    assert len(data_2012) == (8760 + 24)
+
+
+@pytest.mark.parametrize('latitude, longitude, api_key, names, interval',
+                         [(LATITUDE, LONGITUDE, 'BAD', 'tmy-2017', 60),
+                          (51, -5, nrel_api_key, 'tmy-2017', 60),
+                          (LATITUDE, LONGITUDE, nrel_api_key, 'bad', 60),
+                          (LATITUDE, LONGITUDE, nrel_api_key, '2017', 15),
+                          ])
+@needs_pandas_0_22
+@pytest.mark.flaky(reruns=5, reruns_delay=2)
+def test_get_psm3_tmy_errors(
+    latitude, longitude, api_key, names, interval
+):
+    """Test get_psm3() for multiple erroneous input scenarios.
+
+    These scenarios include:
+    * Bad API key -> HTTP 403 forbidden because api_key is rejected
+    * Bad latitude/longitude -> Coordinates were not found in the NSRDB.
+    * Bad name -> Name is not one of the available options.
+    * Bad interval, single year -> Intervals can only be 30 or 60 minutes.
+    """
+    with pytest.raises(HTTPError) as excinfo:
+        psm3.get_psm3(latitude, longitude, api_key, PVLIB_EMAIL,
+                      names=names, interval=interval)
+    # ensure the HTTPError caught isn't due to overuse of the API key
+    assert "OVER_RATE_LIMIT" not in str(excinfo.value)
 
 
 @pytest.fixture

diff --git a/setup.py b/setup.py
@@ -42,7 +42,7 @@
                     'pytz',
                     'requests']
 TESTS_REQUIRE = ['nose', 'pytest', 'pytest-cov', 'pytest-mock',
-                 'pytest-timeout']
+                 'pytest-timeout', 'pytest-rerunfailures']
 EXTRAS_REQUIRE = {
     'optional': ['ephem', 'cython', 'netcdf4', 'nrel-pysam', 'numba',
                  'pvfactors', 'scipy', 'siphon', 'tables'],