Convert to get_cams to add support for CAMS Radiation

pvlib · kandersolar · Jun 13, 2021 · Feb 22, 2021 · Feb 22, 2021 · Feb 22, 2021
commit 673bf3baa44fb1fc844ad232b5de927273904cb0
diff --git a/pvlib/iotools/cams.py b/pvlib/iotools/cams.py
@@ -8,28 +8,29 @@
 import io
 
 
-MCCLEAR_COLUMNS = ['Observation period', 'TOA', 'Clear sky GHI',
-                   'Clear sky BHI', 'Clear sky DHI', 'Clear sky BNI']
+CAMS_INTEGRATED_COLUMNS = ['TOA', 'Clear sky GHI', 'Clear sky BHI',
+                           'Clear sky DHI', 'Clear sky BNI',
+                           'GHI', 'BHI', 'DHI', 'BNI', 'GHI no corr',
+                           'BHI no corr', 'DHI no corr', 'BNI no corr']
 
-MCCLEAR_VERBOSE_COLUMNS = ['sza', 'summer/winter split', 'tco3', 'tcwv',
-                           'AOD BC', 'AOD DU', 'AOD SS', 'AOD OR', 'AOD SU',
-                           'AOD NI', 'AOD AM', 'alpha', 'Aerosol type',
-                           'fiso', 'fvol', 'fgeo', 'albedo']
-
-# Dictionary mapping CAMS MCCLEAR variables to pvlib names
-MCCLEAR_VARIABLE_MAP = {
+# Dictionary mapping CAMS McClear and Radiation variables to pvlib names
+CAMS_VARIABLE_MAP = {
     'TOA': 'ghi_extra',
     'Clear sky GHI': 'ghi_clear',
     'Clear sky BHI': 'bhi_clear',
     'Clear sky DHI': 'dhi_clear',
     'Clear sky BNI': 'dni_clear',
+    'GHI': 'ghi',
+    'BHI': 'bhi',
+    'DHI': 'dhi',
+    'BNI': 'dni',
     'sza': 'solar_zenith',
 }
 
 
 # Dictionary mapping Python time steps to CAMS time step format
-TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H', '1d': 'P01D',
-                  '1M': 'P01M'}
+TIME_STEPS_MAP = {'1min': 'PT01M', '15min': 'PT15M', '1h': 'PT01H',
+                  '1d': 'P01D', '1M': 'P01M'}
 
 TIME_STEPS_IN_HOURS = {'1min': 1/60, '15min': 15/60, '1h': 1, '1d': 24}
 
@@ -40,16 +41,16 @@
                                  '0 year 1 month 0 day 0 h 0 min 0 s': '1M'}
 
 
-def get_cams_mcclear(start_date, end_date, latitude, longitude, email,
-                     altitude=None, time_step='1h', time_ref='UT',
-                     integrated=False, label=None, verbose=False,
-                     map_variables=True, server='www.soda-is.com'):
+def get_cams(start_date, end_date, latitude, longitude, email,
+             service='mcclear', altitude=None, time_step='1h', time_ref='UT',
+             verbose=False, integrated=False, label=None, map_variables=True,
+             server='www.soda-is.com'):
     """
-    Retrieve time-series of clear-sky global, beam, and diffuse radiation
-    anywhere in the world from CAMS McClear [1]_ using the WGET service [2]_.
+    Retrieve time-series of radiation and/or clear-sky global, beam, and
+    diffuse radiation CAMS [2]_ using the WGET service [3]_.
 
 
-    Geographical coverage: wordwide
+    Geographical coverage: -66 to 66 latitude/longitude (radiation) and wordwide (McClear)
     Time coverage: 2004-01-01 to two days ago
     Access: free, but requires registration, see [1]_
     Requests: max. 100 per day
@@ -70,26 +71,34 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email,
         NASA SRTM database
     email: str
         Email address linked to a SoDa account
+    service: {'mcclear', 'cams_radiation'}
+        Specify which whether to retrieve CAMS Radiation or McClear parameters
     time_step: str, {'1min', '15min', '1h', '1d', '1M'}, default: '1h'
         Time step of the time series, either 1 minute, 15 minute, hourly,
         daily, or monthly.
     time_reference: str, {'UT', 'TST'}, default: 'UT'
         'UT' (universal time) or 'TST' (True Solar Time)
+    verbose: boolean, default: False
+        Verbose mode outputs additional parameters (aerosols). Only avaiable
+        for 1 minute and universal time. See [1] for parameter description.
     integrated: boolean, default False
         Whether to return integrated irradiation values (Wh/m^2) from CAMS or
         average irradiance values (W/m^2) as is more commonly used
     label: {‘right’, ‘left’}, default: None
         Which bin edge label to label bucket with. The default is ‘left’ for
         all frequency offsets except for ‘M’ which has a default of ‘right’.
-    verbose: boolean, default: False
-        Verbose mode outputs additional parameters (aerosols). Only avaiable
-        for 1 minute and universal time. See [1] for parameter description.
     map_variables: bool, default: True
         When true, renames columns of the Dataframe to pvlib variable names
-        where applicable. See variable MCCLEAR_VARIABLE_MAP.
+        where applicable. See variable CAMS_VARIABLE_MAP.
     server: str, default: 'www.soda-is.com'
         Main server (www.soda-is.com) or backup mirror server (pro.soda-is.com)
 
+    Returns
+    -------
+    data: pandas.DataFrame
+        timeseries data, see Notes for columns
+    meta: dict
+        metadata for the requested time-series
 
     Notes
     ----------
@@ -103,46 +112,56 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email,
     Observation period       str     Beginning/end of time period
     TOA, ghi_extra           float   Horizontal radiation at top of atmosphere
     Clear sky GHI, ghi_clear float   Clear sky global radiation on horizontal
-    Clear sky BHI, bhi_clear float   Clear sky beam radiation on horizontal
+    Clear sky BHI, bhi       float   Clear sky beam radiation on horizontal
     Clear sky DHI, dhi_clear float   Clear sky diffuse radiation on horizontal
     Clear sky BNI, dni_clear float   Clear sky beam radiation normal to sun
+    GHI, ghi*                float   Global horizontal radiation
+    BHI, bhi*                float   Beam (direct) radiation on horizontal
+    DHI, dhi*                float   Diffuse horizontal radiation
+    BNI, dni*                float   Beam (direct) radiation normal to the sun
+    Reliability*             float   Fraction of reliable data in summarization
     =======================  ======  ==========================================
 
-    For the returned units see the integrated argument. For description of
-    additional output parameters in verbose mode, see [1].
+    *Parameters only returned if service='cams_radiation'. For description of
+    additional output parameters in verbose mode, see [1]_ and [2]_.
+
+    The returned units for the radiation parameters depends on the integrated
+    argument, i.e. integrated=False returns units of W/m2, whereas
+    integrated=True returns units of Wh/m2. 
 
     Note that it is recommended to specify the latitude and longitude to at
     least the fourth decimal place.
 
     Variables corresponding to standard pvlib variables are renamed,
     e.g. `sza` becomes `solar_zenith`. See the
-    `pvlib.iotools.cams.MCCLEAR_VARIABLE_MAP` dict for the complete mapping.
+    `pvlib.iotools.cams.CAMS_VARIABLE_MAP` dict for the complete mapping.
 
     See Also
     --------
-    pvlib.iotools.read_cams_mcclear, pvlib.iotools.parse_cams_mcclear
+    pvlib.iotools.read_cams, pvlib.iotools.parse_cams
 
     References
     ----------
-    .. [1] `CAMS McClear Service Info
+    .. [1] `CAMS Radiation Service Info
+       <http://www.soda-pro.com/web-services/radiation/cams-radiation-service/info>`_
+    .. [2] `CAMS McClear Service Info
        <http://www.soda-pro.com/web-services/radiation/cams-mcclear/info>`_
-    .. [2] `CAMS McClear Automatic Access
+    .. [3] `CAMS McClear Automatic Access
        <http://www.soda-pro.com/help/cams-services/cams-mcclear-service/automatic-access>`_
     """
 
     if time_step in TIME_STEPS_MAP.keys():
         time_step_str = TIME_STEPS_MAP[time_step]
     else:
         print('WARNING: time step not recognized, 1 hour time step used!')
-        time_step_str = 'PT01H'
+        time_step, time_step_str = '1h', 'PT01H'
 
-    names = MCCLEAR_COLUMNS
-    if verbose:
-        if (time_step == '1min') & (time_ref == 'UT'):
-            names += MCCLEAR_VERBOSE_COLUMNS
-        else:
-            verbose = False
-            print("Verbose mode only supports 1 min. UT time series!")
+    if (verbose is True) & ((time_step != '1min') | (time_ref != 'UT')):
+        verbose = False
+        print("Verbose mode only supports 1 min. UT time series!")
+
+    # Format verbose variable to the required format: {'true', 'false'}
+    verbose = str(verbose).lower()
 
     if altitude is None:  # Let SoDa get elevation from the NASA SRTM database
         altitude = -999
@@ -152,52 +171,72 @@ def get_cams_mcclear(start_date, end_date, latitude, longitude, email,
     end_date = end_date.strftime('%Y-%m-%d')
 
     email = email.replace('@', '%2540')  # Format email address
-
-    # Format verbose variable to the required format: {'true', 'false'}
-    verbose = str(verbose).lower()
+    service = 'get_{}'.format(service)  # Format CAMS service string
 
     # Manual format the request url, due to uncommon usage of & and ; in url
     url = ("http://{}/service/wps?Service=WPS&Request=Execute&"
-           "Identifier=get_mcclear&version=1.0.0&RawDataOutput=irradiation&"
+           "Identifier={}&version=1.0.0&RawDataOutput=irradiation&"
            "DataInputs=latitude={};longitude={};altitude={};"
            "date_begin={};date_end={};time_ref={};summarization={};"
            "username={};verbose={}"
-           ).format(server, latitude, longitude, altitude, start_date,
+           ).format(server, service, latitude, longitude, altitude, start_date,
                     end_date, time_ref, time_step_str, email, verbose)
 
     res = requests.get(url)
 
     # Invalid requests returns helpful XML error message
     if res.headers['Content-Type'] == 'application/xml':
-        print('REQUEST ERROR MESSAGE:')
-        print(res.text.split('ows:ExceptionText')[1][1:-2])
-
+        errors = res.text.split('ows:ExceptionText')[1][1:-2]
+        raise requests.HTTPError(errors, response=res)
     # Check if returned file is a csv data file
     elif res.headers['Content-Type'] == 'application/csv':
         fbuf = io.StringIO(res.content.decode('utf-8'))
-        data, meta = parse_cams_mcclear(fbuf, integrated=integrated,
-                                        label=label,
-                                        map_variables=map_variables)
+        data, meta = parse_cams(fbuf, integrated=integrated, label=label,
+                                map_variables=map_variables)
         return data, meta
     else:
-        print('Error in recognizing the file content occurred!')
+        print('Error! File content type not recognized.')
 
 
-def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True):
+def parse_cams(fbuf, integrated=False, label=None, map_variables=True):
     """
-    Parse a CAMS McClear file. CAMS McClear is described in [1]_.
+    Parse a CAMS Radiation or McClear file. The CAMS Radiation and McClear
+    services are described in [1]_ and [2]_.
+
+    Parameters
+    ----------
+    fbuf: file-like object
+        File-like object containing data to read.
+    integrated: boolean, default False
+        Whether to return integrated irradiation values (Wh/m^2) from CAMS or
+        average irradiance values (W/m^2) as is more commonly used
+    label: {‘right’, ‘left’}, default: None
+        Which bin edge label to label bucket with. The default is ‘left’ for
+        all frequency offsets except for ‘M’ which has a default of ‘right’.
+    map_variables: bool, default: True
+        When true, renames columns of the Dataframe to pvlib variable names
+        where applicable. See variable CAMS_VARIABLE_MAP.
+
+    Returns
+    -------
+    data: pandas.DataFrame
+        timeseries data from CAMS Radiation or McClear
+    meta: dict
+        metadata of the CAMS Radiation or McClear time-series
 
     See Also
     --------
-    pvlib.iotools.read_cams_mcclear, pvlib.iotools.get_cams_mcclear
+    pvlib.iotools.read_cams, pvlib.iotools.get_cams
 
     References
     ----------
-    .. [1] `CAMS McClear Service Info
+    .. [1] `CAMS Radiation Service Info
+       <http://www.soda-pro.com/web-services/radiation/cams-radiation-service/info>`_
+    .. [2] `CAMS McClear Service Info
        <http://www.soda-pro.com/web-services/radiation/cams-mcclear/info>`_
     """
     meta = {}
-    # Initial lines of the file contain meta-data, which all start with #
+    # Initial lines starting with # contain meta-data
     while True:
         line = fbuf.readline().rstrip('\n')
         if line.startswith('# Observation period'):
@@ -213,7 +252,7 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True):
     meta['Longitude (positive East, ISO 19115)'] = \
         float(meta['Longitude (positive East, ISO 19115)'])
     meta['Altitude (m)'] =
9E7A
 float(meta['Altitude (m)'])
-    meta['Clear sky radiation unit'] = {True:'Wh/m2', False:'W/m2'}[integrated]
+    meta['Unit for radiation'] = {True:'Wh/m2', False:'W/m2'}[integrated]
 
     # Determine the time_step from the meta-data dictionary
     time_step = SUMMATION_PERIOD_TO_TIME_STEP[
@@ -242,8 +281,8 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True):
     if (time_step == '1M') & (label != 'left'):
         data.index = data.index - pd.Timedelta(days=1)
 
-    if not integrated:  # Convert from Wh/m2 to W/m2
-        integrated_cols = MCCLEAR_COLUMNS[1:6]
+    if not integrated:  # Convert radiation values from Wh/m2 to W/m2
+        integrated_cols = [c for c in CAMS_INTEGRATED_COLUMNS if c in data.columns]
 
         if time_step == '1M':
             time_delta = (pd.to_datetime(obs_period.str[1])
@@ -256,37 +295,48 @@ def parse_cams_mcclear(fbuf, integrated=False, label=None, map_variables=True):
                                      TIME_STEPS_IN_HOURS[time_step])
 
     if map_variables:
-        data = data.rename(columns=MCCLEAR_VARIABLE_MAP)
+        data = data.rename(columns=CAMS_VARIABLE_MAP)
 
     return data, meta
 
 
-def read_cams_mcclear(filename):
+def read_cams(filename, integrated=False, label=None, map_variables=True):
     """
-    Read a CAMS McClear file. CAMS McClear is described in [1]_.
+    Read a CAMS Radiation or McClear file. CAMS radiation and McClear is
+    described in [1]_ and [2]_, respectively.
 
     Parameters
     ----------
     filename: str
         Filename of a file containing data to read.
+    integrated: boolean, default False
+        Whether to return integrated irradiation values (Wh/m^2) from CAMS or
+        average irradiance values (W/m^2) as is more commonly used
+    label: {‘right’, ‘left’}, default: None
+        Which bin edge label to label bucket with. The default is ‘left’ for
+        all frequency offsets except for ‘M’ which has a default of ‘right’.
+    map_variables: bool, default: True
+        When true, renames columns of the Dataframe to pvlib variable names
+        where applicable. See variable CAMS_VARIABLE_MAP.
 
     Returns
     -------
-    data : pandas.DataFrame
-        timeseries data from CAMS McClear
-    meta : dict
-        metadata from CAMS McClear, see
-        :func:`pvlib.iotools.parse_cams_mcclear` for fields
+    data: pandas.DataFrame
+        timeseries data from CAMS Radiation or McClear
+    meta: dict
+        metadata from CAMS Radiation or McClear
 
     See Also
     --------
-    pvlib.iotools.parse_cams_mcclear, pvlib.iotools.get_cams_mcclear
+    pvlib.iotools.parse_cams, pvlib.iotools.get_cams
 
     References
     ----------
-    .. [1] `CAMS McClear Service Info
+    .. [1] `CAMS Radiation Service Info
+       <http://www.soda-pro.com/web-services/radiation/cams-radiation-service/info>`_
+    .. [2] `CAMS McClear Service Info
        <http://www.soda-pro.com/web-services/radiation/cams-mcclear/info>`_
     """
     with open(str(filename), 'r') as fbuf:
-        content = parse_cams_mcclear(fbuf)
+        content = parse_cams(fbuf)
     return content