8000 Remove automatic MIDC variable mapping (#721) · rlpappan/pvlib-python@15b1cbd · GitHub
[go: up one dir, main page]
Skip to content

Commit 15b1cbd

Browse files
lboemanwholmgren
lboeman
authored and
wholmgren
committed
Remove automatic MIDC variable mapping (pvlib#721)
* midc mapper * Remove default variable mapping from midc parser * whatsnew * add the mappings from solarforecastarbiter-core as an example * flake8
1 parent 252c1b2 commit 15b1cbd

File tree

3 files changed

+131
-84
lines changed

3 files changed

+131
-84
lines changed

‎docs/sphinx/source/whatsnew/v0.6.2.rst

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ API Changes
2626
* Deprecated :py:meth:`~pvlib.modelchain.ModelChain.prepare_inputs`
2727
assumption of clear sky if no irradiance fields were provided.
2828
(:issue:`705`, :issue:`707`)
29+
* Remove automatic column name mapping from :py:func:`~pvlib.iotools.read_midc`
30+
and :py:func:`~pvlib.iotools.read_midc_raw_data_from_nrel` and added
31+
optional keyword argument `variable_map` to map columns.
2932

3033
Enhancements
3134
~~~~~~~~~~~~

‎pvlib/iotools/midc.py

Lines changed: 109 additions & 67 deletions
10000
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,94 @@
11
"""Functions to read NREL MIDC data.
22
"""
3-
from functools import partial
43
import pandas as pd
54

6-
# VARIABLE_MAP is a dictionary mapping partial MIDC field names to their
7-
# pvlib names. See docstring of read_midc for description.
8-
9-
VARIABLE_MAP = {
10-
'Direct': 'dni',
11-
'Global': 'ghi',
12-
'Diffuse': 'dhi',
13-
'Airmass': 'airmass',
14-
'Azimuth Angle': 'solar_azimuth',
15-
'Zenith Angle': 'solar_zenith',
16-
'Air Temperature': 'temp_air',
17-
'Temperature': 'temp_air',
18-
'Dew Point Temp': 'temp_dew',
19-
'Relative Humidity': 'relative_humidity',
20-
}
5+
6+
# MIDC_VARIABLE_MAP maps some variables of interest at each MIDC site to their
7+
# pvlib counterparts. The mapping dictionary for a site can be found by looking
8+
# up the Site's id in the dictionary. It is not a comprehensive list, and may
9+
# not be the best fit for your application, but should serve as a base for
10+
# creating your own mappings.
11+
#
12+
# In particular, these mappings coincide with the raw ddata files.
13+
# All site's field list can be found at:
14+
# https://midcdmz.nrel.gov/apps/daily.pl?site=<SITE ID>&live=1
15+
# Where id is the key found in this dictionary
16+
MIDC_VARIABLE_MAP = {
17+
'BMS': {
18+
'Global CMP22 (vent/cor) [W/m^2]': 'ghi',
19+
'Direct NIP [W/m^2]': 'dni',
20+
'Diffuse CM22-1 (vent/cor) [W/m^2]': 'dhi',
21+
'Avg Wind Speed @ 6ft [m/s]': 'wind_speed',
22+
'Tower Dry Bulb Temp [deg C]': 'temp_air',
23+
'Tower RH [%]': 'relative_humidity'},
24+
'UOSMRL': {
25+
'Global CMP22 [W/m^2]': 'ghi',
26+
'Direct NIP [W/m^2]': 'dni',
27+
'Diffuse Schenk [W/m^2]': 'dhi',
28+
'Air Temperature [deg C]': 'temp_air',
29+
'Relative Humidity [%]': 'relative_humidity',
30+
'Avg Wind Speed @ 10m [m/s]': 'wind_speed'},
31+
'HSU': {
32+
'Global Horiz [W/m^2]': 'ghi',
33+
'Direct Normal (calc) [W/m^2]': 'dni',
34+
'Diffuse Horiz (band_corr) [W/m^2]': 'dhi'},
35+
'UTPASRL': {
36+
'Global Horizontal [W/m^2]': 'ghi',
37+
'Direct Normal [W/m^2]': 'dni',
38+
'Diffuse Horizontal [W/m^2]': 'dhi',
39+
'CHP1 Temp [deg C]': 'temp_air'},
40+
'UAT': {
41+
'Global Horiz (platform) [W/m^2]': 'ghi',
42+
'Direct Normal [W/m^2]': 'dni',
43+
'Diffuse Horiz [W/m^2]': 'dhi',
44+
'Air Temperature [deg C]': 'temp_air',
45+
'Rel Humidity [%]': 'relative_humidity',
46+
'Avg Wind Speed @ 3m [m/s]': 'wind_speed'},
47+
'STAC': {
48+
'Global Horizontal [W/m^2]': 'ghi',
49+
'Direct Normal [W/m^2]': 'dni',
50+
'Diffuse Horizontal [W/m^2]': 'dhi',
51+
'Avg Wind Speed @ 10m [m/s]': 'wind_speed',
52+
'Air Temperature [deg C]': 'temp_air',
53+
'Rel Humidity [%]': 'relative_humidity'},
54+
'UNLV': {
55+
'Global Horiz [W/m^2]': 'ghi',
56+
'Direct Normal [W/m^2]': 'dni',
57+
'Diffuse Horiz (calc) [W/m^2]': 'dhi',
58+
'Dry Bulb Temp [deg C]': 'temp_air',
59+
'Avg Wind Speed @ 30ft [m/s]': 'wind_speed'},
60+
'ORNL': {
61+
'Global Horizontal [W/m^2]': 'ghi',
62+
'Direct Normal [W/m^2]': 'dni',
63+
'Diffuse Horizontal [W/m^2]': 'dhi',
64+
'Air Temperature [deg C]': 'temp_air',
65+
'Rel Humidity [%]': 'relative_humidity',
66+
'Avg Wind Speed @ 42ft [m/s]': 'wind_speed'},
67+
'NELHA': {
68+
'Global Horizontal [W/m^2]': 'ghi',
69+
'Air Temperature [W/m^2]': 'temp_air',
70+
'Avg Wind Speed @ 10m [m/s]': 'wind_speed',
71+
'Rel Humidity [%]': 'relative_humidity'},
72+
'ULL': {
73+
'Global Horizontal [W/m^2]': 'ghi',
74+
'Direct Normal [W/m^2]': 'dni',
75+
'Diffuse Horizontal [W/m^2]': 'dhi',
76+
'Air Temperature [deg C]': 'temp_air',
77+
'Rel Humidity [%]': 'relative_humidity',
78+
'Avg Wind Speed @ 3m [m/s]': 'wind_speed'},
79+
'VTIF': {
80+
'Global Horizontal [W/m^2]': 'ghi',
81+
'Direct Normal [W/m^2]': 'dni',
82+
'Diffuse Horizontal [W/m^2]': 'dhi',
83+
'Air Temperature [deg C]': 'temp_air',
84+
'Avg Wind Speed @ 3m [m/s]': 'wind_speed',
85+
'Rel Humidity [%]': 'relative_humidity'},
86+
'NWTC': {
87+
'Global PSP [W/m^2]': 'ghi',
88+
'Temperature @ 2m [deg C]': 'temp_air',
89+
'Avg Wind Speed @ 2m [m/s]': 'wind_speed',
90+
'Relative Humidity [%]': 'relative_humidity'}}
91+
2192

2293
# Maps problematic timezones to 'Etc/GMT' for parsing.
2394

@@ -27,43 +98,6 @@
2798
}
2899

29100

30-
def map_midc_to_pvlib(variable_map, field_name):
31-
"""A mapper function to rename Dataframe columns to their pvlib counterparts.
32-
33-
Parameters
34-
----------
35-
variable_map: Dictionary
36-
A dictionary for mapping MIDC field name to pvlib name. See
37-
VARIABLE_MAP for default value and description of how to construct
38-
this argument.
39-
field_name: string
40-
The Column to map.
41-
42-
Returns
43-
-------
44-
label: string
45-
The pvlib variable name associated with the MIDC field or the input if
46-
a mapping does not exist.
47-
48-
Notes
49-
-----
50-
Will fail if field_name to be mapped matches an entry in VARIABLE_MAP and
51-
does not contain brackets. This should not be an issue unless MIDC file
52-
headers are updated.
53-
54-
"""
55-
new_field_name = field_name
56-
for midc_name, pvlib_name in variable_map.items():
57-
if field_name.startswith(midc_name):
58-
# extract the instrument and units field and then remove units
59-
instrument_units = field_name[len(midc_name):]
60-
units_index = instrument_units.find('[')
61-
instrument = instrument_units[:units_index - 1]
62-
new_field_name = pvlib_name + instrument.replace(' ', '_')
63-
break
64-
return new_field_name
65-
66-
67101
def format_index(data):
68102
"""Create DatetimeIndex for the Dataframe localized to the timezone provided
69103
as the label of the second (time) column.
@@ -114,7 +148,7 @@ def format_index_raw(data):
114148
return data
115149

116150

117-
def read_midc(filename, variable_map=VARIABLE_MAP, raw_data=False):
151+
def read_midc(filename, variable_map={}, raw_data=False):
118152
"""Read in National Renewable Energy Laboratory Measurement and
119153
Instrumentation Data Center [1]_ weather data.
120154
@@ -123,9 +157,9 @@ def read_midc(filename, variable_map=VARIABLE_MAP, raw_data=False):
123157
filename: string
124158
Filename or url of data to read.
125159
variable_map: dictionary
126-
Dictionary for mapping MIDC field names to pvlib names. See variable
127-
`VARIABLE_MAP` for default and Notes section below for a description of
128-
its format.
160+
Dictionary for mapping MIDC field names to pvlib names. Used to rename
161+
the columns of the resulting DataFrame. Does not map names by default.
162+
See Notes for an example.
129163
raw_data: boolean
130164
Set to true to use format_index_raw to correctly format the date/time
131165
columns of MIDC raw data files.
@@ -137,14 +171,19 @@ def read_midc(filename, variable_map=VARIABLE_MAP, raw_data=False):
137171
138172
Notes
139173
-----
140-
Keys of the `variable_map` dictionary should include the first part
141-
of a MIDC field name which indicates the variable being measured.
174+
The `variable_map` argument should map fields from MIDC data to pvlib
175+
names.
176+
177+
e.g. If a MIDC file contains the variable 'Global Horizontal [W/m^2]',
178+
passing the dictionary below will rename the column to 'ghi' in
179+
the returned Dataframe.
142180
143-
e.g. 'Global PSP [W/m^2]' is entered as a key of 'Global'
181+
{
182+
'Global Horizontal [W/m^2]': ghi,
183+
}
144184
145-
The 'PSP' indicating instrument is appended to the pvlib variable name
146-
after mapping to differentiate measurements of the same variable. For a
147-
full list of pvlib variable names see the `Variable Style Rules
185+
See the MIDC_VARIABLE_MAP for collection of mappings by site.
186+
For a full list of pvlib variable names see the `Variable Style Rules
148187
<https://pvlib-python.readthedocs.io/en/latest/variables_style_rules.html>`_.
149188
150189
Be sure to check the units for the variables you will use on the
@@ -160,12 +199,11 @@ def read_midc(filename, variable_map=VARIABLE_MAP, raw_data=False):
160199
data = format_index_raw(data)
161200
else:
162201
data = format_index(data)
163-
mapper = partial(map_midc_to_pvlib, variable_map)
164-
data = data.rename(columns=mapper)
202+
data = data.rename(columns=variable_map)
165203
return data
166204

167205

168-
def read_midc_raw_data_from_nrel(site, start, end):
206+
def read_midc_raw_data_from_nrel(site, start, end, variable_map={}):
169207
"""Request and read MIDC data directly from the raw data api.
170208
171209
Parameters
@@ -176,6 +214,10 @@ def read_midc_raw_data_from_nrel(site, start, end):
176214
Start date for requested data.
177215
end: datetime
178216
End date for requested data.
217+
variable_map: dict
218+
A dictionary mapping MIDC field names to pvlib names. Used to
219+
rename columns of the resulting DataFrame. See Notes of
220+
:py:func:`pvlib.iotools.read_midc` for example.
179221
180222
Returns
181223
-------
@@ -194,4 +236,4 @@ def read_midc_raw_data_from_nrel(site, start, end):
194236
'end': end.strftime('%Y%m%d')}
195237
endpoint = 'https://midcdmz.nrel.gov/apps/data_api.pl?'
196238
url = endpoint + '&'.join(['{}={}'.format(k, v) for k, v in args.items()])
197-
return read_midc(url, raw_data=True)
239+
return read_midc(url, variable_map=variable_map, raw_data=True)

‎pvlib/test/test_midc.py

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@
99
from pvlib.iotools import midc
1010

1111

12+
@pytest.fixture
13+
def test_mapping():
14+
return {
15+
'Direct Normal [W/m^2]': 'dni',
16+
'Global PSP [W/m^2]': 'ghi',
17+
'Rel Humidity [%]': 'relative_humidity',
18+
'Temperature @ 2m [deg C]': 'temp_air',
19+
'Non Existant': 'variable',
20+
}
21+
22+
1223
test_dir = os.path.dirname(
1324
os.path.abspath(inspect.getfile(inspect.currentframe())))
1425
midc_testfile = os.path.join(test_dir, '../data/midc_20181014.txt')
@@ -17,16 +28,6 @@
1728
'?site=UAT&begin=20181018&end=20181019')
1829

1930

20-
@pytest.mark.parametrize('field_name,expected', [
21-
('Temperature @ 2m [deg C]', 'temp_air_@_2m'),
22-
('Global PSP [W/m^2]', 'ghi_PSP'),
23-
('Temperature @ 50m [deg C]', 'temp_air_@_50m'),
24-
('Other Variable [units]', 'Other Variable [units]'),
25-
])
26-
def test_read_midc_mapper_function(field_name, expected):
27-
assert midc.map_midc_to_pvlib(midc.VARIABLE_MAP, field_name) == expected
28-
29-
3031
def test_midc_format_index():
3132
data = pd.read_csv(midc_testfile)
3233
data = midc.format_index(data)
@@ -57,17 +58,18 @@ def test_midc_format_index_raw():
5758
assert data.index[-1] == end
5859

5960

60-
def test_read_midc_var_mapping_as_arg():
61-
data = midc.read_midc(midc_testfile, variable_map=midc.VARIABLE_MAP)
62-
assert 'ghi_PSP' in data.columns
63-
assert 'temp_air_@_2m' in data.columns
64-
assert 'temp_air_@_50m' in data.columns
61+
def test_read_midc_var_mapping_as_arg(test_mapping):
62+
data = midc.read_midc(midc_testfile, variable_map=test_mapping)
63+
assert 'ghi' in data.columns
64+
assert 'temp_air' in data.columns
6565

6666

6767
@network
6868
def test_read_midc_raw_data_from_nrel():
6969
start_ts = pd.Timestamp('20181018')
7070
end_ts = pd.Timestamp('20181019')
71-
data = midc.read_midc_raw_data_from_nrel('UAT', start_ts, end_ts)
72-
assert 'dni_Normal' in data.columns
71+
var_map = midc.MIDC_VARIABLE_MAP['UAT']
72+
data = midc.read_midc_raw_data_from_nrel('UAT', start_ts, end_ts, var_map)
73+
for k, v in var_map.items():
74+
assert v in data.columns
7375
assert data.index.size == 2880

0 commit comments

Comments
 (0)
0