forked from QuantConnect/Lean
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathLeanDataReader.cs
More file actions
193 lines (177 loc) · 8.14 KB
/
LeanDataReader.cs
File metadata and controls
193 lines (177 loc) · 8.14 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Ionic.Zip;
using NodaTime;
using QuantConnect.Data;
using QuantConnect.Logging;
using QuantConnect.Securities;
using QuantConnect.Util;
namespace QuantConnect.ToolBox
{
/// <summary>
/// This class reads data directly from disk and returns the data without the data
/// entering the Lean data enumeration stack
/// </summary>
public class LeanDataReader
{
private readonly DateTime _date;
private readonly string _zipPath;
private readonly string _zipentry;
private readonly SubscriptionDataConfig _config;
/// <summary>
/// The LeanDataReader constructor
/// </summary>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="symbol">The <see cref="Symbol"/> that will be read</param>
/// <param name="resolution">The <see cref="Resolution"/> that will be read</param>
/// <param name="date">The <see cref="DateTime"/> that will be read</param>
/// <param name="dataFolder">The root data folder</param>
public LeanDataReader(SubscriptionDataConfig config, Symbol symbol, Resolution resolution, DateTime date, string dataFolder)
{
_date = date;
_zipPath = LeanData.GenerateZipFilePath(dataFolder, symbol, date, resolution, config.TickType);
_zipentry = LeanData.GenerateZipEntryName(symbol, date, resolution, config.TickType);
_config = config;
}
/// <summary>
/// Initialize a instance of LeanDataReader from a path to a zipped data file.
/// It also supports declaring the zip entry CSV file for options and futures.
/// </summary>
/// <param name="filepath">Absolute or relative path to a zipped data file, optionally the zip entry file can be declared by using '#' as separator.</param>
/// <example>
/// var dataReader = LeanDataReader("../relative/path/to/file.zip")
/// var dataReader = LeanDataReader("absolute/path/to/file.zip#zipEntry.csv")
/// </example>
public LeanDataReader(string filepath)
{
Symbol symbol;
DateTime date;
Resolution resolution;
string zipEntry = null;
var isFutureOrOption = filepath.Contains('#', StringComparison.InvariantCulture);
if (isFutureOrOption)
{
zipEntry = filepath.Split('#')[1];
filepath = filepath.Split('#')[0];
}
var fileInfo = new FileInfo(filepath);
if (!LeanData.TryParsePath(fileInfo.FullName, out symbol, out date, out resolution, out var tickType, out var dataType))
{
throw new ArgumentException($"File {filepath} cannot be parsed.");
}
if (isFutureOrOption)
{
symbol = LeanData.ReadSymbolFromZipEntry(symbol, resolution, zipEntry);
}
var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
var dataTimeZone = marketHoursDataBase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
var exchangeTimeZone = marketHoursDataBase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone;
var config = new SubscriptionDataConfig(dataType, symbol, resolution,
dataTimeZone, exchangeTimeZone, tickType: tickType,
fillForward: false, extendedHours: true, isInternalFeed: true);
_date = date;
_zipPath = fileInfo.FullName;
_zipentry = zipEntry;
_config = config;
}
/// <summary>
/// Enumerate over the tick zip file and return a list of BaseData.
/// </summary>
/// <returns>IEnumerable of ticks</returns>
public IEnumerable<BaseData> Parse()
{
if (!File.Exists(_zipPath))
{
Log.Error($"LeanDataReader.Parse(): File does not exist: {_zipPath}");
yield break;
}
var factory = (BaseData) ObjectActivator.GetActivator(_config.Type).Invoke(new object[0]);
if (_config.Type.ImplementsStreamReader())
{
using (var zip = new ZipFile(_zipPath))
{
foreach (var zipEntry in zip.Where(x => _zipentry == null || string.Equals(x.FileName, _zipentry, StringComparison.OrdinalIgnoreCase)))
{
// we get the contract symbol from the zip entry if not already provided with the zip entry
var symbol = _config.Symbol;
if(_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, zipEntry.FileName);
}
using (var entryReader = new StreamReader(zipEntry.OpenReader()))
{
while (!entryReader.EndOfStream)
{
var dataPoint = factory.Reader(_config, entryReader, _date, false);
dataPoint.Symbol = symbol;
yield return dataPoint;
}
}
}
}
}
// for futures and options if no entry was provided we just read all
else if (_zipentry == null && (_config.SecurityType == SecurityType.Future || _config.SecurityType.IsOption()))
{
foreach (var entries in Compression.Unzip(_zipPath))
{
// we get the contract symbol from the zip entry
var symbol = LeanData.ReadSymbolFromZipEntry(_config.Symbol, _config.Resolution, entries.Key);
foreach (var line in entries.Value)
{
var dataPoint = factory.Reader(_config, line, _date, false);
dataPoint.Symbol = symbol;
yield return dataPoint;
}
}
}
else
{
ZipFile zipFile;
using (var unzipped = Compression.Unzip(_zipPath, _zipentry, out zipFile))
{
if (unzipped == null)
yield break;
string line;
while ((line = unzipped.ReadLine()) != null)
{
yield return factory.Reader(_config, line, _date, false);
}
}
zipFile.Dispose();
}
}
/// <summary>
/// Returns the data time zone
/// </summary>
/// <returns><see cref="NodaTime.DateTimeZone"/> representing the data timezone</returns>
public DateTimeZone GetDataTimeZone()
{
return _config.DataTimeZone;
}
/// <summary>
/// Returns the Exchange time zone
/// </summary>
/// <returns><see cref="NodaTime.DateTimeZone"/> representing the exchange timezone</returns>
public DateTimeZone GetExchangeTimeZone()
{
return _config.ExchangeTimeZone;
}
}
}