8000 Avoid IEEE754 converting fraction of seconds · PyMySQL/PyMySQL@1df94d3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 1df94d3

Browse files
timonwongtimonwong
committed
Avoid IEEE754 converting fraction of seconds
Due to the limitations of IEEE754, we may lost precision when converting string back to datetime objects, for example: `float('0.511581') * 1e6`, gives us `0.511580`, which is unexpected. In order to address that issue, use of `float` is avoided
1 parent 2525d0a commit 1df94d3

File tree

2 files changed

+86
-26
lines changed

2 files changed

+86
-26
lines changed

‎pymysql/converters.py

Lines changed: 42 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import datetime
44
from decimal import Decimal
5+
import re
56
import time
67

78
from .constants import FIELD_TYPE, FLAG
@@ -145,6 +146,16 @@ def escape_date(obj, mapping=None):
145146
def escape_struct_time(obj, mapping=None):
146147
return escape_datetime(datetime.datetime(*obj[:6]))
147148

149+
def _convert_second_fraction(s):
150+
if not s:
151+
return 0
152+
# Pad zeros to ensure the fraction length in microseconds
153+
s = s.ljust(6, '0')
154+
return int(s[:6])
155+
156+
DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\ 10000 d{1,6}))?")
157+
158+
148159
def convert_datetime(obj):
149160
"""Returns a DATETIME or TIMESTAMP column value as a datetime object:
150161
@@ -163,23 +174,20 @@ def convert_datetime(obj):
163174
"""
164175
if not PY2 and isinstance(obj, (bytes, bytearray)):
165176
obj = obj.decode('ascii')
166-
if ' ' in obj:
167-
sep = ' '
168-
elif 'T' in obj:
169-
sep = 'T'
170-
else:
177+
178+
m = DATETIME_RE.match(obj)
179+
if not m:
171180
return convert_date(obj)
172181

173182
try:
174-
ymd, hms = obj.split(sep, 1)
175-
usecs = '0'
176-
if '.' in hms:
177-
hms, usecs = hms.split('.')
178-
usecs = float('0.' + usecs) * 1e6
179-
return datetime.datetime(*[ int(x) for x in ymd.split('-')+hms.split(':')+[usecs] ])
183+
groups = list(m.groups())
184+
groups[-1] = _convert_second_fraction(groups[-1])
185+
return datetime.datetime(*[ int(x) for x in groups ])
180186
except ValueError:
181187
return convert_date(obj)
182188

189+
TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
190+
183191

184192
def convert_timedelta(obj):
185193
"""Returns a TIME column as a timedelta object:
@@ -200,16 +208,17 @@ def convert_timedelta(obj):
200208
"""
201209
if not PY2 and isinstance(obj, (bytes, bytearray)):
202210
obj = obj.decode('ascii')
211+
212+
m = TIMEDELTA_RE.match(obj)
213+
if not m:
214+
return None
215+
203216
try:
204-
microseconds = 0
205-
if "." in obj:
206-
(obj, tail) = obj.split('.')
207-
microseconds = float('0.' + tail) * 1e6
208-
hours, minutes, seconds = obj.split(':')
209-
negate = 1
210-
if hours.startswith("-"):
211-
hours = hours[1:]
212-
negate = -1
217+
groups = list(m.groups())
218+
groups[-1] = _convert_second_fraction(groups[-1])
219+
negate = -1 if groups[0] else 1
220+
hours, minutes, seconds, microseconds = groups[1:]
221+
213222
tdelta = datetime.timedelta(
214223
hours = int(hours),
215224
minutes = int(minutes),
@@ -220,6 +229,9 @@ def convert_timedelta(obj):
220229
except ValueError:
221230
return None
222231

232+
TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
233+
234+
223235
def convert_time(obj):
224236
"""Returns a TIME column as a time object:
225237
@@ -244,17 +256,21 @@ def convert_time(obj):
244256
"""
245257
if not PY2 and isinstance(obj, (bytes, bytearray)):
246258
obj = obj.decode('ascii')
259+
260+
m = TIME_RE.match(obj)
261+
if not m:
262+
return None
263+
247264
try:
248-
microseconds = 0
249-
if "." in obj:
250-
(obj, tail) = obj.split('.')
251-
microseconds = float('0.' + tail) * 1e6
252-
hours, minutes, seconds = obj.split(':')
265+
groups = list(m.groups())
266+
groups[-1] = _convert_second_fraction(groups[-1])
267+
hours, minutes, seconds, microseconds = groups
253268
return datetime.time(hour=int(hours), minute=int(minutes),
254269
second=int(seconds), microsecond=int(microseconds))
255270
except ValueError:
256271
return None
257272

273+
258274
def convert_date(obj):
259275
"""Returns a DATE column as a date object:
260276
@@ -324,7 +340,7 @@ def through(x):
324340
#def convert_bit(b):
325341
# b = "\x00" * (8 - len(b)) + b # pad w/ zeroes
326342
# return struct.unpack(">Q", b)[0]
327-
#
343+
#
328344
# the snippet above is right, but MySQLdb doesn't process bits,
329345
# so we shouldn't either
330346
convert_bit = through

‎pymysql/tests/test_converters.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import datetime
12
from unittest import TestCase
23

34
from pymysql._compat import PY2
@@ -21,3 +22,46 @@ def test_escape_string_bytes(self):
2122
converters.escape_string(b"foo\nbar"),
2223
b"foo\\nbar"
2324
)
25+
26+
def test_convert_datetime(self):
27+
expected = datetime.datetime(2007, 2, 24, 23, 6, 20)
28+
dt = converters.convert_datetime('2007-02-24 23:06:20')
29+
self.assertEqual(dt, expected)
30+
31+
def test_convert_datetime_with_fsp(self):
32+
expected = datetime.datetime(2007, 2, 24, 23, 6, 20, 511581)
33+
dt = converters.convert_datetime('2007-02-24 23:06:20.511581')
34+
self.assertEqual(dt, expected)
35+
36+
def _test_convert_timedelta(self, with_negate=False, with_fsp=False):
37+
d = {'hours': 789, 'minutes': 12, 'seconds': 34}
38+
s = '%(hours)s:%(minutes)s:%(seconds)s' % d
39+
if with_fsp:
40+
d['microseconds'] = 511581
41+
s += '.%(microseconds)s' % d
42+
43+
expected = datetime.timedelta(**d)
44+
if with_negate:
45+
expected = -expected
46+
s = '-' + s
47+
48+
tdelta = converters.convert_timedelta(s)
49+
self.assertEqual(tdelta, expected)
50+
51+
def test_convert_timedelta(self):
52+
self._test_convert_timedelta(with_negate=False, with_fsp=False)
53+
self._test_convert_timedelta(with_negate=True, with_fsp=False)
54+
55+
def test_convert_timedelta_with_fsp(self):
56+
self._test_convert_timedelta(with_negate=False, with_fsp=True)
57+
self._test_convert_timedelta(with_negate=False, with_fsp=True)
58+
59+
def test_convert_time(self):
60+
expected = datetime.time(23, 6, 20)
61+
time_obj = converters.convert_time('23:06:20')
62+
self.assertEqual(time_obj, expected)
63+
64+
def test_convert_time_with_fsp(self):
65+
expected = datetime.time(23, 6, 20, 511581)
66+
time_obj = converters.convert_time('23:06:20.511581')
67+
self.assertEqual(time_obj, expected)

0 commit comments

Comments
 (0)
0