8000 Avoid IEEE754 converting fraction of seconds (#487) · codepongo/PyMySQL@3397c23 · GitHub
[go: up one dir, main page]
Skip to content

Commit 3397c23

Browse files
timonwongmethane
timonwong
authored and
methane
committed
Avoid IEEE754 converting fraction of seconds (PyMySQL#487)
Due to the limitations of IEEE754, we may lost precision when converting string back to datetime objects, for example: `float('0.511581') * 1e6`, gives us `0.511580`, which is unexpected. In order to address that issue, use of `float` is avoided
1 parent 2525d0a commit 3397c23

File tree

2 files changed

+86
-26
lines changed

2 files changed

+86
-26
lines changed

‎pymysql/converters.py

Lines changed: 42 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import datetime
44
from decimal import Decimal
5+
import re
56
import time
67

78
from .constants import FIELD_TYPE, FLAG
@@ -145,6 +146,16 @@ def escape_date(obj, mapping=None):
145146
def escape_struct_time(obj, mapping=None):
146147
return escape_datetime(datetime.datetime(*obj[:6]))
147148

149+
def _convert_second_fraction(s):
150+
if not s:
151+
return 0
152+
# Pad zeros to ensure the fraction length in microseconds
153+
s = s.ljust(6, '0')
154+
return int(s[:6])
155+
156+
DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
157+
158+
148159
def convert_datetime(obj):
149160
"""Returns a DATETIME or TIMESTAMP column value as a datetime object:
150161
@@ -163,23 +174,20 @@ def convert_datetime(obj):
163174
"""
164175
if not PY2 and isinstance(obj, (bytes, bytearray)):
165176
obj = obj.decode('ascii')
166-
if ' ' in obj:
167-
sep = ' '
168-
elif 'T' in obj:
169-
sep = 'T'
170-
else:
177+
178+
m = DATETIME_RE.match(obj)
179+
if not m:
171180
return convert_date(obj)
172181

173182
try:
174-
ymd, hms = obj.split(sep, 1)
175-
usecs = '0'
176-
if '.' in hms:
177-
hms, usecs = hms.split('.')
178-
usecs = float('0.' + usecs) * 1e6
179-
return datetime.datetime(*[ int(x) for x in ymd.split('-')+hms.split(':')+[usecs] ])
183+
groups = list(m.groups())
184+
groups[-1] = _convert_second_fraction(groups[-1])
185+
return datetime.datetime(*[ int(x) for x in groups ])
180186
except ValueError:
181187
return convert_date(obj)
182188

189+
TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
190+
183191

184192
def convert_timedelta(obj):
185193
"""Returns a TIME column as a timedelta object:
@@ -200,16 +208,17 @@ def convert_timedelta(obj):
200208
"""
201209
if not PY2 and isinstance(obj, (bytes, bytearray)):
202210
obj = obj.decode('ascii')
211+
212+
m = TIMEDELTA_RE.match(obj)
213+
if not m:
214+
return None
215+
203216
try:
204-
microseconds = 0
205-
if "." in obj:
206-
(obj, tail) = obj.split('.')
207-
microseconds = float('0.' + tail) * 1e6
208-
hours, minutes, seconds = obj.split(':')
209-
negate = 1
210-
if hours.startswith("-"):
211-
hours = hours[1:]
212-
negate = -1
217+
groups = list(m.groups())
218+
groups[-1] = _convert_second_fraction(groups[-1])
219+
negate = -1 if groups[0] else 1
220+
hours, minutes, seconds, microseconds = groups[1:]
221+
213222
tdelta = datetime.timedelta(
214223
hours = int(hours),
215224
minutes = int(minutes),
@@ -220,6 +229,9 @@ def convert_timedelta(obj):
220229
except ValueError:
221230
return None
222231

232+
TIME_RE< B41A /span> = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
233+
234+
223235
def convert_time(obj):
224236
"""Returns a TIME column as a time object:
225237
@@ -244,17 +256,21 @@ def convert_time(obj):
244256
"""
245257
if not PY2 and isinstance(obj, (bytes, bytearray)):
246258
obj = obj.decode('ascii')
259+
260+
m = TIME_RE.match(obj)
261+
if not m:
262+
return None
263+
247264
try:
248-
microseconds = 0
249-
if "." in obj:
250-
(obj, tail) = obj.split('.')
251-
microseconds = float('0.' + tail) * 1e6
252-
hours, minutes, seconds = obj.split(':')
265+
groups = list(m.groups())
266+
groups[-1] = _convert_second_fraction(groups[-1])
267+
hours, minutes, seconds, microseconds = groups
253268
return datetime.time(hour=int(hours), minute=int(minutes),
254269
second=int(seconds), microsecond=int(microseconds))
255270
except ValueError:
256271
return None
257272

273+
258274
def convert_date(obj):
259275
"""Returns a DATE column as a date object:
260276
@@ -324,7 +340,7 @@ def through(x):
324340
#def convert_bit(b):
325341
# b = "\x00" * (8 - len(b)) + b # pad w/ zeroes
326342
# return struct.unpack(">Q", b)[0]
327-
#
343+
#
328344
# the snippet above is right, but MySQLdb doesn't process bits,
329345
# so we shouldn't either
330346
convert_bit = through

‎pymysql/tests/test_converters.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import datetime
12
from unittest import TestCase
23

34
from pymysql._compat import PY2
@@ -21,3 +22,46 @@ def test_escape_string_bytes(self):
2122
converters.escape_string(b"foo\nbar"),
2223
b"foo\\nbar"
2324
)
25+
26+
def test_convert_datetime(self):
27+
expected = datetime.datetime(2007, 2, 24, 23, 6, 20)
28+
dt = converters.convert_datetime('2007-02-24 23:06:20')
29+
self.assertEqual(dt, expected)
30+
31+
def test_convert_datetime_with_fsp(self):
32+
expected = datetime.datetime(2007, 2, 24, 23, 6, 20, 511581)
33+
dt = converters.convert_datetime('2007-02-24 23:06:20.511581')
34+
self.assertEqual(dt, expected)
35+
36+
def _test_convert_timedelta(self, with_negate=False, with_fsp=False):
37+
d = {'hours': 789, 'minutes': 12, 'seconds': 34}
38+
s = '%(hours)s:%(minutes)s:%(seconds)s' % d
39+
if with_fsp:
40+
d['microseconds'] = 511581
41+
s += '.%(microseconds)s' % d
42+
43+
expected = datetime.timedelta(**d)
44+
if with_negate:
45+
expected = -expected
46+
s = '-' + s
47+
48+
tdelta = converters.convert_timedelta(s)
49+
self.assertEqual(tdelta, expected)
50+
51+
def test_convert_timedelta(self):
52+
self._test_convert_timedelta(with_negate=False, with_fsp=False)
53+
self._test_convert_timedelta(with_negate=True, with_fsp=False)
54+
55+
def test_convert_timedelta_with_fsp(self):
56+
self._test_convert_timedelta(with_negate=False, with_fsp=True)
57+
self._test_convert_timedelta(with_negate=False, with_fsp=True)
58+
59+
def test_convert_time(self):
60+
expected = datetime.time(23, 6, 20)
61+
time_obj = converters.convert_time('23:06:20')
62+
self.assertEqual(time_obj, expected)
63+
64+
def test_convert_time_with_fsp(self):
65+
expected = datetime.time(23, 6, 20, 511581)
66+
time_obj = converters.convert_time('23:06:20.511581')
67+
self.assertEqual(time_obj, expected)

0 commit comments

Comments
 (0)
0