From 1df94d39d7d87410e6a1053266c9d8f32c52c7ae Mon Sep 17 00:00:00 2001 From: Timon Wong Date: Tue, 19 Jul 2016 16:11:28 +0800 Subject: [PATCH] Avoid IEEE754 converting fraction of seconds Due to the limitations of IEEE754, we may lost precision when converting string back to datetime objects, for example: `float('0.511581') * 1e6`, gives us `0.511580`, which is unexpected. In order to address that issue, use of `float` is avoided --- pymysql/converters.py | 68 ++++++++++++++++++++------------ pymysql/tests/test_converters.py | 44 +++++++++++++++++++++ 2 files changed, 86 insertions(+), 26 deletions(-) diff --git a/pymysql/converters.py b/pymysql/converters.py index 8f8e55d4..4c00bc45 100644 --- a/pymysql/converters.py +++ b/pymysql/converters.py @@ -2,6 +2,7 @@ import datetime from decimal import Decimal +import re import time from .constants import FIELD_TYPE, FLAG @@ -145,6 +146,16 @@ def escape_date(obj, mapping=None): def escape_struct_time(obj, mapping=None): return escape_datetime(datetime.datetime(*obj[:6])) +def _convert_second_fraction(s): + if not s: + return 0 + # Pad zeros to ensure the fraction length in microseconds + s = s.ljust(6, '0') + return int(s[:6]) + +DATETIME_RE = re.compile(r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") + + def convert_datetime(obj): """Returns a DATETIME or TIMESTAMP column value as a datetime object: @@ -163,23 +174,20 @@ def convert_datetime(obj): """ if not PY2 and isinstance(obj, (bytes, bytearray)): obj = obj.decode('ascii') - if ' ' in obj: - sep = ' ' - elif 'T' in obj: - sep = 'T' - else: + + m = DATETIME_RE.match(obj) + if not m: return convert_date(obj) try: - ymd, hms = obj.split(sep, 1) - usecs = '0' - if '.' in hms: - hms, usecs = hms.split('.') - usecs = float('0.' + usecs) * 1e6 - return datetime.datetime(*[ int(x) for x in ymd.split('-')+hms.split(':')+[usecs] ]) + groups = list(m.groups()) + groups[-1] = _convert_second_fraction(groups[-1]) + return datetime.datetime(*[ int(x) for x in groups ]) except ValueError: return convert_date(obj) +TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") + def convert_timedelta(obj): """Returns a TIME column as a timedelta object: @@ -200,16 +208,17 @@ def convert_timedelta(obj): """ if not PY2 and isinstance(obj, (bytes, bytearray)): obj = obj.decode('ascii') + + m = TIMEDELTA_RE.match(obj) + if not m: + return None + try: - microseconds = 0 - if "." in obj: - (obj, tail) = obj.split('.') - microseconds = float('0.' + tail) * 1e6 - hours, minutes, seconds = obj.split(':') - negate = 1 - if hours.startswith("-"): - hours = hours[1:] - negate = -1 + groups = list(m.groups()) + groups[-1] = _convert_second_fraction(groups[-1]) + negate = -1 if groups[0] else 1 + hours, minutes, seconds, microseconds = groups[1:] + tdelta = datetime.timedelta( hours = int(hours), minutes = int(minutes), @@ -220,6 +229,9 @@ def convert_timedelta(obj): except ValueError: return None +TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?") + + def convert_time(obj): """Returns a TIME column as a time object: @@ -244,17 +256,21 @@ def convert_time(obj): """ if not PY2 and isinstance(obj, (bytes, bytearray)): obj = obj.decode('ascii') + + m = TIME_RE.match(obj) + if not m: + return None + try: - microseconds = 0 - if "." in obj: - (obj, tail) = obj.split('.') - microseconds = float('0.' + tail) * 1e6 - hours, minutes, seconds = obj.split(':') + groups = list(m.groups()) + groups[-1] = _convert_second_fraction(groups[-1]) + hours, minutes, seconds, microseconds = groups return datetime.time(hour=int(hours), minute=int(minutes), second=int(seconds), microsecond=int(microseconds)) except ValueError: return None + def convert_date(obj): """Returns a DATE column as a date object: @@ -324,7 +340,7 @@ def through(x): #def convert_bit(b): # b = "\x00" * (8 - len(b)) + b # pad w/ zeroes # return struct.unpack(">Q", b)[0] -# +# # the snippet above is right, but MySQLdb doesn't process bits, # so we shouldn't either convert_bit = through diff --git a/pymysql/tests/test_converters.py b/pymysql/tests/test_converters.py index e15e0cb5..b7b5a984 100644 --- a/pymysql/tests/test_converters.py +++ b/pymysql/tests/test_converters.py @@ -1,3 +1,4 @@ +import datetime from unittest import TestCase from pymysql._compat import PY2 @@ -21,3 +22,46 @@ def test_escape_string_bytes(self): converters.escape_string(b"foo\nbar"), b"foo\\nbar" ) + + def test_convert_datetime(self): + expected = datetime.datetime(2007, 2, 24, 23, 6, 20) + dt = converters.convert_datetime('2007-02-24 23:06:20') + self.assertEqual(dt, expected) + + def test_convert_datetime_with_fsp(self): + expected = datetime.datetime(2007, 2, 24, 23, 6, 20, 511581) + dt = converters.convert_datetime('2007-02-24 23:06:20.511581') + self.assertEqual(dt, expected) + + def _test_convert_timedelta(self, with_negate=False, with_fsp=False): + d = {'hours': 789, 'minutes': 12, 'seconds': 34} + s = '%(hours)s:%(minutes)s:%(seconds)s' % d + if with_fsp: + d['microseconds'] = 511581 + s += '.%(microseconds)s' % d + + expected = datetime.timedelta(**d) + if with_negate: + expected = -expected + s = '-' + s + + tdelta = converters.convert_timedelta(s) + self.assertEqual(tdelta, expected) + + def test_convert_timedelta(self): + self._test_convert_timedelta(with_negate=False, with_fsp=False) + self._test_convert_timedelta(with_negate=True, with_fsp=False) + + def test_convert_timedelta_with_fsp(self): + self._test_convert_timedelta(with_negate=False, with_fsp=True) + self._test_convert_timedelta(with_negate=False, with_fsp=True) + + def test_convert_time(self): + expected = datetime.time(23, 6, 20) + time_obj = converters.convert_time('23:06:20') + self.assertEqual(time_obj, expected) + + def test_convert_time_with_fsp(self): + expected = datetime.time(23, 6, 20, 511581) + time_obj = converters.convert_time('23:06:20.511581') + self.assertEqual(time_obj, expected)